From dac265209899c9d97dfcbfd204b24b4e9f950834 Mon Sep 17 00:00:00 2001 From: crystall-bitquill <97126568+crystall-bitquill@users.noreply.github.com> Date: Thu, 31 Oct 2024 09:28:36 -0700 Subject: [PATCH] test: benchmarks with telemetry (#233) --- package.json | 4 +- tests/benchmarks.ts | 100 -------- tests/plugin_benchmarks.ts | 43 +++- tests/plugin_manager_benchmarks.ts | 41 +++- tests/plugin_manager_telemetry_benchmarks.ts | 244 +++++++++++++++++++ tests/plugin_telemetry_benchmarks.ts | 193 +++++++++++++++ tests/testplugin/benchmark_plugin.ts | 4 +- 7 files changed, 508 insertions(+), 121 deletions(-) delete mode 100644 tests/benchmarks.ts create mode 100644 tests/plugin_manager_telemetry_benchmarks.ts create mode 100644 tests/plugin_telemetry_benchmarks.ts diff --git a/package.json b/package.json index 8a2f31ec..3741d85f 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,11 @@ "mysql" ], "scripts": { - "bench": "npx tsx tests/benchmarks.ts", - "bench2": " npx tsx --tsconfig ./tsconfig.json tests/benchmarks.ts", "check": "prettier . --check --config .prettierrc", "bench-plugin-manager": "npx tsx tests/plugin_manager_benchmarks.ts", "bench-plugins": "npx tsx tests/plugin_benchmarks.ts", + "bench-plugin-manager-otel": "npx tsx tests/plugin_manager_telemetry_benchmarks.ts", + "bench-plugins-otel": "npx tsx tests/plugin_telemetry_benchmarks.ts", "format": "prettier . --write --config .prettierrc", "integration": "jest --config=jest.integration.config.json --runInBand --verbose", "debug-integration": "node --inspect-brk=0.0.0.0:5005 ./node_modules/jest/bin/jest.js --config=jest.integration.config.json --runInBand", diff --git a/tests/benchmarks.ts b/tests/benchmarks.ts deleted file mode 100644 index 6d8f5193..00000000 --- a/tests/benchmarks.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import dotenv from "dotenv"; -import { add, complete, cycle, save, suite } from "benny"; -import { Client } from "pg"; -import { AwsPGClient } from "../pg/lib"; -import { AwsMySQLClient } from "../mysql/lib"; -import { createConnection } from "mysql2"; - -dotenv.config(); - -const PG_DB_USER = process.env.PG_DB_USER; -const PG_DB_HOST = process.env.PG_DB_HOST; -const PG_DB_PASSWORD = process.env.PG_DB_PASSWORD; -const PG_DB_NAME = process.env.PG_DB_NAME; -const MYSQL_DB_USER = process.env.MYSQL_DB_USER; -const MYSQL_DB_HOST = process.env.MYSQL_DB_HOST; -const MYSQL_DB_PASSWORD = process.env.MYSQL_DB_PASSWORD; -const MYSQL_DB_NAME = process.env.MYSQL_DB_NAME; - -const pgClient = new Client({ - user: PG_DB_USER, - host: PG_DB_HOST, - database: PG_DB_NAME, - password: PG_DB_PASSWORD, - port: 5432 -}); - -const mysqlClient = createConnection({ - user: MYSQL_DB_USER, - host: MYSQL_DB_HOST, - database: MYSQL_DB_NAME, - password: MYSQL_DB_PASSWORD, - port: 3306 -}); - -const pgWrapperClient = new AwsPGClient({ - user: PG_DB_USER, - host: PG_DB_HOST, - database: PG_DB_NAME, - password: PG_DB_PASSWORD, - port: 5432 -}); - -const mysqlWrapperClient = new AwsMySQLClient({ - user: MYSQL_DB_USER, - host: MYSQL_DB_HOST, - database: MYSQL_DB_NAME, - password: MYSQL_DB_PASSWORD, - port: 3306 -}); - -pgClient.connect(); -pgWrapperClient.connect(); -mysqlClient.connect(); -mysqlWrapperClient.connect(); - -suite( - "Benchmarks", - - add("pg baseline", async () => { - await pgClient.query("select 1"); - }), - - add("pg execute pipeline", async () => { - await pgWrapperClient.query("select 1"); - }), - - add("mysql baseline", () => { - mysqlClient.query("select 1"); - }), - - add("mysql execute pipeline", async () => { - await mysqlWrapperClient.query({ sql: "select 1" }); - }), - - cycle(), - complete(async () => { - await pgClient.end(); - await pgWrapperClient.end(); - mysqlClient.end(); - await mysqlWrapperClient.end(); - }), - save({ file: "benchmarks", format: "json", details: true }), - save({ file: "benchmarks", format: "chart.html", details: true }) -); diff --git a/tests/plugin_benchmarks.ts b/tests/plugin_benchmarks.ts index fb35dad8..099cd502 100644 --- a/tests/plugin_benchmarks.ts +++ b/tests/plugin_benchmarks.ts @@ -24,8 +24,10 @@ import { add, complete, configure, cycle, save, suite } from "benny"; import { TestConnectionWrapper } from "./testplugin/test_connection_wrapper"; import { HostInfoBuilder } from "../common/lib/host_info_builder"; import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; -import { ClientWrapper } from "../common/lib/client_wrapper"; import { AwsPGClient } from "../pg/lib"; +import { NullTelemetryFactory } from "../common/lib/utils/telemetry/null_telemetry_factory"; +import { ConnectionProviderManager } from "../common/lib/connection_provider_manager"; +import { PgClientWrapper } from "../common/lib/pg_client_wrapper"; const mockConnectionProvider = mock(); const mockPluginService = mock(PluginService); @@ -33,14 +35,14 @@ const mockClient = mock(AwsPGClient); const hostInfo = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(); -const mockClientWrapper: ClientWrapper = { - client: instance(mockClient), - hostInfo: hostInfo, - properties: new Map() -}; +const mockClientWrapper = new PgClientWrapper(instance(mockClient), hostInfo, new Map()); + +const telemetryFactory = new NullTelemetryFactory(); when(mockClient.query(anything())).thenReturn(); when(mockPluginService.getCurrentHostInfo()).thenReturn(hostInfo); +when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); +when(mockPluginService.getCurrentClient()).thenReturn(mockClientWrapper.client); const connectionString = "my.domain.com"; const pluginServiceManagerContainer = new PluginServiceManagerContainer(); @@ -57,9 +59,24 @@ WrapperProperties.HOST.set(propsExecute, connectionString); WrapperProperties.HOST.set(propsReadWrite, connectionString); WrapperProperties.HOST.set(props, connectionString); -const pluginManagerExecute = new PluginManager(pluginServiceManagerContainer, propsExecute, instance(mockConnectionProvider), null); -const pluginManagerReadWrite = new PluginManager(pluginServiceManagerContainer, propsReadWrite, instance(mockConnectionProvider), null); -const pluginManager = new PluginManager(pluginServiceManagerContainer, props, instance(mockConnectionProvider), null); +const pluginManagerExecute = new PluginManager( + pluginServiceManagerContainer, + propsExecute, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); +const pluginManagerReadWrite = new PluginManager( + pluginServiceManagerContainer, + propsReadWrite, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); +const pluginManager = new PluginManager( + pluginServiceManagerContainer, + props, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + new NullTelemetryFactory() +); suite( "Plugin benchmarks", @@ -92,14 +109,15 @@ suite( }), add("executeStatementBaseline", async () => { - const wrapper = new TestConnectionWrapper(propsExecute, pluginManagerExecute, instance(mockPluginService)); - await pluginManagerReadWrite.init(); + const wrapper = new TestConnectionWrapper(props, pluginManager, instance(mockPluginService)); + await pluginManager.init(); + await wrapper.executeQuery(propsExecute, "select 1", mockClientWrapper); await wrapper.end(); }), add("executeStatementWithExecuteTimePlugin", async () => { const wrapper = new TestConnectionWrapper(propsExecute, pluginManagerExecute, instance(mockPluginService)); - await pluginManagerReadWrite.init(); + await pluginManagerExecute.init(); await wrapper.executeQuery(propsExecute, "select 1", mockClientWrapper); await wrapper.end(); }), @@ -107,5 +125,6 @@ suite( cycle(), complete(), save({ file: "plugin_benchmarks", format: "json", details: true }), + save({ file: "plugin_benchmarks", format: "csv", details: true }), save({ file: "plugin_benchmarks", format: "chart.html", details: true }) ); diff --git a/tests/plugin_manager_benchmarks.ts b/tests/plugin_manager_benchmarks.ts index 76ff7531..1372738e 100644 --- a/tests/plugin_manager_benchmarks.ts +++ b/tests/plugin_manager_benchmarks.ts @@ -17,7 +17,7 @@ import { add, cycle, suite, save, complete, configure } from "benny"; import { ConnectionPlugin, PluginManager } from "../common/lib"; import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; -import { instance, mock } from "ts-mockito"; +import { instance, mock, when } from "ts-mockito"; import { ConnectionProvider } from "../common/lib/connection_provider"; import { HostInfoBuilder } from "../common/lib/host_info_builder"; import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; @@ -27,10 +27,18 @@ import { HostChangeOptions } from "../common/lib/host_change_options"; import { WrapperProperties } from "../common/lib/wrapper_property"; import { DefaultPlugin } from "../common/lib/plugins/default_plugin"; import { BenchmarkPluginFactory } from "./testplugin/benchmark_plugin_factory"; +import { NullTelemetryFactory } from "../common/lib/utils/telemetry/null_telemetry_factory"; +import { ConnectionProviderManager } from "../common/lib/connection_provider_manager"; +import { PgDatabaseDialect } from "../pg/lib/dialect/pg_database_dialect"; +import { NodePostgresDriverDialect } from "../pg/lib/dialect/node_postgres_driver_dialect"; const mockConnectionProvider = mock(); const mockHostListProviderService = mock(); const mockPluginService = mock(PluginService); +const telemetryFactory = new NullTelemetryFactory(); +when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); +when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); +when(mockPluginService.getDriverDialect()).thenReturn(new NodePostgresDriverDialect()); const pluginServiceManagerContainer = new PluginServiceManagerContainer(); pluginServiceManagerContainer.pluginService = instance(mockPluginService); @@ -40,15 +48,25 @@ const propsWithPlugins = new Map(); WrapperProperties.PLUGINS.set(propsWithNoPlugins, ""); -const pluginManagerWithNoPlugins = new PluginManager(pluginServiceManagerContainer, propsWithNoPlugins, instance(mockConnectionProvider), null); -const pluginManagerWithPlugins = new PluginManager(pluginServiceManagerContainer, propsWithPlugins, instance(mockConnectionProvider), null); +const pluginManagerWithNoPlugins = new PluginManager( + pluginServiceManagerContainer, + propsWithNoPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); +const pluginManagerWithPlugins = new PluginManager( + pluginServiceManagerContainer, + propsWithPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); async function createPlugins(pluginService: PluginService, connectionProvider: ConnectionProvider, props: Map) { const plugins = new Array(); for (let i = 0; i < 10; i++) { plugins.push(await new BenchmarkPluginFactory().getInstance(pluginService, props)); } - plugins.push(new DefaultPlugin(pluginService, connectionProvider, null)); + plugins.push(new DefaultPlugin(pluginService, new ConnectionProviderManager(instance(mockConnectionProvider), null))); return plugins; } @@ -62,12 +80,22 @@ suite( }), add("initPluginManagerWithPlugins", async () => { - const manager = new PluginManager(pluginServiceManagerContainer, propsWithPlugins, instance(mockConnectionProvider), null); + const manager = new PluginManager( + pluginServiceManagerContainer, + propsWithPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + new NullTelemetryFactory() + ); await manager.init(await createPlugins(instance(mockPluginService), instance(mockConnectionProvider), propsWithPlugins)); }), add("initPluginManagerWithNoPlugins", async () => { - const manager = new PluginManager(pluginServiceManagerContainer, propsWithNoPlugins, instance(mockConnectionProvider), null); + const manager = new PluginManager( + pluginServiceManagerContainer, + propsWithNoPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + new NullTelemetryFactory() + ); await manager.init(); }), @@ -150,5 +178,6 @@ suite( cycle(), complete(), save({ file: "plugin_manager_benchmarks", format: "json", details: true }), + save({ file: "plugin_manager_benchmarks", format: "csv", details: true }), save({ file: "plugin_manager_benchmarks", format: "chart.html", details: true }) ); diff --git a/tests/plugin_manager_telemetry_benchmarks.ts b/tests/plugin_manager_telemetry_benchmarks.ts new file mode 100644 index 00000000..9bd7a8b5 --- /dev/null +++ b/tests/plugin_manager_telemetry_benchmarks.ts @@ -0,0 +1,244 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { add, cycle, suite, save, complete, configure } from "benny"; +import { ConnectionPlugin, PluginManager } from "../common/lib"; +import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; +import { instance, mock, when } from "ts-mockito"; +import { ConnectionProvider } from "../common/lib/connection_provider"; +import { HostInfoBuilder } from "../common/lib/host_info_builder"; +import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; +import { PluginService } from "../common/lib/plugin_service"; +import { HostListProviderService } from "../common/lib/host_list_provider_service"; +import { HostChangeOptions } from "../common/lib/host_change_options"; +import { WrapperProperties } from "../common/lib/wrapper_property"; +import { DefaultPlugin } from "../common/lib/plugins/default_plugin"; +import { BenchmarkPluginFactory } from "./testplugin/benchmark_plugin_factory"; +import { NullTelemetryFactory } from "../common/lib/utils/telemetry/null_telemetry_factory"; +import { OpenTelemetryFactory } from "../common/lib/utils/telemetry/open_telemetry_factory"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; +import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc"; +import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; +import { context } from "@opentelemetry/api"; +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { AWSXRayPropagator } from "@opentelemetry/propagator-aws-xray"; +import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; +import { AwsInstrumentation } from "@opentelemetry/instrumentation-aws-sdk"; +import { AWSXRayIdGenerator } from "@opentelemetry/id-generator-aws-xray"; +import { ConnectionProviderManager } from "../common/lib/connection_provider_manager"; +import { PgDatabaseDialect } from "../pg/lib/dialect/pg_database_dialect"; +import { NodePostgresDriverDialect } from "../pg/lib/dialect/node_postgres_driver_dialect"; + +const mockConnectionProvider = mock(); +const mockHostListProviderService = mock(); +const mockPluginService = mock(PluginService); +const telemetryFactory = new OpenTelemetryFactory(); +when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); +when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); +when(mockPluginService.getDriverDialect()).thenReturn(new NodePostgresDriverDialect()); + +const pluginServiceManagerContainer = new PluginServiceManagerContainer(); +pluginServiceManagerContainer.pluginService = instance(mockPluginService); + +const propsWithNoPlugins = new Map(); +const propsWithPlugins = new Map(); + +WrapperProperties.PLUGINS.set(propsWithNoPlugins, ""); +WrapperProperties.ENABLE_TELEMETRY.set(propsWithNoPlugins, true); +WrapperProperties.TELEMETRY_METRICS_BACKEND.set(propsWithNoPlugins, "OTLP"); +WrapperProperties.TELEMETRY_TRACES_BACKEND.set(propsWithNoPlugins, "OTLP"); +WrapperProperties.ENABLE_TELEMETRY.set(propsWithPlugins, true); +WrapperProperties.TELEMETRY_METRICS_BACKEND.set(propsWithPlugins, "OTLP"); +WrapperProperties.TELEMETRY_TRACES_BACKEND.set(propsWithPlugins, "OTLP"); + +const pluginManagerWithNoPlugins = new PluginManager( + pluginServiceManagerContainer, + propsWithNoPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); +const pluginManagerWithPlugins = new PluginManager( + pluginServiceManagerContainer, + propsWithPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); + +async function createPlugins(pluginService: PluginService, connectionProvider: ConnectionProvider, props: Map) { + const plugins = new Array(); + for (let i = 0; i < 10; i++) { + plugins.push(await new BenchmarkPluginFactory().getInstance(pluginService, props)); + } + plugins.push(new DefaultPlugin(pluginService, new ConnectionProviderManager(instance(mockConnectionProvider), null))); + return plugins; +} + +const traceExporter = new OTLPTraceExporter({ url: "http://localhost:4317" }); +const resource = Resource.default().merge( + new Resource({ + [ATTR_SERVICE_NAME]: "aws-advanced-nodejs-wrapper" + }) +); + +const metricReader = new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter(), + exportIntervalMillis: 1000 +}); + +const contextManager = new AsyncHooksContextManager(); +contextManager.enable(); +context.setGlobalContextManager(contextManager); + +const sdk = new NodeSDK({ + textMapPropagator: new AWSXRayPropagator(), + instrumentations: [ + new HttpInstrumentation(), + new AwsInstrumentation({ + suppressInternalInstrumentation: true + }) + ], + resource: resource, + traceExporter: traceExporter, + metricReader: metricReader, + idGenerator: new AWSXRayIdGenerator() +}); + +// This enables the API to record telemetry. +sdk.start(); + +// Shut down the SDK on process exit. +process.on("SIGTERM", () => { + sdk + .shutdown() + .then(() => console.log("Tracing and Metrics terminated")) + .catch((error) => console.log("Error terminating tracing and metrics", error)) + .finally(() => process.exit(0)); +}); + +suite( + "Plugin Manager Benchmarks", + + configure({ + cases: { + delay: 0.5 + } + }), + + add("initPluginManagerWithPlugins", async () => { + const manager = new PluginManager( + pluginServiceManagerContainer, + propsWithPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + new NullTelemetryFactory() + ); + await manager.init(await createPlugins(instance(mockPluginService), instance(mockConnectionProvider), propsWithPlugins)); + }), + + add("initPluginManagerWithNoPlugins", async () => { + const manager = new PluginManager( + pluginServiceManagerContainer, + propsWithNoPlugins, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + new NullTelemetryFactory() + ); + await manager.init(); + }), + + add("connectWithPlugins", async () => { + await pluginManagerWithPlugins.init(await createPlugins(instance(mockPluginService), instance(mockConnectionProvider), propsWithPlugins)); + await pluginManagerWithPlugins.connect( + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(), + propsWithPlugins, + true + ); + }), + + add("connectWithNoPlugins", async () => { + await pluginManagerWithNoPlugins.init(); + await pluginManagerWithNoPlugins.connect( + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(), + propsWithPlugins, + true + ); + }), + + add("executeWithPlugins", async () => { + await pluginManagerWithPlugins.init(await createPlugins(instance(mockPluginService), instance(mockConnectionProvider), propsWithPlugins)); + await pluginManagerWithPlugins.execute( + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(), + propsWithPlugins, + "execute", + () => Promise.resolve(1), + null + ); + }), + + add("executeWithNoPlugins", async () => { + await pluginManagerWithNoPlugins.init(); + await pluginManagerWithNoPlugins.execute( + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(), + propsWithNoPlugins, + "execute", + () => Promise.resolve(1), + null + ); + }), + + add("initHostProviderWithPlugins", async () => { + await pluginManagerWithPlugins.init(await createPlugins(instance(mockPluginService), instance(mockConnectionProvider), propsWithPlugins)); + await pluginManagerWithPlugins.initHostProvider( + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(), + propsWithPlugins, + instance(mockHostListProviderService) + ); + }), + + add("initHostProvidersWithNoPlugins", async () => { + await pluginManagerWithNoPlugins.init(); + await pluginManagerWithNoPlugins.initHostProvider( + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(), + propsWithNoPlugins, + instance(mockHostListProviderService) + ); + }), + + add("notifyConnectionChangedWithPlugins", async () => { + await pluginManagerWithPlugins.init(await createPlugins(instance(mockPluginService), instance(mockConnectionProvider), propsWithPlugins)); + await pluginManagerWithPlugins.notifyConnectionChanged(new Set([HostChangeOptions.INITIAL_CONNECTION]), null); + }), + + add("notifyConnectionChangedWithNoPlugins", async () => { + await pluginManagerWithNoPlugins.init(); + await pluginManagerWithNoPlugins.notifyConnectionChanged(new Set([HostChangeOptions.INITIAL_CONNECTION]), null); + }), + + add("releaseResourcesWithPlugins", async () => { + await pluginManagerWithPlugins.releaseResources(); + }), + + add("releaseResourcesWithNoPlugins", async () => { + await pluginManagerWithNoPlugins.releaseResources(); + }), + + cycle(), + complete(), + save({ file: "plugin_manager_benchmarks", format: "json", details: true }), + save({ file: "plugin_manager_benchmarks", format: "csv", details: true }), + save({ file: "plugin_manager_benchmarks", format: "chart.html", details: true }) +); diff --git a/tests/plugin_telemetry_benchmarks.ts b/tests/plugin_telemetry_benchmarks.ts new file mode 100644 index 00000000..a395ddb2 --- /dev/null +++ b/tests/plugin_telemetry_benchmarks.ts @@ -0,0 +1,193 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { anything, instance, mock, when } from "ts-mockito"; +import { ConnectionProvider } from "../common/lib/connection_provider"; +import { PluginService } from "../common/lib/plugin_service"; +import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; +import { WrapperProperties } from "../common/lib/wrapper_property"; +import { PluginManager } from "../common/lib"; +import { add, complete, configure, cycle, save, suite } from "benny"; +import { TestConnectionWrapper } from "./testplugin/test_connection_wrapper"; +import { HostInfoBuilder } from "../common/lib/host_info_builder"; +import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; +import { AwsPGClient } from "../pg/lib"; +import { NullTelemetryFactory } from "../common/lib/utils/telemetry/null_telemetry_factory"; +import { OpenTelemetryFactory } from "../common/lib/utils/telemetry/open_telemetry_factory"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; +import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc"; +import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; +import { context } from "@opentelemetry/api"; +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { AWSXRayPropagator } from "@opentelemetry/propagator-aws-xray"; +import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; +import { AwsInstrumentation } from "@opentelemetry/instrumentation-aws-sdk"; +import { AWSXRayIdGenerator } from "@opentelemetry/id-generator-aws-xray"; +import { ConnectionProviderManager } from "../common/lib/connection_provider_manager"; +import { PgClientWrapper } from "../common/lib/pg_client_wrapper"; + +const mockConnectionProvider = mock(); +const mockPluginService = mock(PluginService); +const mockClient = mock(AwsPGClient); + +const hostInfo = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("host").build(); + +const mockClientWrapper = new PgClientWrapper(instance(mockClient), hostInfo, new Map()); + +const telemetryFactory = new OpenTelemetryFactory(); + +when(mockClient.query(anything())).thenReturn(); +when(mockPluginService.getCurrentHostInfo()).thenReturn(hostInfo); +when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); +when(mockPluginService.getCurrentClient()).thenReturn(mockClientWrapper.client); + +const connectionString = "my.domain.com"; +const pluginServiceManagerContainer = new PluginServiceManagerContainer(); +pluginServiceManagerContainer.pluginService = instance(mockPluginService); + +const propsExecute = new Map(); +const propsReadWrite = new Map(); +const props = new Map(); + +WrapperProperties.PLUGINS.set(propsExecute, "executeTime"); +WrapperProperties.PLUGINS.set(propsReadWrite, "readWriteSplitting"); +WrapperProperties.PLUGINS.set(props, ""); +WrapperProperties.HOST.set(propsExecute, connectionString); +WrapperProperties.HOST.set(propsReadWrite, connectionString); +WrapperProperties.HOST.set(props, connectionString); +WrapperProperties.ENABLE_TELEMETRY.set(propsExecute, true); +WrapperProperties.ENABLE_TELEMETRY.set(propsExecute, true); +WrapperProperties.ENABLE_TELEMETRY.set(propsExecute, true); +WrapperProperties.TELEMETRY_METRICS_BACKEND.set(propsExecute, "OTLP"); +WrapperProperties.TELEMETRY_METRICS_BACKEND.set(propsReadWrite, "OTLP"); +WrapperProperties.TELEMETRY_METRICS_BACKEND.set(props, "OTLP"); +WrapperProperties.TELEMETRY_TRACES_BACKEND.set(propsExecute, "OTLP"); +WrapperProperties.TELEMETRY_TRACES_BACKEND.set(propsReadWrite, "OTLP"); +WrapperProperties.TELEMETRY_TRACES_BACKEND.set(props, "OTLP"); + +const pluginManagerExecute = new PluginManager( + pluginServiceManagerContainer, + propsExecute, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); +const pluginManagerReadWrite = new PluginManager( + pluginServiceManagerContainer, + propsReadWrite, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + telemetryFactory +); +const pluginManager = new PluginManager( + pluginServiceManagerContainer, + props, + new ConnectionProviderManager(instance(mockConnectionProvider), null), + new NullTelemetryFactory() +); + +const traceExporter = new OTLPTraceExporter({ url: "http://localhost:4317" }); +const resource = Resource.default().merge( + new Resource({ + [ATTR_SERVICE_NAME]: "aws-advanced-nodejs-wrapper" + }) +); + +const metricReader = new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter(), + exportIntervalMillis: 1000 +}); + +const contextManager = new AsyncHooksContextManager(); +contextManager.enable(); +context.setGlobalContextManager(contextManager); + +const sdk = new NodeSDK({ + textMapPropagator: new AWSXRayPropagator(), + instrumentations: [ + new HttpInstrumentation(), + new AwsInstrumentation({ + suppressInternalInstrumentation: true + }) + ], + resource: resource, + traceExporter: traceExporter, + metricReader: metricReader, + idGenerator: new AWSXRayIdGenerator() +}); + +// This enables the API to record telemetry. +sdk.start(); + +// Shut down the SDK on process exit. +process.on("SIGTERM", () => { + sdk + .shutdown() + .then(() => console.log("Tracing and Metrics terminated")) + .catch((error) => console.log("Error terminating tracing and metrics", error)) + .finally(() => process.exit(0)); +}); + +suite( + "Plugin benchmarks", + + configure({ + cases: { + delay: 0.5 + } + }), + + add("initAndReleaseBaseline", async () => { + const wrapper = new TestConnectionWrapper(props, pluginManager, instance(mockPluginService)); + await pluginManager.init(); + await wrapper.releaseResources(); + await wrapper.end(); + }), + + add("initAndReleaseWithExecuteTimePlugin", async () => { + const wrapper = new TestConnectionWrapper(propsExecute, pluginManagerExecute, instance(mockPluginService)); + await pluginManagerExecute.init(); + await wrapper.releaseResources(); + await wrapper.end(); + }), + + add("initAndReleaseWithReadWriteSplittingPlugin", async () => { + const wrapper = new TestConnectionWrapper(propsReadWrite, pluginManagerReadWrite, instance(mockPluginService)); + await pluginManagerReadWrite.init(); + await wrapper.releaseResources(); + await wrapper.end(); + }), + + add("executeStatementBaseline", async () => { + const wrapper = new TestConnectionWrapper(propsExecute, pluginManagerExecute, instance(mockPluginService)); + await pluginManagerExecute.init(); + await wrapper.end(); + }), + + add("executeStatementWithExecuteTimePlugin", async () => { + const wrapper = new TestConnectionWrapper(propsExecute, pluginManagerExecute, instance(mockPluginService)); + await pluginManagerExecute.init(); + await wrapper.executeQuery(propsExecute, "select 1", mockClientWrapper); + await wrapper.end(); + }), + + cycle(), + complete(), + save({ file: "plugin_benchmarks", format: "json", details: true }), + save({ file: "plugin_benchmarks", format: "csv", details: true }), + save({ file: "plugin_benchmarks", format: "chart.html", details: true }) +); diff --git a/tests/testplugin/benchmark_plugin.ts b/tests/testplugin/benchmark_plugin.ts index 5de62a76..5e41609e 100644 --- a/tests/testplugin/benchmark_plugin.ts +++ b/tests/testplugin/benchmark_plugin.ts @@ -25,6 +25,7 @@ import { HostInfoBuilder } from "../../common/lib/host_info_builder"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; export class BenchmarkPlugin implements ConnectionPlugin { + name: string = this.constructor.name; resources: Array = new Array(); getSubscribedMethods(): Set { @@ -64,9 +65,10 @@ export class BenchmarkPlugin implements ConnectionPlugin { return Promise.resolve(OldConnectionSuggestionAction.NO_OPINION); } - notifyHostListChanged(changes: Map>): void { + notifyHostListChanged(changes: Map>): Promise { logger.debug(`notifyHostListChanged = ${JSON.stringify(Array.from(changes))}`); this.resources.push("notifyHostListChanged"); + return Promise.resolve(); } acceptsStrategy(role: HostRole, strategy: string): boolean {