From a9f9be1efa270f725def642a7dc239a3a392f5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Fri, 18 Oct 2024 11:11:22 +0200 Subject: [PATCH] chore: add a class to handle aggreggation queries (#8446) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## About the changes We have many aggregation queries that run on a schedule: https://github.com/Unleash/unleash/blob/f63496d47f6808a246732f5ddee9b9136e23eca7/src/lib/metrics.ts#L714-L719 These staticCounters are usually doing db query aggregations that traverse tables and we run all of them in parallel: https://github.com/Unleash/unleash/blob/f63496d47f6808a246732f5ddee9b9136e23eca7/src/lib/metrics.ts#L410-L412 This can add strain to the db. This PR suggests a way of handling these queries in a more structured way, allowing us to run them sequentially (therefore spreading the load): https://github.com/Unleash/unleash/blob/f02fe87835c30d77ff8b6755268e123585820511/src/lib/metrics-gauge.ts#L38-L40 As an additional benefit, we get both the gauge definition and the queries in a single place: https://github.com/Unleash/unleash/blob/f02fe87835c30d77ff8b6755268e123585820511/src/lib/metrics.ts#L131-L141 This PR only tackles 1 metric, and it only focuses on gauges to gather initial feedback. The plan is to migrate these metrics and eventually incorporate more types (e.g. counters) --------- Co-authored-by: Nuno Góis --- .../instance-stats-service.test.ts | 42 +- .../instance-stats/instance-stats-service.ts | 108 +- .../features/scheduler/schedule-services.ts | 6 +- src/lib/metrics-gauge.test.ts | 114 ++ src/lib/metrics-gauge.ts | 94 + src/lib/metrics.test.ts | 31 +- src/lib/metrics.ts | 1707 +++++++++-------- .../e2e/api/admin/instance-admin.e2e.test.ts | 13 + 8 files changed, 1204 insertions(+), 911 deletions(-) create mode 100644 src/lib/metrics-gauge.test.ts create mode 100644 src/lib/metrics-gauge.ts diff --git a/src/lib/features/instance-stats/instance-stats-service.test.ts b/src/lib/features/instance-stats/instance-stats-service.test.ts index d3f423bec1a6..26a5f7009502 100644 --- a/src/lib/features/instance-stats/instance-stats-service.test.ts +++ b/src/lib/features/instance-stats/instance-stats-service.test.ts @@ -4,11 +4,18 @@ import createStores from '../../../test/fixtures/store'; import VersionService from '../../services/version-service'; import { createFakeGetActiveUsers } from './getActiveUsers'; import { createFakeGetProductionChanges } from './getProductionChanges'; - +import { registerPrometheusMetrics } from '../../metrics'; +import { register } from 'prom-client'; +import type { IClientInstanceStore } from '../../types'; let instanceStatsService: InstanceStatsService; let versionService: VersionService; - +let clientInstanceStore: IClientInstanceStore; +let updateMetrics: () => Promise; beforeEach(() => { + jest.clearAllMocks(); + + register.clear(); + const config = createTestConfig(); const stores = createStores(); versionService = new VersionService( @@ -17,6 +24,7 @@ beforeEach(() => { createFakeGetActiveUsers(), createFakeGetProductionChanges(), ); + clientInstanceStore = stores.clientInstanceStore; instanceStatsService = new InstanceStatsService( stores, config, @@ -25,23 +33,28 @@ beforeEach(() => { createFakeGetProductionChanges(), ); - jest.spyOn(instanceStatsService, 'refreshAppCountSnapshot'); - jest.spyOn(instanceStatsService, 'getLabeledAppCounts'); + const { collectDbMetrics } = registerPrometheusMetrics( + config, + stores, + undefined as unknown as string, + config.eventBus, + instanceStatsService, + ); + updateMetrics = collectDbMetrics; + + jest.spyOn(clientInstanceStore, 'getDistinctApplicationsCount'); jest.spyOn(instanceStatsService, 'getStats'); - // validate initial state without calls to these methods - expect(instanceStatsService.refreshAppCountSnapshot).toHaveBeenCalledTimes( - 0, - ); expect(instanceStatsService.getStats).toHaveBeenCalledTimes(0); }); test('get snapshot should not call getStats', async () => { - await instanceStatsService.refreshAppCountSnapshot(); - expect(instanceStatsService.getLabeledAppCounts).toHaveBeenCalledTimes(1); + await updateMetrics(); + expect( + clientInstanceStore.getDistinctApplicationsCount, + ).toHaveBeenCalledTimes(3); expect(instanceStatsService.getStats).toHaveBeenCalledTimes(0); - // subsequent calls to getStatsSnapshot don't call getStats for (let i = 0; i < 3; i++) { const { clientApps } = await instanceStatsService.getStats(); expect(clientApps).toStrictEqual([ @@ -51,12 +64,11 @@ test('get snapshot should not call getStats', async () => { ]); } // after querying the stats snapshot no call to getStats should be issued - expect(instanceStatsService.getLabeledAppCounts).toHaveBeenCalledTimes(1); + expect( + clientInstanceStore.getDistinctApplicationsCount, + ).toHaveBeenCalledTimes(3); }); test('before the snapshot is refreshed we can still get the appCount', async () => { - expect(instanceStatsService.refreshAppCountSnapshot).toHaveBeenCalledTimes( - 0, - ); expect(instanceStatsService.getAppCountSnapshot('7d')).toBeUndefined(); }); diff --git a/src/lib/features/instance-stats/instance-stats-service.ts b/src/lib/features/instance-stats/instance-stats-service.ts index 3fb505fd6480..24e04ef3dfaa 100644 --- a/src/lib/features/instance-stats/instance-stats-service.ts +++ b/src/lib/features/instance-stats/instance-stats-service.ts @@ -109,9 +109,9 @@ export class InstanceStatsService { private appCount?: Partial<{ [key in TimeRange]: number }>; - private getActiveUsers: GetActiveUsers; + getActiveUsers: GetActiveUsers; - private getProductionChanges: GetProductionChanges; + getProductionChanges: GetProductionChanges; private featureStrategiesReadModel: IFeatureStrategiesReadModel; @@ -180,25 +180,6 @@ export class InstanceStatsService { this.featureStrategiesReadModel = featureStrategiesReadModel; } - async refreshAppCountSnapshot(): Promise< - Partial<{ [key in TimeRange]: number }> - > { - try { - this.appCount = await this.getLabeledAppCounts(); - return this.appCount; - } catch (error) { - this.logger.warn( - 'Unable to retrieve statistics. This will be retried', - error, - ); - return { - '7d': 0, - '30d': 0, - allTime: 0, - }; - } - } - getProjectModeCount(): Promise { return this.projectStore.getProjectModeCounts(); } @@ -231,9 +212,6 @@ export class InstanceStatsService { return settings?.enabled || false; } - /** - * use getStatsSnapshot for low latency, sacrificing data-freshness - */ async getStats(): Promise { const versionInfo = await this.versionService.getVersionInfo(); const [ @@ -265,22 +243,22 @@ export class InstanceStatsService { ] = await Promise.all([ this.getToggleCount(), this.getArchivedToggleCount(), - this.userStore.count(), - this.userStore.countServiceAccounts(), - this.apiTokenStore.countByType(), + this.getRegisteredUsers(), + this.countServiceAccounts(), + this.countApiTokensByType(), this.getActiveUsers(), this.getProjectModeCount(), - this.contextFieldStore.count(), - this.groupStore.count(), - this.roleStore.count(), - this.roleStore.filteredCount({ type: CUSTOM_ROOT_ROLE_TYPE }), - this.roleStore.filteredCountInUse({ type: CUSTOM_ROOT_ROLE_TYPE }), - this.environmentStore.count(), - this.segmentStore.count(), - this.strategyStore.count(), + this.contextFieldCount(), + this.groupCount(), + this.roleCount(), + this.customRolesCount(), + this.customRolesCountInUse(), + this.environmentCount(), + this.segmentCount(), + this.strategiesCount(), this.hasSAML(), this.hasOIDC(), - this.appCount ? this.appCount : this.refreshAppCountSnapshot(), + this.appCount ? this.appCount : this.getLabeledAppCounts(), this.eventStore.deprecatedFilteredCount({ type: FEATURES_EXPORTED, }), @@ -288,7 +266,7 @@ export class InstanceStatsService { type: FEATURES_IMPORTED, }), this.getProductionChanges(), - this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets(), + this.countPreviousDayHourlyMetricsBuckets(), this.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(), this.featureStrategiesReadModel.getMaxConstraintValues(), this.featureStrategiesReadModel.getMaxConstraintsPerStrategy(), @@ -330,6 +308,59 @@ export class InstanceStatsService { }; } + groupCount(): Promise { + return this.groupStore.count(); + } + + roleCount(): Promise { + return this.roleStore.count(); + } + + customRolesCount(): Promise { + return this.roleStore.filteredCount({ type: CUSTOM_ROOT_ROLE_TYPE }); + } + + customRolesCountInUse(): Promise { + return this.roleStore.filteredCountInUse({ + type: CUSTOM_ROOT_ROLE_TYPE, + }); + } + + segmentCount(): Promise { + return this.segmentStore.count(); + } + + contextFieldCount(): Promise { + return this.contextFieldStore.count(); + } + + strategiesCount(): Promise { + return this.strategyStore.count(); + } + + environmentCount(): Promise { + return this.environmentStore.count(); + } + + countPreviousDayHourlyMetricsBuckets(): Promise<{ + enabledCount: number; + variantCount: number; + }> { + return this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets(); + } + + countApiTokensByType(): Promise> { + return this.apiTokenStore.countByType(); + } + + getRegisteredUsers(): Promise { + return this.userStore.count(); + } + + countServiceAccounts(): Promise { + return this.userStore.countServiceAccounts(); + } + async getLabeledAppCounts(): Promise< Partial<{ [key in TimeRange]: number }> > { @@ -338,11 +369,12 @@ export class InstanceStatsService { this.clientInstanceStore.getDistinctApplicationsCount(30), this.clientInstanceStore.getDistinctApplicationsCount(), ]); - return { + this.appCount = { '7d': t7d, '30d': t30d, allTime, }; + return this.appCount; } getAppCountSnapshot(range: TimeRange): number | undefined { diff --git a/src/lib/features/scheduler/schedule-services.ts b/src/lib/features/scheduler/schedule-services.ts index 95e03c114b19..69d3a218fc1e 100644 --- a/src/lib/features/scheduler/schedule-services.ts +++ b/src/lib/features/scheduler/schedule-services.ts @@ -59,8 +59,12 @@ export const scheduleServices = async ( 'updateLastSeen', ); + // TODO this works fine for keeping labeledAppCounts up to date, but + // it would be nice if we can keep client_apps_total prometheus metric + // up to date. We'd need to have access to DbMetricsMonitor, which is + // where the metric is registered and call an update only for that metric schedulerService.schedule( - instanceStatsService.refreshAppCountSnapshot.bind(instanceStatsService), + instanceStatsService.getLabeledAppCounts.bind(instanceStatsService), minutesToMilliseconds(5), 'refreshAppCountSnapshot', ); diff --git a/src/lib/metrics-gauge.test.ts b/src/lib/metrics-gauge.test.ts new file mode 100644 index 000000000000..e024563f9628 --- /dev/null +++ b/src/lib/metrics-gauge.test.ts @@ -0,0 +1,114 @@ +import { register } from 'prom-client'; +import { createTestConfig } from '../test/config/test-config'; +import type { IUnleashConfig } from './types'; +import { DbMetricsMonitor } from './metrics-gauge'; + +const prometheusRegister = register; +let config: IUnleashConfig; +let dbMetrics: DbMetricsMonitor; + +beforeAll(async () => { + config = createTestConfig({ + server: { + serverMetrics: true, + }, + }); +}); + +beforeEach(async () => { + dbMetrics = new DbMetricsMonitor(config); +}); + +test('should collect registered metrics', async () => { + dbMetrics.registerGaugeDbMetric({ + name: 'my_metric', + help: 'This is the answer to life, the univers, and everything', + labelNames: [], + query: () => Promise.resolve(42), + map: (result) => ({ value: result }), + }); + + await dbMetrics.refreshDbMetrics(); + + const metrics = await prometheusRegister.metrics(); + expect(metrics).toMatch(/my_metric 42/); +}); + +test('should collect registered metrics with labels', async () => { + dbMetrics.registerGaugeDbMetric({ + name: 'life_the_universe_and_everything', + help: 'This is the answer to life, the univers, and everything', + labelNames: ['test'], + query: () => Promise.resolve(42), + map: (result) => ({ value: result, labels: { test: 'case' } }), + }); + + await dbMetrics.refreshDbMetrics(); + + const metrics = await prometheusRegister.metrics(); + expect(metrics).toMatch( + /life_the_universe_and_everything\{test="case"\} 42/, + ); +}); + +test('should collect multiple registered metrics with and without labels', async () => { + dbMetrics.registerGaugeDbMetric({ + name: 'my_first_metric', + help: 'This is the answer to life, the univers, and everything', + labelNames: [], + query: () => Promise.resolve(42), + map: (result) => ({ value: result }), + }); + + dbMetrics.registerGaugeDbMetric({ + name: 'my_other_metric', + help: 'This is Eulers number', + labelNames: ['euler'], + query: () => Promise.resolve(Math.E), + map: (result) => ({ value: result, labels: { euler: 'number' } }), + }); + + await dbMetrics.refreshDbMetrics(); + + const metrics = await prometheusRegister.metrics(); + expect(metrics).toMatch(/my_first_metric 42/); + expect(metrics).toMatch(/my_other_metric\{euler="number"\} 2.71828/); +}); + +test('should support different label and value pairs', async () => { + dbMetrics.registerGaugeDbMetric({ + name: 'multi_dimensional', + help: 'This metric has different values for different labels', + labelNames: ['version', 'range'], + query: () => Promise.resolve(2), + map: (result) => [ + { value: result, labels: { version: '1', range: 'linear' } }, + { + value: result * result, + labels: { version: '2', range: 'square' }, + }, + { value: result / 2, labels: { version: '3', range: 'half' } }, + ], + }); + + await dbMetrics.refreshDbMetrics(); + + const metrics = await prometheusRegister.metrics(); + expect(metrics).toMatch( + /multi_dimensional\{version="1",range="linear"\} 2\nmulti_dimensional\{version="2",range="square"\} 4\nmulti_dimensional\{version="3",range="half"\} 1/, + ); + expect( + await dbMetrics.findValue('multi_dimensional', { range: 'linear' }), + ).toBe(2); + expect( + await dbMetrics.findValue('multi_dimensional', { range: 'half' }), + ).toBe(1); + expect( + await dbMetrics.findValue('multi_dimensional', { range: 'square' }), + ).toBe(4); + expect( + await dbMetrics.findValue('multi_dimensional', { range: 'x' }), + ).toBeUndefined(); + expect(await dbMetrics.findValue('multi_dimensional')).toBe(2); // first match + expect(await dbMetrics.findValue('other')).toBeUndefined(); +}); diff --git a/src/lib/metrics-gauge.ts b/src/lib/metrics-gauge.ts new file mode 100644 index 000000000000..7f1e5cc06265 --- /dev/null +++ b/src/lib/metrics-gauge.ts @@ -0,0 +1,94 @@ +import type { Logger } from './logger'; +import type { IUnleashConfig } from './types'; +import { createGauge, type Gauge } from './util/metrics'; + +type Query = () => Promise; +type MetricValue = { + value: number; + labels?: Record; +}; +type MapResult = ( + result: R, +) => MetricValue | MetricValue[]; + +type GaugeDefinition = { + name: string; + help: string; + labelNames: L[]; + query: Query; + map: MapResult; +}; + +type Task = () => Promise; + +interface GaugeUpdater { + target: Gauge; + task: Task; +} +export class DbMetricsMonitor { + private updaters: Map = new Map(); + private log: Logger; + + constructor({ getLogger }: Pick) { + this.log = getLogger('gauge-metrics'); + } + + private asArray(value: T | T[]): T[] { + return Array.isArray(value) ? value : [value]; + } + + registerGaugeDbMetric( + definition: GaugeDefinition, + ): Task { + const gauge = createGauge(definition); + const task = async () => { + try { + const result = await definition.query(); + if (result !== null && result !== undefined) { + const results = this.asArray(definition.map(result)); + gauge.reset(); + for (const r of results) { + if (r.labels) { + gauge.labels(r.labels).set(r.value); + } else { + gauge.set(r.value); + } + } + } + } catch (e) { + this.log.warn(`Failed to refresh ${definition.name}`, e); + } + }; + this.updaters.set(definition.name, { target: gauge, task }); + return task; + } + + refreshDbMetrics = async () => { + const tasks = Array.from(this.updaters.entries()).map( + ([name, updater]) => ({ name, task: updater.task }), + ); + for (const { name, task } of tasks) { + this.log.debug(`Refreshing metric ${name}`); + await task(); + } + }; + + async findValue( + name: string, + labels?: Record, + ): Promise { + const gauge = await this.updaters.get(name)?.target.gauge?.get(); + if (gauge && gauge.values.length > 0) { + const values = labels + ? gauge.values.filter(({ labels: l }) => { + return Object.entries(labels).every( + ([key, value]) => l[key] === value, + ); + }) + : gauge.values; + // return first value + return values.map(({ value }) => value).shift(); + } + return undefined; + } +} diff --git a/src/lib/metrics.test.ts b/src/lib/metrics.test.ts index bf92983caf55..8cfe78aa5d0a 100644 --- a/src/lib/metrics.test.ts +++ b/src/lib/metrics.test.ts @@ -15,7 +15,11 @@ import { FEATURE_UPDATED, PROJECT_ENVIRONMENT_REMOVED, } from './types/events'; -import { createMetricsMonitor } from './metrics'; +import { + createMetricsMonitor, + registerPrometheusMetrics, + registerPrometheusPostgresMetrics, +} from './metrics'; import createStores from '../test/fixtures/store'; import { InstanceStatsService } from './features/instance-stats/instance-stats-service'; import VersionService from './services/version-service'; @@ -46,6 +50,7 @@ let schedulerService: SchedulerService; let featureLifeCycleStore: IFeatureLifecycleStore; let featureLifeCycleReadModel: IFeatureLifecycleReadModel; let db: ITestDb; +let refreshDbMetrics: () => Promise; beforeAll(async () => { const config = createTestConfig({ @@ -102,16 +107,16 @@ beforeAll(async () => { }, }; - await monitor.startMonitoring( - config, - stores, - '4.0.0', - eventBus, - statsService, - schedulerService, - // @ts-ignore - We don't want a full knex implementation for our tests, it's enough that it actually yields the numbers we want. - metricsDbConf, - ); + const { collectDbMetrics, collectStaticCounters } = + registerPrometheusMetrics( + config, + stores, + '4.0.0', + eventBus, + statsService, + ); + refreshDbMetrics = collectDbMetrics; + await collectStaticCounters(); }); afterAll(async () => { @@ -212,6 +217,7 @@ test('should collect metrics for function timings', async () => { }); test('should collect metrics for feature flag size', async () => { + await refreshDbMetrics(); const metrics = await prometheusRegister.metrics(); expect(metrics).toMatch(/feature_toggles_total\{version="(.*)"\} 0/); }); @@ -222,12 +228,13 @@ test('should collect metrics for archived feature flag size', async () => { }); test('should collect metrics for total client apps', async () => { - await statsService.refreshAppCountSnapshot(); + await refreshDbMetrics(); const metrics = await prometheusRegister.metrics(); expect(metrics).toMatch(/client_apps_total\{range="(.*)"\} 0/); }); test('Should collect metrics for database', async () => { + registerPrometheusPostgresMetrics(db.rawDatabase, eventBus, '15.0.0'); const metrics = await prometheusRegister.metrics(); expect(metrics).toMatch(/db_pool_max/); expect(metrics).toMatch(/db_pool_min/); diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index be7311952cf8..e35245add754 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -25,7 +25,7 @@ import { PROJECT_DELETED, } from './types/events'; import type { IUnleashConfig } from './types/option'; -import type { ISettingStore, IUnleashStores } from './types/stores'; +import type { IUnleashStores } from './types/stores'; import { hoursToMilliseconds, minutesToMilliseconds } from 'date-fns'; import type { InstanceStatsService } from './features/instance-stats/instance-stats-service'; import type { IEnvironment, ISdkHeartbeat } from './types'; @@ -37,367 +37,792 @@ import { } from './util/metrics'; import type { SchedulerService } from './services'; import type { IClientMetricsEnv } from './features/metrics/client-metrics/client-metrics-store-v2-type'; - -export default class MetricsMonitor { - constructor() {} - - async startMonitoring( - config: IUnleashConfig, - stores: IUnleashStores, - version: string, - eventBus: EventEmitter, - instanceStatsService: InstanceStatsService, - schedulerService: SchedulerService, - db: Knex, - ): Promise { - if (!config.server.serverMetrics) { - return Promise.resolve(); - } - - const { eventStore, environmentStore } = stores; - const { flagResolver } = config; - - const cachedEnvironments: () => Promise = memoizee( - async () => environmentStore.getAll(), - { - promise: true, - maxAge: hoursToMilliseconds(1), - }, - ); - - collectDefaultMetrics(); - - const requestDuration = createSummary({ - name: 'http_request_duration_milliseconds', - help: 'App response time', - labelNames: ['path', 'method', 'status', 'appName'], - percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], - maxAgeSeconds: 600, - ageBuckets: 5, - }); - const schedulerDuration = createSummary({ - name: 'scheduler_duration_seconds', - help: 'Scheduler duration time', - labelNames: ['jobId'], - percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], - maxAgeSeconds: 600, - ageBuckets: 5, - }); - const dbDuration = createSummary({ - name: 'db_query_duration_seconds', - help: 'DB query duration time', - labelNames: ['store', 'action'], - percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], - maxAgeSeconds: 600, - ageBuckets: 5, - }); - const functionDuration = createSummary({ - name: 'function_duration_seconds', - help: 'Function duration time', - labelNames: ['functionName', 'className'], - percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], - maxAgeSeconds: 600, - ageBuckets: 5, - }); - const featureFlagUpdateTotal = createCounter({ - name: 'feature_toggle_update_total', - help: 'Number of times a toggle has been updated. Environment label would be "n/a" when it is not available, e.g. when a feature flag is created.', - labelNames: [ - 'toggle', - 'project', - 'environment', - 'environmentType', - 'action', - ], - }); - const featureFlagUsageTotal = createCounter({ - name: 'feature_toggle_usage_total', - help: 'Number of times a feature flag has been used', - labelNames: ['toggle', 'active', 'appName'], - }); - const featureFlagsTotal = createGauge({ - name: 'feature_toggles_total', - help: 'Number of feature flags', +import { DbMetricsMonitor } from './metrics-gauge'; + +export function registerPrometheusPostgresMetrics( + db: Knex, + eventBus: EventEmitter, + postgresVersion: string, +) { + if (db?.client) { + const dbPoolMin = createGauge({ + name: 'db_pool_min', + help: 'Minimum DB pool size', + }); + dbPoolMin.set(db.client.pool.min); + const dbPoolMax = createGauge({ + name: 'db_pool_max', + help: 'Maximum DB pool size', + }); + dbPoolMax.set(db.client.pool.max); + const dbPoolFree = createGauge({ + name: 'db_pool_free', + help: 'Current free connections in DB pool', + }); + const dbPoolUsed = createGauge({ + name: 'db_pool_used', + help: 'Current connections in use in DB pool', + }); + const dbPoolPendingCreates = createGauge({ + name: 'db_pool_pending_creates', + help: 'how many asynchronous create calls are running in DB pool', + }); + const dbPoolPendingAcquires = createGauge({ + name: 'db_pool_pending_acquires', + help: 'how many acquires are waiting for a resource to be released in DB pool', + }); + + eventBus.on(DB_POOL_UPDATE, (data) => { + dbPoolFree.set(data.free); + dbPoolUsed.set(data.used); + dbPoolPendingCreates.set(data.pendingCreates); + dbPoolPendingAcquires.set(data.pendingAcquires); + }); + + const database_version = createGauge({ + name: 'postgres_version', + help: 'Which version of postgres is running (SHOW server_version)', labelNames: ['version'], }); - const maxFeatureEnvironmentStrategies = createGauge({ - name: 'max_feature_environment_strategies', - help: 'Maximum number of environment strategies in one feature', - labelNames: ['feature', 'environment'], - }); - const maxFeatureStrategies = createGauge({ - name: 'max_feature_strategies', - help: 'Maximum number of strategies in one feature', - labelNames: ['feature'], - }); - const maxConstraintValues = createGauge({ - name: 'max_constraint_values', - help: 'Maximum number of constraint values used in a single constraint', - labelNames: ['feature', 'environment'], - }); - const maxConstraintsPerStrategy = createGauge({ - name: 'max_strategy_constraints', - help: 'Maximum number of constraints used on a single strategy', - labelNames: ['feature', 'environment'], - }); - const largestProjectEnvironment = createGauge({ - name: 'largest_project_environment_size', - help: 'The largest project environment size (bytes) based on strategies, constraints, variants and parameters', - labelNames: ['project', 'environment'], - }); - const largestFeatureEnvironment = createGauge({ - name: 'largest_feature_environment_size', - help: 'The largest feature environment size (bytes) base on strategies, constraints, variants and parameters', - labelNames: ['feature', 'environment'], - }); - - const featureTogglesArchivedTotal = createGauge({ - name: 'feature_toggles_archived_total', - help: 'Number of archived feature flags', - }); - const usersTotal = createGauge({ - name: 'users_total', - help: 'Number of users', - }); - const serviceAccounts = createGauge({ - name: 'service_accounts_total', - help: 'Number of service accounts', - }); - const apiTokens = createGauge({ - name: 'api_tokens_total', - help: 'Number of API tokens', - labelNames: ['type'], - }); - const enabledMetricsBucketsPreviousDay = createGauge({ - name: 'enabled_metrics_buckets_previous_day', - help: 'Number of hourly enabled/disabled metric buckets in the previous day', - }); - const variantMetricsBucketsPreviousDay = createGauge({ - name: 'variant_metrics_buckets_previous_day', - help: 'Number of hourly variant metric buckets in the previous day', - }); - const usersActive7days = createGauge({ - name: 'users_active_7', - help: 'Number of users active in the last 7 days', - }); - const usersActive30days = createGauge({ - name: 'users_active_30', - help: 'Number of users active in the last 30 days', - }); - const usersActive60days = createGauge({ - name: 'users_active_60', - help: 'Number of users active in the last 60 days', - }); - const usersActive90days = createGauge({ - name: 'users_active_90', - help: 'Number of users active in the last 90 days', - }); - const projectsTotal = createGauge({ - name: 'projects_total', - help: 'Number of projects', - labelNames: ['mode'], - }); - const environmentsTotal = createGauge({ - name: 'environments_total', - help: 'Number of environments', - }); - const groupsTotal = createGauge({ - name: 'groups_total', - help: 'Number of groups', - }); - - const rolesTotal = createGauge({ - name: 'roles_total', - help: 'Number of roles', - }); - - const customRootRolesTotal = createGauge({ - name: 'custom_root_roles_total', - help: 'Number of custom root roles', - }); - - const customRootRolesInUseTotal = createGauge({ - name: 'custom_root_roles_in_use_total', - help: 'Number of custom root roles in use', - }); - - const segmentsTotal = createGauge({ - name: 'segments_total', - help: 'Number of segments', - }); - - const contextTotal = createGauge({ - name: 'context_total', - help: 'Number of context', - }); - - const strategiesTotal = createGauge({ - name: 'strategies_total', - help: 'Number of strategies', - }); - - const clientAppsTotal = createGauge({ - name: 'client_apps_total', - help: 'Number of registered client apps aggregated by range by last seen', - labelNames: ['range'], - }); - - const samlEnabled = createGauge({ - name: 'saml_enabled', - help: 'Whether SAML is enabled', - }); - - const oidcEnabled = createGauge({ - name: 'oidc_enabled', - help: 'Whether OIDC is enabled', - }); - - const clientSdkVersionUsage = createCounter({ - name: 'client_sdk_versions', - help: 'Which sdk versions are being used', - labelNames: [ - 'sdk_name', - 'sdk_version', - 'platform_name', - 'platform_version', - 'yggdrasil_version', - 'spec_version', - ], - }); - - const productionChanges30 = createGauge({ - name: 'production_changes_30', - help: 'Changes made to production environment last 30 days', - labelNames: ['environment'], - }); - const productionChanges60 = createGauge({ - name: 'production_changes_60', - help: 'Changes made to production environment last 60 days', - labelNames: ['environment'], - }); - const productionChanges90 = createGauge({ - name: 'production_changes_90', - help: 'Changes made to production environment last 90 days', - labelNames: ['environment'], - }); - - const rateLimits = createGauge({ - name: 'rate_limits', - help: 'Rate limits (per minute) for METHOD/ENDPOINT pairs', - labelNames: ['endpoint', 'method'], - }); - const featureCreatedByMigration = createCounter({ - name: 'feature_created_by_migration_count', - help: 'Feature createdBy migration count', - }); - const eventCreatedByMigration = createCounter({ - name: 'event_created_by_migration_count', - help: 'Event createdBy migration count', - }); - const proxyRepositoriesCreated = createCounter({ - name: 'proxy_repositories_created', - help: 'Proxy repositories created', - }); - const frontendApiRepositoriesCreated = createCounter({ - name: 'frontend_api_repositories_created', - help: 'Frontend API repositories created', - }); - const mapFeaturesForClientDuration = createHistogram({ - name: 'map_features_for_client_duration', - help: 'Duration of mapFeaturesForClient function', - }); - - const featureLifecycleStageDuration = createGauge({ - name: 'feature_lifecycle_stage_duration', - labelNames: ['stage', 'project_id'], - help: 'Duration of feature lifecycle stages', - }); - - const onboardingDuration = createGauge({ - name: 'onboarding_duration', - labelNames: ['event'], - help: 'firstLogin, secondLogin, firstFeatureFlag, firstPreLive, firstLive from first user creation', - }); - const projectOnboardingDuration = createGauge({ - name: 'project_onboarding_duration', - labelNames: ['event', 'project'], - help: 'firstFeatureFlag, firstPreLive, firstLive from project creation', - }); - - const featureLifecycleStageCountByProject = createGauge({ - name: 'feature_lifecycle_stage_count_by_project', - help: 'Count features in a given stage by project id', - labelNames: ['stage', 'project_id'], - }); - - const featureLifecycleStageEnteredCounter = createCounter({ - name: 'feature_lifecycle_stage_entered', - help: 'Count how many features entered a given stage', - labelNames: ['stage'], - }); - - const projectActionsCounter = createCounter({ - name: 'project_actions_count', - help: 'Count project actions', - labelNames: ['action'], - }); - - const projectEnvironmentsDisabled = createCounter({ - name: 'project_environments_disabled', - help: 'How many "environment disabled" events we have received for each project', - labelNames: ['project_id'], - }); - - const orphanedTokensTotal = createGauge({ - name: 'orphaned_api_tokens_total', - help: 'Number of API tokens without a project', - }); + database_version.labels({ version: postgresVersion }).set(1); + } +} - const orphanedTokensActive = createGauge({ - name: 'orphaned_api_tokens_active', - help: 'Number of API tokens without a project, last seen within 3 months', - }); +export function registerPrometheusMetrics( + config: IUnleashConfig, + stores: IUnleashStores, + version: string, + eventBus: EventEmitter, + instanceStatsService: InstanceStatsService, +) { + const resolveEnvironmentType = async ( + environment: string, + cachedEnvironments: () => Promise, + ): Promise => { + const environments = await cachedEnvironments(); + const env = environments.find((e) => e.name === environment); - const legacyTokensTotal = createGauge({ - name: 'legacy_api_tokens_total', - help: 'Number of API tokens with v1 format', - }); + if (env) { + return env.type; + } else { + return 'unknown'; + } + }; + + const { eventStore, environmentStore } = stores; + const { flagResolver, db } = config; + const dbMetrics = new DbMetricsMonitor(config); + + const cachedEnvironments: () => Promise = memoizee( + async () => environmentStore.getAll(), + { + promise: true, + maxAge: hoursToMilliseconds(1), + }, + ); + + const requestDuration = createSummary({ + name: 'http_request_duration_milliseconds', + help: 'App response time', + labelNames: ['path', 'method', 'status', 'appName'], + percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], + maxAgeSeconds: 600, + ageBuckets: 5, + }); + const schedulerDuration = createSummary({ + name: 'scheduler_duration_seconds', + help: 'Scheduler duration time', + labelNames: ['jobId'], + percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], + maxAgeSeconds: 600, + ageBuckets: 5, + }); + const dbDuration = createSummary({ + name: 'db_query_duration_seconds', + help: 'DB query duration time', + labelNames: ['store', 'action'], + percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], + maxAgeSeconds: 600, + ageBuckets: 5, + }); + const functionDuration = createSummary({ + name: 'function_duration_seconds', + help: 'Function duration time', + labelNames: ['functionName', 'className'], + percentiles: [0.1, 0.5, 0.9, 0.95, 0.99], + maxAgeSeconds: 600, + ageBuckets: 5, + }); + const featureFlagUpdateTotal = createCounter({ + name: 'feature_toggle_update_total', + help: 'Number of times a toggle has been updated. Environment label would be "n/a" when it is not available, e.g. when a feature flag is created.', + labelNames: [ + 'toggle', + 'project', + 'environment', + 'environmentType', + 'action', + ], + }); + const featureFlagUsageTotal = createCounter({ + name: 'feature_toggle_usage_total', + help: 'Number of times a feature flag has been used', + labelNames: ['toggle', 'active', 'appName'], + }); + + dbMetrics.registerGaugeDbMetric({ + name: 'feature_toggles_total', + help: 'Number of feature flags', + labelNames: ['version'], + query: () => instanceStatsService.getToggleCount(), + map: (value) => ({ value, labels: { version } }), + }); + + dbMetrics.registerGaugeDbMetric({ + name: 'max_feature_environment_strategies', + help: 'Maximum number of environment strategies in one feature', + labelNames: ['feature', 'environment'], + query: () => + stores.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(), + map: (result) => ({ + value: result.count, + labels: { + environment: result.environment, + feature: result.feature, + }, + }), + }); + + dbMetrics.registerGaugeDbMetric({ + name: 'max_feature_strategies', + help: 'Maximum number of strategies in one feature', + labelNames: ['feature'], + query: () => + stores.featureStrategiesReadModel.getMaxFeatureStrategies(), + map: (result) => ({ + value: result.count, + labels: { feature: result.feature }, + }), + }); + + const maxConstraintValues = createGauge({ + name: 'max_constraint_values', + help: 'Maximum number of constraint values used in a single constraint', + labelNames: ['feature', 'environment'], + }); + const maxConstraintsPerStrategy = createGauge({ + name: 'max_strategy_constraints', + help: 'Maximum number of constraints used on a single strategy', + labelNames: ['feature', 'environment'], + }); + const largestProjectEnvironment = createGauge({ + name: 'largest_project_environment_size', + help: 'The largest project environment size (bytes) based on strategies, constraints, variants and parameters', + labelNames: ['project', 'environment'], + }); + const largestFeatureEnvironment = createGauge({ + name: 'largest_feature_environment_size', + help: 'The largest feature environment size (bytes) base on strategies, constraints, variants and parameters', + labelNames: ['feature', 'environment'], + }); + + const featureTogglesArchivedTotal = createGauge({ + name: 'feature_toggles_archived_total', + help: 'Number of archived feature flags', + }); + const usersTotal = createGauge({ + name: 'users_total', + help: 'Number of users', + }); + const serviceAccounts = createGauge({ + name: 'service_accounts_total', + help: 'Number of service accounts', + }); + const apiTokens = createGauge({ + name: 'api_tokens_total', + help: 'Number of API tokens', + labelNames: ['type'], + }); + const enabledMetricsBucketsPreviousDay = createGauge({ + name: 'enabled_metrics_buckets_previous_day', + help: 'Number of hourly enabled/disabled metric buckets in the previous day', + }); + const variantMetricsBucketsPreviousDay = createGauge({ + name: 'variant_metrics_buckets_previous_day', + help: 'Number of hourly variant metric buckets in the previous day', + }); + const usersActive7days = createGauge({ + name: 'users_active_7', + help: 'Number of users active in the last 7 days', + }); + const usersActive30days = createGauge({ + name: 'users_active_30', + help: 'Number of users active in the last 30 days', + }); + const usersActive60days = createGauge({ + name: 'users_active_60', + help: 'Number of users active in the last 60 days', + }); + const usersActive90days = createGauge({ + name: 'users_active_90', + help: 'Number of users active in the last 90 days', + }); + const projectsTotal = createGauge({ + name: 'projects_total', + help: 'Number of projects', + labelNames: ['mode'], + }); + const environmentsTotal = createGauge({ + name: 'environments_total', + help: 'Number of environments', + }); + const groupsTotal = createGauge({ + name: 'groups_total', + help: 'Number of groups', + }); + + const rolesTotal = createGauge({ + name: 'roles_total', + help: 'Number of roles', + }); + + const customRootRolesTotal = createGauge({ + name: 'custom_root_roles_total', + help: 'Number of custom root roles', + }); + + const customRootRolesInUseTotal = createGauge({ + name: 'custom_root_roles_in_use_total', + help: 'Number of custom root roles in use', + }); + + const segmentsTotal = createGauge({ + name: 'segments_total', + help: 'Number of segments', + }); + + const contextTotal = createGauge({ + name: 'context_total', + help: 'Number of context', + }); + + const strategiesTotal = createGauge({ + name: 'strategies_total', + help: 'Number of strategies', + }); + + dbMetrics.registerGaugeDbMetric({ + name: 'client_apps_total', + help: 'Number of registered client apps aggregated by range by last seen', + labelNames: ['range'], + query: () => instanceStatsService.getLabeledAppCounts(), + map: (result) => + Object.entries(result).map(([range, count]) => ({ + value: count, + labels: { range }, + })), + }); + + const samlEnabled = createGauge({ + name: 'saml_enabled', + help: 'Whether SAML is enabled', + }); + + const oidcEnabled = createGauge({ + name: 'oidc_enabled', + help: 'Whether OIDC is enabled', + }); + + const clientSdkVersionUsage = createCounter({ + name: 'client_sdk_versions', + help: 'Which sdk versions are being used', + labelNames: [ + 'sdk_name', + 'sdk_version', + 'platform_name', + 'platform_version', + 'yggdrasil_version', + 'spec_version', + ], + }); + + const productionChanges30 = createGauge({ + name: 'production_changes_30', + help: 'Changes made to production environment last 30 days', + labelNames: ['environment'], + }); + const productionChanges60 = createGauge({ + name: 'production_changes_60', + help: 'Changes made to production environment last 60 days', + labelNames: ['environment'], + }); + const productionChanges90 = createGauge({ + name: 'production_changes_90', + help: 'Changes made to production environment last 90 days', + labelNames: ['environment'], + }); + + const rateLimits = createGauge({ + name: 'rate_limits', + help: 'Rate limits (per minute) for METHOD/ENDPOINT pairs', + labelNames: ['endpoint', 'method'], + }); + rateLimits + .labels({ + endpoint: '/api/client/metrics', + method: 'POST', + }) + .set(config.metricsRateLimiting.clientMetricsMaxPerMinute); + rateLimits + .labels({ + endpoint: '/api/client/register', + method: 'POST', + }) + .set(config.metricsRateLimiting.clientRegisterMaxPerMinute); + rateLimits + .labels({ + endpoint: '/api/frontend/metrics', + method: 'POST', + }) + .set(config.metricsRateLimiting.frontendMetricsMaxPerMinute); + rateLimits + .labels({ + endpoint: '/api/frontend/register', + method: 'POST', + }) + .set(config.metricsRateLimiting.frontendRegisterMaxPerMinute); + rateLimits + .labels({ + endpoint: '/api/admin/user-admin', + method: 'POST', + }) + .set(config.rateLimiting.createUserMaxPerMinute); + rateLimits + .labels({ + endpoint: '/auth/simple', + method: 'POST', + }) + .set(config.rateLimiting.simpleLoginMaxPerMinute); + rateLimits + .labels({ + endpoint: '/auth/reset/password-email', + method: 'POST', + }) + .set(config.rateLimiting.passwordResetMaxPerMinute); + rateLimits + .labels({ + endpoint: '/api/signal-endpoint/:name', + method: 'POST', + }) + .set(config.rateLimiting.callSignalEndpointMaxPerSecond * 60); + + const featureCreatedByMigration = createCounter({ + name: 'feature_created_by_migration_count', + help: 'Feature createdBy migration count', + }); + const eventCreatedByMigration = createCounter({ + name: 'event_created_by_migration_count', + help: 'Event createdBy migration count', + }); + const proxyRepositoriesCreated = createCounter({ + name: 'proxy_repositories_created', + help: 'Proxy repositories created', + }); + const frontendApiRepositoriesCreated = createCounter({ + name: 'frontend_api_repositories_created', + help: 'Frontend API repositories created', + }); + const mapFeaturesForClientDuration = createHistogram({ + name: 'map_features_for_client_duration', + help: 'Duration of mapFeaturesForClient function', + }); + + const featureLifecycleStageDuration = createGauge({ + name: 'feature_lifecycle_stage_duration', + labelNames: ['stage', 'project_id'], + help: 'Duration of feature lifecycle stages', + }); + + const onboardingDuration = createGauge({ + name: 'onboarding_duration', + labelNames: ['event'], + help: 'firstLogin, secondLogin, firstFeatureFlag, firstPreLive, firstLive from first user creation', + }); + const projectOnboardingDuration = createGauge({ + name: 'project_onboarding_duration', + labelNames: ['event', 'project'], + help: 'firstFeatureFlag, firstPreLive, firstLive from project creation', + }); + + const featureLifecycleStageCountByProject = createGauge({ + name: 'feature_lifecycle_stage_count_by_project', + help: 'Count features in a given stage by project id', + labelNames: ['stage', 'project_id'], + }); + + const featureLifecycleStageEnteredCounter = createCounter({ + name: 'feature_lifecycle_stage_entered', + help: 'Count how many features entered a given stage', + labelNames: ['stage'], + }); + + const projectActionsCounter = createCounter({ + name: 'project_actions_count', + help: 'Count project actions', + labelNames: ['action'], + }); + + const projectEnvironmentsDisabled = createCounter({ + name: 'project_environments_disabled', + help: 'How many "environment disabled" events we have received for each project', + labelNames: ['project_id'], + }); + + const orphanedTokensTotal = createGauge({ + name: 'orphaned_api_tokens_total', + help: 'Number of API tokens without a project', + }); + + const orphanedTokensActive = createGauge({ + name: 'orphaned_api_tokens_active', + help: 'Number of API tokens without a project, last seen within 3 months', + }); + + const legacyTokensTotal = createGauge({ + name: 'legacy_api_tokens_total', + help: 'Number of API tokens with v1 format', + }); + + const legacyTokensActive = createGauge({ + name: 'legacy_api_tokens_active', + help: 'Number of API tokens with v1 format, last seen within 3 months', + }); + + const exceedsLimitErrorCounter = createCounter({ + name: 'exceeds_limit_error', + help: 'The number of exceeds limit errors registered by this instance.', + labelNames: ['resource', 'limit'], + }); + + const requestOriginCounter = createCounter({ + name: 'request_origin_counter', + help: 'Number of authenticated requests, including origin information.', + labelNames: ['type', 'method', 'source'], + }); + + const resourceLimit = createGauge({ + name: 'resource_limit', + help: 'The maximum number of resources allowed.', + labelNames: ['resource'], + }); + + const addonEventsHandledCounter = createCounter({ + name: 'addon_events_handled', + help: 'Events handled by addons and the result.', + labelNames: ['result', 'destination'], + }); + + // register event listeners + eventBus.on( + events.EXCEEDS_LIMIT, + ({ resource, limit }: { resource: string; limit: number }) => { + exceedsLimitErrorCounter.increment({ resource, limit }); + }, + ); + + eventBus.on( + events.STAGE_ENTERED, + (entered: { stage: string; feature: string }) => { + if (flagResolver.isEnabled('trackLifecycleMetrics')) { + logger.info( + `STAGE_ENTERED listened ${JSON.stringify(entered)}`, + ); + } + featureLifecycleStageEnteredCounter.increment({ + stage: entered.stage, + }); + }, + ); - const legacyTokensActive = createGauge({ - name: 'legacy_api_tokens_active', - help: 'Number of API tokens with v1 format, last seen within 3 months', - }); + eventBus.on( + events.REQUEST_TIME, + ({ path, method, time, statusCode, appName }) => { + requestDuration + .labels({ + path, + method, + status: statusCode, + appName, + }) + .observe(time); + }, + ); + + eventBus.on(events.SCHEDULER_JOB_TIME, ({ jobId, time }) => { + schedulerDuration.labels(jobId).observe(time); + }); + + eventBus.on(events.FUNCTION_TIME, ({ functionName, className, time }) => { + functionDuration + .labels({ + functionName, + className, + }) + .observe(time); + }); + + eventBus.on(events.EVENTS_CREATED_BY_PROCESSED, ({ updated }) => { + eventCreatedByMigration.inc(updated); + }); + + eventBus.on(events.FEATURES_CREATED_BY_PROCESSED, ({ updated }) => { + featureCreatedByMigration.inc(updated); + }); + + eventBus.on(events.DB_TIME, ({ store, action, time }) => { + dbDuration + .labels({ + store, + action, + }) + .observe(time); + }); + + eventBus.on(events.PROXY_REPOSITORY_CREATED, () => { + proxyRepositoriesCreated.inc(); + }); + + eventBus.on(events.FRONTEND_API_REPOSITORY_CREATED, () => { + frontendApiRepositoriesCreated.inc(); + }); + + eventBus.on(events.PROXY_FEATURES_FOR_TOKEN_TIME, ({ duration }) => { + mapFeaturesForClientDuration.observe(duration); + }); + + events.onMetricEvent( + eventBus, + events.REQUEST_ORIGIN, + ({ type, method, source }) => { + requestOriginCounter.increment({ type, method, source }); + }, + ); + + eventStore.on(FEATURE_CREATED, ({ featureName, project }) => { + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment: 'n/a', + environmentType: 'n/a', + action: 'created', + }); + }); + eventStore.on(FEATURE_VARIANTS_UPDATED, ({ featureName, project }) => { + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment: 'n/a', + environmentType: 'n/a', + action: 'updated', + }); + }); + eventStore.on(FEATURE_METADATA_UPDATED, ({ featureName, project }) => { + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment: 'n/a', + environmentType: 'n/a', + action: 'updated', + }); + }); + eventStore.on(FEATURE_UPDATED, ({ featureName, project }) => { + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment: 'default', + environmentType: 'production', + action: 'updated', + }); + }); + eventStore.on( + FEATURE_STRATEGY_ADD, + async ({ featureName, project, environment }) => { + const environmentType = await resolveEnvironmentType( + environment, + cachedEnvironments, + ); + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment, + environmentType, + action: 'updated', + }); + }, + ); + eventStore.on( + FEATURE_STRATEGY_REMOVE, + async ({ featureName, project, environment }) => { + const environmentType = await resolveEnvironmentType( + environment, + cachedEnvironments, + ); + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment, + environmentType, + action: 'updated', + }); + }, + ); + eventStore.on( + FEATURE_STRATEGY_UPDATE, + async ({ featureName, project, environment }) => { + const environmentType = await resolveEnvironmentType( + environment, + cachedEnvironments, + ); + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment, + environmentType, + action: 'updated', + }); + }, + ); + eventStore.on( + FEATURE_ENVIRONMENT_DISABLED, + async ({ featureName, project, environment }) => { + const environmentType = await resolveEnvironmentType( + environment, + cachedEnvironments, + ); + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment, + environmentType, + action: 'updated', + }); + }, + ); + eventStore.on( + FEATURE_ENVIRONMENT_ENABLED, + async ({ featureName, project, environment }) => { + const environmentType = await resolveEnvironmentType( + environment, + cachedEnvironments, + ); + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment, + environmentType, + action: 'updated', + }); + }, + ); + eventStore.on(FEATURE_ARCHIVED, ({ featureName, project }) => { + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment: 'n/a', + environmentType: 'n/a', + action: 'archived', + }); + }); + eventStore.on(FEATURE_REVIVED, ({ featureName, project }) => { + featureFlagUpdateTotal.increment({ + toggle: featureName, + project, + environment: 'n/a', + environmentType: 'n/a', + action: 'revived', + }); + }); + eventStore.on(PROJECT_CREATED, () => { + projectActionsCounter.increment({ action: PROJECT_CREATED }); + }); + eventStore.on(PROJECT_ARCHIVED, () => { + projectActionsCounter.increment({ action: PROJECT_ARCHIVED }); + }); + eventStore.on(PROJECT_REVIVED, () => { + projectActionsCounter.increment({ action: PROJECT_REVIVED }); + }); + eventStore.on(PROJECT_DELETED, () => { + projectActionsCounter.increment({ action: PROJECT_DELETED }); + }); + + const logger = config.getLogger('metrics.ts'); + eventBus.on(CLIENT_METRICS, (metrics: IClientMetricsEnv[]) => { + try { + for (const metric of metrics) { + featureFlagUsageTotal.increment( + { + toggle: metric.featureName, + active: 'true', + appName: metric.appName, + }, + metric.yes, + ); + featureFlagUsageTotal.increment( + { + toggle: metric.featureName, + active: 'false', + appName: metric.appName, + }, + metric.no, + ); + } + } catch (e) { + logger.warn('Metrics registration failed', e); + } + }); - const exceedsLimitErrorCounter = createCounter({ - name: 'exceeds_limit_error', - help: 'The number of exceeds limit errors registered by this instance.', - labelNames: ['resource', 'limit'], - }); + eventStore.on(CLIENT_REGISTER, (heartbeatEvent: ISdkHeartbeat) => { + if (!heartbeatEvent.sdkName || !heartbeatEvent.sdkVersion) { + return; + } - const requestOriginCounter = createCounter({ - name: 'request_origin_counter', - help: 'Number of authenticated requests, including origin information.', - labelNames: ['type', 'method', 'source'], - }); + if (flagResolver.isEnabled('extendedMetrics')) { + clientSdkVersionUsage.increment({ + sdk_name: heartbeatEvent.sdkName, + sdk_version: heartbeatEvent.sdkVersion, + platform_name: + heartbeatEvent.metadata?.platformName ?? 'not-set', + platform_version: + heartbeatEvent.metadata?.platformVersion ?? 'not-set', + yggdrasil_version: + heartbeatEvent.metadata?.yggdrasilVersion ?? 'not-set', + spec_version: heartbeatEvent.metadata?.specVersion ?? 'not-set', + }); + } else { + clientSdkVersionUsage.increment({ + sdk_name: heartbeatEvent.sdkName, + sdk_version: heartbeatEvent.sdkVersion, + platform_name: 'not-set', + platform_version: 'not-set', + yggdrasil_version: 'not-set', + spec_version: 'not-set', + }); + } + }); - const resourceLimit = createGauge({ - name: 'resource_limit', - help: 'The maximum number of resources allowed.', - labelNames: ['resource'], - }); + eventStore.on(PROJECT_ENVIRONMENT_REMOVED, ({ project }) => { + projectEnvironmentsDisabled.increment({ project_id: project }); + }); - const addonEventsHandledCounter = createCounter({ - name: 'addon_events_handled', - help: 'Events handled by addons and the result.', - labelNames: ['result', 'destination'], - }); + eventBus.on(events.ADDON_EVENTS_HANDLED, ({ result, destination }) => { + addonEventsHandledCounter.increment({ result, destination }); + }); - async function collectStaticCounters() { + return { + collectDbMetrics: dbMetrics.refreshDbMetrics, + collectStaticCounters: async () => { try { - const stats = await instanceStatsService.getStats(); const [ - maxStrategies, - maxEnvironmentStrategies, maxConstraintValuesResult, maxConstraintsPerStrategyResult, stageCountByProjectResult, @@ -408,8 +833,6 @@ export default class MetricsMonitor { instanceOnboardingMetrics, projectsOnboardingMetrics, ] = await Promise.all([ - stores.featureStrategiesReadModel.getMaxFeatureStrategies(), - stores.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(), stores.featureStrategiesReadModel.getMaxConstraintValues(), stores.featureStrategiesReadModel.getMaxConstraintsPerStrategy(), stores.featureLifecycleReadModel.getStageCountByProject(), @@ -429,17 +852,18 @@ export default class MetricsMonitor { : Promise.resolve([]), ]); - featureFlagsTotal.reset(); - featureFlagsTotal.labels({ version }).set(stats.featureToggles); - featureTogglesArchivedTotal.reset(); - featureTogglesArchivedTotal.set(stats.archivedFeatureToggles); + featureTogglesArchivedTotal.set( + await instanceStatsService.getArchivedToggleCount(), + ); usersTotal.reset(); - usersTotal.set(stats.users); + usersTotal.set(await instanceStatsService.getRegisteredUsers()); serviceAccounts.reset(); - serviceAccounts.set(stats.serviceAccounts); + serviceAccounts.set( + await instanceStatsService.countServiceAccounts(), + ); stageDurationByProject.forEach((stage) => { featureLifecycleStageDuration @@ -450,30 +874,6 @@ export default class MetricsMonitor { .set(stage.duration); }); - eventBus.on( - events.STAGE_ENTERED, - (entered: { stage: string; feature: string }) => { - if (flagResolver.isEnabled('trackLifecycleMetrics')) { - logger.info( - `STAGE_ENTERED listened ${JSON.stringify(entered)}`, - ); - } - featureLifecycleStageEnteredCounter.increment({ - stage: entered.stage, - }); - }, - ); - - eventBus.on( - events.EXCEEDS_LIMIT, - ({ - resource, - limit, - }: { resource: string; limit: number }) => { - exceedsLimitErrorCounter.increment({ resource, limit }); - }, - ); - featureLifecycleStageCountByProject.reset(); stageCountByProjectResult.forEach((stageResult) => featureLifecycleStageCountByProject @@ -486,7 +886,10 @@ export default class MetricsMonitor { apiTokens.reset(); - for (const [type, value] of stats.apiTokens) { + for (const [ + type, + value, + ] of await instanceStatsService.countApiTokensByType()) { apiTokens.labels({ type }).set(value); } @@ -502,21 +905,6 @@ export default class MetricsMonitor { legacyTokensActive.reset(); legacyTokensActive.set(deprecatedTokens.activeLegacyTokens); - if (maxEnvironmentStrategies) { - maxFeatureEnvironmentStrategies.reset(); - maxFeatureEnvironmentStrategies - .labels({ - environment: maxEnvironmentStrategies.environment, - feature: maxEnvironmentStrategies.feature, - }) - .set(maxEnvironmentStrategies.count); - } - if (maxStrategies) { - maxFeatureStrategies.reset(); - maxFeatureStrategies - .labels({ feature: maxStrategies.feature }) - .set(maxStrategies.count); - } if (maxConstraintValuesResult) { maxConstraintValues.reset(); maxConstraintValues @@ -586,488 +974,135 @@ export default class MetricsMonitor { resourceLimit.labels({ resource }).set(limit); } + const previousDayMetricsBucketsCount = + await instanceStatsService.countPreviousDayHourlyMetricsBuckets(); enabledMetricsBucketsPreviousDay.reset(); enabledMetricsBucketsPreviousDay.set( - stats.previousDayMetricsBucketsCount.enabledCount, + previousDayMetricsBucketsCount.enabledCount, ); variantMetricsBucketsPreviousDay.reset(); variantMetricsBucketsPreviousDay.set( - stats.previousDayMetricsBucketsCount.variantCount, + previousDayMetricsBucketsCount.variantCount, ); + const activeUsers = await instanceStatsService.getActiveUsers(); usersActive7days.reset(); - usersActive7days.set(stats.activeUsers.last7); + usersActive7days.set(activeUsers.last7); usersActive30days.reset(); - usersActive30days.set(stats.activeUsers.last30); + usersActive30days.set(activeUsers.last30); usersActive60days.reset(); - usersActive60days.set(stats.activeUsers.last60); + usersActive60days.set(activeUsers.last60); usersActive90days.reset(); - usersActive90days.set(stats.activeUsers.last90); + usersActive90days.set(activeUsers.last90); + const productionChanges = + await instanceStatsService.getProductionChanges(); productionChanges30.reset(); - productionChanges30.set(stats.productionChanges.last30); + productionChanges30.set(productionChanges.last30); productionChanges60.reset(); - productionChanges60.set(stats.productionChanges.last60); + productionChanges60.set(productionChanges.last60); productionChanges90.reset(); - productionChanges90.set(stats.productionChanges.last90); + productionChanges90.set(productionChanges.last90); + const projects = + await instanceStatsService.getProjectModeCount(); projectsTotal.reset(); - stats.projects.forEach((projectStat) => { + projects.forEach((projectStat) => { projectsTotal .labels({ mode: projectStat.mode }) .set(projectStat.count); }); environmentsTotal.reset(); - environmentsTotal.set(stats.environments); + environmentsTotal.set( + await instanceStatsService.environmentCount(), + ); groupsTotal.reset(); - groupsTotal.set(stats.groups); + groupsTotal.set(await instanceStatsService.groupCount()); rolesTotal.reset(); - rolesTotal.set(stats.roles); + rolesTotal.set(await instanceStatsService.roleCount()); customRootRolesTotal.reset(); - customRootRolesTotal.set(stats.customRootRoles); + customRootRolesTotal.set( + await instanceStatsService.customRolesCount(), + ); customRootRolesInUseTotal.reset(); - customRootRolesInUseTotal.set(stats.customRootRolesInUse); + customRootRolesInUseTotal.set( + await instanceStatsService.customRolesCountInUse(), + ); segmentsTotal.reset(); - segmentsTotal.set(stats.segments); + segmentsTotal.set(await instanceStatsService.segmentCount()); contextTotal.reset(); - contextTotal.set(stats.contextFields); + contextTotal.set( + await instanceStatsService.contextFieldCount(), + ); strategiesTotal.reset(); - strategiesTotal.set(stats.strategies); + strategiesTotal.set( + await instanceStatsService.strategiesCount(), + ); samlEnabled.reset(); - samlEnabled.set(stats.SAMLenabled ? 1 : 0); + samlEnabled.set((await instanceStatsService.hasSAML()) ? 1 : 0); oidcEnabled.reset(); - oidcEnabled.set(stats.OIDCenabled ? 1 : 0); - - clientAppsTotal.reset(); - stats.clientApps.forEach(({ range, count }) => - clientAppsTotal.labels({ range }).set(count), - ); - - rateLimits.reset(); - rateLimits - .labels({ - endpoint: '/api/client/metrics', - method: 'POST', - }) - .set(config.metricsRateLimiting.clientMetricsMaxPerMinute); - rateLimits - .labels({ - endpoint: '/api/client/register', - method: 'POST', - }) - .set(config.metricsRateLimiting.clientRegisterMaxPerMinute); - rateLimits - .labels({ - endpoint: '/api/frontend/metrics', - method: 'POST', - }) - .set( - config.metricsRateLimiting.frontendMetricsMaxPerMinute, - ); - rateLimits - .labels({ - endpoint: '/api/frontend/register', - method: 'POST', - }) - .set( - config.metricsRateLimiting.frontendRegisterMaxPerMinute, - ); - rateLimits - .labels({ - endpoint: '/api/admin/user-admin', - method: 'POST', - }) - .set(config.rateLimiting.createUserMaxPerMinute); - rateLimits - .labels({ - endpoint: '/auth/simple', - method: 'POST', - }) - .set(config.rateLimiting.simpleLoginMaxPerMinute); - rateLimits - .labels({ - endpoint: '/auth/reset/password-email', - method: 'POST', - }) - .set(config.rateLimiting.passwordResetMaxPerMinute); - rateLimits - .labels({ - endpoint: '/api/signal-endpoint/:name', - method: 'POST', - }) - .set( - config.rateLimiting.callSignalEndpointMaxPerSecond * 60, - ); + oidcEnabled.set((await instanceStatsService.hasOIDC()) ? 1 : 0); } catch (e) {} - } - - await schedulerService.schedule( - collectStaticCounters.bind(this), - hoursToMilliseconds(2), - 'collectStaticCounters', - 0, // no jitter - ); - - eventBus.on( - events.REQUEST_TIME, - ({ path, method, time, statusCode, appName }) => { - requestDuration - .labels({ - path, - method, - status: statusCode, - appName, - }) - .observe(time); - }, - ); - - eventBus.on(events.SCHEDULER_JOB_TIME, ({ jobId, time }) => { - schedulerDuration.labels(jobId).observe(time); - }); - - eventBus.on( - events.FUNCTION_TIME, - ({ functionName, className, time }) => { - functionDuration - .labels({ - functionName, - className, - }) - .observe(time); - }, - ); - - eventBus.on(events.EVENTS_CREATED_BY_PROCESSED, ({ updated }) => { - eventCreatedByMigration.inc(updated); - }); - - eventBus.on(events.FEATURES_CREATED_BY_PROCESSED, ({ updated }) => { - featureCreatedByMigration.inc(updated); - }); - - eventBus.on(events.DB_TIME, ({ store, action, time }) => { - dbDuration - .labels({ - store, - action, - }) - .observe(time); - }); + }, + }; +} +export default class MetricsMonitor { + constructor() {} - eventBus.on(events.PROXY_REPOSITORY_CREATED, () => { - proxyRepositoriesCreated.inc(); - }); + async startMonitoring( + config: IUnleashConfig, + stores: IUnleashStores, + version: string, + eventBus: EventEmitter, + instanceStatsService: InstanceStatsService, + schedulerService: SchedulerService, + db: Knex, + ): Promise { + if (!config.server.serverMetrics) { + return Promise.resolve(); + } - eventBus.on(events.FRONTEND_API_REPOSITORY_CREATED, () => { - frontendApiRepositoriesCreated.inc(); - }); + collectDefaultMetrics(); - eventBus.on(events.PROXY_FEATURES_FOR_TOKEN_TIME, ({ duration }) => { - mapFeaturesForClientDuration.observe(duration); - }); + const { collectStaticCounters, collectDbMetrics } = + registerPrometheusMetrics( + config, + stores, + version, + eventBus, + instanceStatsService, + ); - events.onMetricEvent( - eventBus, - events.REQUEST_ORIGIN, - ({ type, method, source }) => { - requestOriginCounter.increment({ type, method, source }); - }, - ); + const postgresVersion = await stores.settingStore.postgresVersion(); + registerPrometheusPostgresMetrics(db, eventBus, postgresVersion); - eventStore.on(FEATURE_CREATED, ({ featureName, project }) => { - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment: 'n/a', - environmentType: 'n/a', - action: 'created', - }); - }); - eventStore.on(FEATURE_VARIANTS_UPDATED, ({ featureName, project }) => { - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment: 'n/a', - environmentType: 'n/a', - action: 'updated', - }); - }); - eventStore.on(FEATURE_METADATA_UPDATED, ({ featureName, project }) => { - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment: 'n/a', - environmentType: 'n/a', - action: 'updated', - }); - }); - eventStore.on(FEATURE_UPDATED, ({ featureName, project }) => { - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment: 'default', - environmentType: 'production', - action: 'updated', - }); - }); - eventStore.on( - FEATURE_STRATEGY_ADD, - async ({ featureName, project, environment }) => { - const environmentType = await this.resolveEnvironmentType( - environment, - cachedEnvironments, - ); - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment, - environmentType, - action: 'updated', - }); - }, - ); - eventStore.on( - FEATURE_STRATEGY_REMOVE, - async ({ featureName, project, environment }) => { - const environmentType = await this.resolveEnvironmentType( - environment, - cachedEnvironments, - ); - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment, - environmentType, - action: 'updated', - }); - }, - ); - eventStore.on( - FEATURE_STRATEGY_UPDATE, - async ({ featureName, project, environment }) => { - const environmentType = await this.resolveEnvironmentType( - environment, - cachedEnvironments, - ); - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment, - environmentType, - action: 'updated', - }); - }, - ); - eventStore.on( - FEATURE_ENVIRONMENT_DISABLED, - async ({ featureName, project, environment }) => { - const environmentType = await this.resolveEnvironmentType( - environment, - cachedEnvironments, - ); - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment, - environmentType, - action: 'updated', - }); - }, - ); - eventStore.on( - FEATURE_ENVIRONMENT_ENABLED, - async ({ featureName, project, environment }) => { - const environmentType = await this.resolveEnvironmentType( - environment, - cachedEnvironments, - ); - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment, - environmentType, - action: 'updated', - }); - }, + await schedulerService.schedule( + async () => + Promise.all([collectStaticCounters(), collectDbMetrics()]), + hoursToMilliseconds(2), + 'collectStaticCounters', ); - eventStore.on(FEATURE_ARCHIVED, ({ featureName, project }) => { - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment: 'n/a', - environmentType: 'n/a', - action: 'archived', - }); - }); - eventStore.on(FEATURE_REVIVED, ({ featureName, project }) => { - featureFlagUpdateTotal.increment({ - toggle: featureName, - project, - environment: 'n/a', - environmentType: 'n/a', - action: 'revived', - }); - }); - eventStore.on(PROJECT_CREATED, () => { - projectActionsCounter.increment({ action: PROJECT_CREATED }); - }); - eventStore.on(PROJECT_ARCHIVED, () => { - projectActionsCounter.increment({ action: PROJECT_ARCHIVED }); - }); - eventStore.on(PROJECT_REVIVED, () => { - projectActionsCounter.increment({ action: PROJECT_REVIVED }); - }); - eventStore.on(PROJECT_DELETED, () => { - projectActionsCounter.increment({ action: PROJECT_DELETED }); - }); - - const logger = config.getLogger('metrics.ts'); - eventBus.on(CLIENT_METRICS, (metrics: IClientMetricsEnv[]) => { - try { - for (const metric of metrics) { - featureFlagUsageTotal.increment( - { - toggle: metric.featureName, - active: 'true', - appName: metric.appName, - }, - metric.yes, - ); - featureFlagUsageTotal.increment( - { - toggle: metric.featureName, - active: 'false', - appName: metric.appName, - }, - metric.no, - ); - } - } catch (e) { - logger.warn('Metrics registration failed', e); - } - }); - - eventStore.on(CLIENT_REGISTER, (heartbeatEvent: ISdkHeartbeat) => { - if (!heartbeatEvent.sdkName || !heartbeatEvent.sdkVersion) { - return; - } - - if (flagResolver.isEnabled('extendedMetrics')) { - clientSdkVersionUsage.increment({ - sdk_name: heartbeatEvent.sdkName, - sdk_version: heartbeatEvent.sdkVersion, - platform_name: - heartbeatEvent.metadata?.platformName ?? 'not-set', - platform_version: - heartbeatEvent.metadata?.platformVersion ?? 'not-set', - yggdrasil_version: - heartbeatEvent.metadata?.yggdrasilVersion ?? 'not-set', - spec_version: - heartbeatEvent.metadata?.specVersion ?? 'not-set', - }); - } else { - clientSdkVersionUsage.increment({ - sdk_name: heartbeatEvent.sdkName, - sdk_version: heartbeatEvent.sdkVersion, - platform_name: 'not-set', - platform_version: 'not-set', - yggdrasil_version: 'not-set', - spec_version: 'not-set', - }); - } - }); - - eventStore.on(PROJECT_ENVIRONMENT_REMOVED, ({ project }) => { - projectEnvironmentsDisabled.increment({ project_id: project }); - }); - - eventBus.on(events.ADDON_EVENTS_HANDLED, ({ result, destination }) => { - addonEventsHandledCounter.increment({ result, destination }); - }); - - await this.configureDbMetrics( - db, - eventBus, - schedulerService, - stores.settingStore, + await schedulerService.schedule( + async () => + this.registerPoolMetrics.bind(this, db.client.pool, eventBus), + minutesToMilliseconds(1), + 'registerPoolMetrics', + 0, // no jitter ); return Promise.resolve(); } - async configureDbMetrics( - db: Knex, - eventBus: EventEmitter, - schedulerService: SchedulerService, - settingStore: ISettingStore, - ): Promise { - if (db?.client) { - const dbPoolMin = createGauge({ - name: 'db_pool_min', - help: 'Minimum DB pool size', - }); - dbPoolMin.set(db.client.pool.min); - const dbPoolMax = createGauge({ - name: 'db_pool_max', - help: 'Maximum DB pool size', - }); - dbPoolMax.set(db.client.pool.max); - const dbPoolFree = createGauge({ - name: 'db_pool_free', - help: 'Current free connections in DB pool', - }); - const dbPoolUsed = createGauge({ - name: 'db_pool_used', - help: 'Current connections in use in DB pool', - }); - const dbPoolPendingCreates = createGauge({ - name: 'db_pool_pending_creates', - help: 'how many asynchronous create calls are running in DB pool', - }); - const dbPoolPendingAcquires = createGauge({ - name: 'db_pool_pending_acquires', - help: 'how many acquires are waiting for a resource to be released in DB pool', - }); - - eventBus.on(DB_POOL_UPDATE, (data) => { - dbPoolFree.set(data.free); - dbPoolUsed.set(data.used); - dbPoolPendingCreates.set(data.pendingCreates); - dbPoolPendingAcquires.set(data.pendingAcquires); - }); - - await schedulerService.schedule( - async () => - this.registerPoolMetrics.bind( - this, - db.client.pool, - eventBus, - ), - minutesToMilliseconds(1), - 'registerPoolMetrics', - 0, // no jitter - ); - const postgresVersion = await settingStore.postgresVersion(); - const database_version = createGauge({ - name: 'postgres_version', - help: 'Which version of postgres is running (SHOW server_version)', - labelNames: ['version'], - }); - database_version.labels({ version: postgresVersion }).set(1); - } - } - // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types registerPoolMetrics(pool: any, eventBus: EventEmitter) { try { @@ -1080,26 +1115,8 @@ export default class MetricsMonitor { // eslint-disable-next-line no-empty } catch (e) {} } - - async resolveEnvironmentType( - environment: string, - cachedEnvironments: () => Promise, - ): Promise { - const environments = await cachedEnvironments(); - const env = environments.find((e) => e.name === environment); - - if (env) { - return env.type; - } else { - return 'unknown'; - } - } } export function createMetricsMonitor(): MetricsMonitor { return new MetricsMonitor(); } - -module.exports = { - createMetricsMonitor, -}; diff --git a/src/test/e2e/api/admin/instance-admin.e2e.test.ts b/src/test/e2e/api/admin/instance-admin.e2e.test.ts index 9b1cdb34ed1a..a3e5c08c7386 100644 --- a/src/test/e2e/api/admin/instance-admin.e2e.test.ts +++ b/src/test/e2e/api/admin/instance-admin.e2e.test.ts @@ -6,10 +6,12 @@ import { import getLogger from '../../../fixtures/no-logger'; import type { IUnleashStores } from '../../../../lib/types'; import { ApiTokenType } from '../../../../lib/types/models/api-token'; +import { registerPrometheusMetrics } from '../../../../lib/metrics'; let app: IUnleashTest; let db: ITestDb; let stores: IUnleashStores; +let refreshDbMetrics: () => Promise; beforeAll(async () => { db = await dbInit('instance_admin_api_serial', getLogger); @@ -26,6 +28,15 @@ beforeAll(async () => { }, db.rawDatabase, ); + + const { collectDbMetrics } = registerPrometheusMetrics( + app.config, + stores, + undefined as unknown as string, + app.config.eventBus, + app.services.instanceStatsService, + ); + refreshDbMetrics = collectDbMetrics; }); afterAll(async () => { @@ -39,6 +50,8 @@ test('should return instance statistics', async () => { createdByUserId: 9999, }); + await refreshDbMetrics(); + return app.request .get('/api/admin/instance-admin/statistics') .expect('Content-Type', /json/)