From dc9a58bc650039507d81d69ad293dd821ff3801a Mon Sep 17 00:00:00 2001 From: sjaanus Date: Thu, 19 Dec 2024 12:49:48 +0200 Subject: [PATCH 1/4] feat: store memory footprints to grafana --- .../client-feature-toggle.controller.ts | 47 ++++++++++++++++++- .../delta/client-feature-toggle-delta.ts | 31 +++++++++--- .../delta/createClientFeatureToggleDelta.ts | 1 + .../delta/revision-delta.ts | 2 +- src/lib/metric-events.ts | 8 +++- src/lib/metrics.ts | 20 ++++++++ 6 files changed, 100 insertions(+), 9 deletions(-) diff --git a/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts b/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts index ee19793d6ba7..24f4b2058069 100644 --- a/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts +++ b/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts @@ -35,6 +35,7 @@ import { import type ConfigurationRevisionService from '../feature-toggle/configuration-revision-service'; import type { ClientFeatureToggleService } from './client-feature-toggle-service'; import { + CLIENT_FEATURES_MEMORY, CLIENT_METRICS_NAMEPREFIX, CLIENT_METRICS_TAGS, } from '../../internals'; @@ -69,6 +70,8 @@ export default class FeatureController extends Controller { private eventBus: EventEmitter; + private clientFeaturesCacheMap = new Map(); + private featuresAndSegments: ( query: IFeatureToggleQuery, etag: string, @@ -162,6 +165,36 @@ export default class FeatureController extends Controller { private async resolveFeaturesAndSegments( query?: IFeatureToggleQuery, ): Promise<[FeatureConfigurationClient[], IClientSegment[]]> { + if (this.flagResolver.isEnabled('deltaApi')) { + const features = + await this.clientFeatureToggleService.getClientFeatures(query); + + const segments = + await this.clientFeatureToggleService.getActiveSegmentsForClient(); + + try { + console.log(`storing features ${features.length}`); + const featuresSize = this.getCacheSizeInBytes(features); + const segmentsSize = this.getCacheSizeInBytes(segments); + console.log( + `features storing ${featuresSize}, ${segmentsSize}`, + ); + this.clientFeaturesCacheMap.set( + JSON.stringify(query), + featuresSize + segmentsSize, + ); + + await this.clientFeatureToggleService.getClientDelta( + undefined, + query!, + ); + this.storeFootprint(); + } catch (e) { + this.logger.error('Delta diff failed', e); + } + + return [features, segments]; + } return Promise.all([ this.clientFeatureToggleService.getClientFeatures(query), this.clientFeatureToggleService.getActiveSegmentsForClient(), @@ -270,7 +303,6 @@ export default class FeatureController extends Controller { query, etag, ); - if (this.clientSpecService.requestSupportsSpec(req, 'segments')) { this.openApiService.respondWithValidation( 200, @@ -335,4 +367,17 @@ export default class FeatureController extends Controller { }, ); } + + storeFootprint() { + let memory = 0; + for (const value of this.clientFeaturesCacheMap.values()) { + memory += value; + } + this.eventBus.emit(CLIENT_FEATURES_MEMORY, { memory }); + } + + getCacheSizeInBytes(value: any): number { + const jsonString = JSON.stringify(value); + return Buffer.byteLength(jsonString, 'utf8'); + } } diff --git a/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts b/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts index 271221426dba..8d2c8aeb8e9a 100644 --- a/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts +++ b/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts @@ -13,6 +13,8 @@ import type { FeatureConfigurationDeltaClient, IClientFeatureToggleDeltaReadModel, } from './client-feature-toggle-delta-read-model-type'; +import { CLIENT_DELTA_MEMORY } from '../../../metric-events'; +import type EventEmitter from 'events'; type DeletedFeature = { name: string; @@ -105,20 +107,21 @@ export class ClientFeatureToggleDelta { private currentRevisionId: number = 0; - private interval: NodeJS.Timer; - private flagResolver: IFlagResolver; private configurationRevisionService: ConfigurationRevisionService; private readonly segmentReadModel: ISegmentReadModel; + private eventBus: EventEmitter; + constructor( clientFeatureToggleDeltaReadModel: IClientFeatureToggleDeltaReadModel, segmentReadModel: ISegmentReadModel, eventStore: IEventStore, configurationRevisionService: ConfigurationRevisionService, flagResolver: IFlagResolver, + eventBus: EventEmitter, ) { this.eventStore = eventStore; this.configurationRevisionService = configurationRevisionService; @@ -126,6 +129,7 @@ export class ClientFeatureToggleDelta { clientFeatureToggleDeltaReadModel; this.flagResolver = flagResolver; this.segmentReadModel = segmentReadModel; + this.eventBus = eventBus; this.onUpdateRevisionEvent = this.onUpdateRevisionEvent.bind(this); this.delta = {}; @@ -161,6 +165,8 @@ export class ClientFeatureToggleDelta { await this.updateSegments(); } + // TODO: 19.12 this logic seems to be not logical, when no revisionId is coming, it should not go to db, but take latest from cache + // Should get the latest state if revision does not exist or if sdkRevision is not present // We should be able to do this without going to the database by merging revisions from the delta with // the base case @@ -203,12 +209,13 @@ export class ClientFeatureToggleDelta { private async onUpdateRevisionEvent() { if (this.flagResolver.isEnabled('deltaApi')) { - await this.listenToRevisionChange(); + await this.updateFeaturesDelta(); await this.updateSegments(); + this.storeFootprint(); } } - public async listenToRevisionChange() { + public async updateFeaturesDelta() { const keys = Object.keys(this.delta); if (keys.length === 0) return; @@ -248,7 +255,6 @@ export class ClientFeatureToggleDelta { removed, }); } - this.currentRevisionId = latestRevision; } @@ -279,8 +285,9 @@ export class ClientFeatureToggleDelta { removed: [], }, ]); - this.delta[environment] = delta; + + this.storeFootprint(); } async getClientFeatures( @@ -294,4 +301,16 @@ export class ClientFeatureToggleDelta { private async updateSegments(): Promise { this.segments = await this.segmentReadModel.getActiveForClient(); } + + storeFootprint() { + const featuresMemory = this.getCacheSizeInBytes(this.delta); + const segmentsMemory = this.getCacheSizeInBytes(this.segments); + const memory = featuresMemory + segmentsMemory; + this.eventBus.emit(CLIENT_DELTA_MEMORY, { memory }); + } + + getCacheSizeInBytes(value: any): number { + const jsonString = JSON.stringify(value); + return Buffer.byteLength(jsonString, 'utf8'); + } } diff --git a/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts b/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts index 9252357b2f32..f55551883405 100644 --- a/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts +++ b/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts @@ -28,6 +28,7 @@ export const createClientFeatureToggleDelta = ( eventStore, configurationRevisionService, flagResolver, + eventBus, ); return clientFeatureToggleDelta; diff --git a/src/lib/features/client-feature-toggles/delta/revision-delta.ts b/src/lib/features/client-feature-toggles/delta/revision-delta.ts index da8c553d1a6c..112bd3de60cb 100644 --- a/src/lib/features/client-feature-toggles/delta/revision-delta.ts +++ b/src/lib/features/client-feature-toggles/delta/revision-delta.ts @@ -12,7 +12,7 @@ export class RevisionDelta { private delta: Revision[]; private maxLength: number; - constructor(data: Revision[] = [], maxLength: number = 100) { + constructor(data: Revision[] = [], maxLength: number = 20) { this.delta = data; this.maxLength = maxLength; } diff --git a/src/lib/metric-events.ts b/src/lib/metric-events.ts index 280f66c981e9..6591ea4c672f 100644 --- a/src/lib/metric-events.ts +++ b/src/lib/metric-events.ts @@ -16,6 +16,8 @@ const REQUEST_ORIGIN = 'request_origin' as const; const ADDON_EVENTS_HANDLED = 'addon-event-handled' as const; const CLIENT_METRICS_NAMEPREFIX = 'client-api-nameprefix'; const CLIENT_METRICS_TAGS = 'client-api-tags'; +const CLIENT_FEATURES_MEMORY = 'client_features_memory'; +const CLIENT_DELTA_MEMORY = 'client_delta_memory'; type MetricEvent = | typeof REQUEST_TIME @@ -32,7 +34,9 @@ type MetricEvent = | typeof EXCEEDS_LIMIT | typeof REQUEST_ORIGIN | typeof CLIENT_METRICS_NAMEPREFIX - | typeof CLIENT_METRICS_TAGS; + | typeof CLIENT_METRICS_TAGS + | typeof CLIENT_FEATURES_MEMORY + | typeof CLIENT_DELTA_MEMORY; type RequestOriginEventPayload = { type: 'UI' | 'API'; @@ -82,6 +86,8 @@ export { ADDON_EVENTS_HANDLED, CLIENT_METRICS_NAMEPREFIX, CLIENT_METRICS_TAGS, + CLIENT_FEATURES_MEMORY, + CLIENT_DELTA_MEMORY, type MetricEvent, type MetricEventPayload, emitMetricEvent, diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index a0d8486f4114..02b1373336cc 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -624,6 +624,16 @@ export function registerPrometheusMetrics( help: 'Number of API tokens without a project', }); + const clientFeaturesMemory = createGauge({ + name: 'client_features_memory', + help: 'The amount of memory client features endpoint is using for caching', + }); + + const clientDeltaMemory = createGauge({ + name: 'client_delta_memory', + help: 'The amount of memory client features delta endpoint is using for caching', + }); + const orphanedTokensActive = createGauge({ name: 'orphaned_api_tokens_active', help: 'Number of API tokens without a project, last seen within 3 months', @@ -752,6 +762,16 @@ export function registerPrometheusMetrics( tagsUsed.inc(); }); + eventBus.on(events.CLIENT_FEATURES_MEMORY, (event: { memory: number }) => { + clientFeaturesMemory.reset(); + clientFeaturesMemory.set(event.memory); + }); + + eventBus.on(events.CLIENT_DELTA_MEMORY, (event: { memory: number }) => { + clientDeltaMemory.reset(); + clientDeltaMemory.set(event.memory); + }); + events.onMetricEvent( eventBus, events.REQUEST_ORIGIN, From 8d77f698ef171a3c6234154905f4463458bc49db Mon Sep 17 00:00:00 2001 From: sjaanus Date: Thu, 19 Dec 2024 12:54:41 +0200 Subject: [PATCH 2/4] feat: store memory footprints to grafana --- .../delta/client-feature-toggle-delta.ts | 21 +++++++++++++------ .../delta/createClientFeatureToggleDelta.ts | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts b/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts index 8d2c8aeb8e9a..bc48523c86dc 100644 --- a/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts +++ b/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts @@ -5,6 +5,7 @@ import type { IFeatureToggleQuery, IFlagResolver, ISegmentReadModel, + IUnleashConfig, } from '../../../types'; import type ConfigurationRevisionService from '../../feature-toggle/configuration-revision-service'; import { UPDATE_REVISION } from '../../feature-toggle/configuration-revision-service'; @@ -15,6 +16,7 @@ import type { } from './client-feature-toggle-delta-read-model-type'; import { CLIENT_DELTA_MEMORY } from '../../../metric-events'; import type EventEmitter from 'events'; +import type { Logger } from '../../../logger'; type DeletedFeature = { name: string; @@ -115,13 +117,15 @@ export class ClientFeatureToggleDelta { private eventBus: EventEmitter; + private readonly logger: Logger; + constructor( clientFeatureToggleDeltaReadModel: IClientFeatureToggleDeltaReadModel, segmentReadModel: ISegmentReadModel, eventStore: IEventStore, configurationRevisionService: ConfigurationRevisionService, flagResolver: IFlagResolver, - eventBus: EventEmitter, + config: IUnleashConfig, ) { this.eventStore = eventStore; this.configurationRevisionService = configurationRevisionService; @@ -129,7 +133,8 @@ export class ClientFeatureToggleDelta { clientFeatureToggleDeltaReadModel; this.flagResolver = flagResolver; this.segmentReadModel = segmentReadModel; - this.eventBus = eventBus; + this.eventBus = config.eventBus; + this.logger = config.getLogger('delta/client-feature-toggle-delta.js'); this.onUpdateRevisionEvent = this.onUpdateRevisionEvent.bind(this); this.delta = {}; @@ -303,10 +308,14 @@ export class ClientFeatureToggleDelta { } storeFootprint() { - const featuresMemory = this.getCacheSizeInBytes(this.delta); - const segmentsMemory = this.getCacheSizeInBytes(this.segments); - const memory = featuresMemory + segmentsMemory; - this.eventBus.emit(CLIENT_DELTA_MEMORY, { memory }); + try { + const featuresMemory = this.getCacheSizeInBytes(this.delta); + const segmentsMemory = this.getCacheSizeInBytes(this.segments); + const memory = featuresMemory + segmentsMemory; + this.eventBus.emit(CLIENT_DELTA_MEMORY, { memory }); + } catch (e) { + this.logger.error('Client delta footprint error', e); + } } getCacheSizeInBytes(value: any): number { diff --git a/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts b/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts index f55551883405..e8540c0ab83e 100644 --- a/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts +++ b/src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts @@ -28,7 +28,7 @@ export const createClientFeatureToggleDelta = ( eventStore, configurationRevisionService, flagResolver, - eventBus, + config, ); return clientFeatureToggleDelta; From 9e199fd3fcae14bc2bbfcd878621191df7c54c28 Mon Sep 17 00:00:00 2001 From: sjaanus Date: Thu, 19 Dec 2024 12:55:30 +0200 Subject: [PATCH 3/4] feat: store memory footprints to grafana --- .../client-feature-toggles/delta/client-feature-toggle-delta.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts b/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts index bc48523c86dc..0d50b8d5a6c9 100644 --- a/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts +++ b/src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts @@ -90,7 +90,6 @@ export const calculateRequiredClientRevision = ( const targetedRevisions = revisions.filter( (revision) => revision.revisionId > requiredRevisionId, ); - console.log('targeted revisions', targetedRevisions); const projectFeatureRevisions = targetedRevisions.map((revision) => filterRevisionByProject(revision, projects), ); From dd848fcc4667c425b1af0c77a5a2455ef804f091 Mon Sep 17 00:00:00 2001 From: sjaanus Date: Thu, 19 Dec 2024 12:56:50 +0200 Subject: [PATCH 4/4] feat: store memory footprints to grafana --- .../client-feature-toggle.controller.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts b/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts index 24f4b2058069..b40fb31afbf5 100644 --- a/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts +++ b/src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts @@ -173,12 +173,8 @@ export default class FeatureController extends Controller { await this.clientFeatureToggleService.getActiveSegmentsForClient(); try { - console.log(`storing features ${features.length}`); const featuresSize = this.getCacheSizeInBytes(features); const segmentsSize = this.getCacheSizeInBytes(segments); - console.log( - `features storing ${featuresSize}, ${segmentsSize}`, - ); this.clientFeaturesCacheMap.set( JSON.stringify(query), featuresSize + segmentsSize,