feat: extended SDK metrics (#7527)

This adds an extended metrics format to the metrics ingested by Unleash
and sent by running SDKs in the wild. Notably, we don't store this
information anywhere new in this PR, this is just streamed out to
Victoria metrics - the point of this project is insight, not analysis.

Two things to look out for in this PR:

- I've chosen to take extend the registration event and also send that
when we receive metrics. This means that the new data is received on
startup and on heartbeat. This takes us in the direction of collapsing
these two calls into one at a later point
- I've wrapped the existing metrics events in some "type safety", it
ain't much because we have 0 type safety on the event emitter so this
also has some if checks that look funny in TS that actually check if the
data shape is correct. Existing tests that check this are more or less
preserved

src/lib/__snapshots__/create-config.test.ts.snap

@@ -100,6 +100,7 @@ exports[`should create default config 1`] = `
       "enableLicenseChecker": false,
       "encryptEmails": false,
       "estimateTrafficDataCost": false,
+      "extendedMetrics": false,
       "extendedUsageMetrics": false,
       "featureLifecycle": false,
       "featureSearchFeedback": {

src/lib/features/metrics/client-metrics/metrics-service-v2.ts

@@ -4,15 +4,15 @@ import {
     type IFlagResolver,
     type IUnleashConfig,
 } from '../../../types';
-import type { IUnleashStores } from '../../../types';
+import type { ISdkHeartbeat, IUnleashStores } from '../../../types';
 import type { ToggleMetricsSummary } from '../../../types/models/metrics';
 import type {
     IClientMetricsEnv,
     IClientMetricsStoreV2,
 } from './client-metrics-store-v2-type';
 import { clientMetricsSchema } from '../shared/schema';
 import { compareAsc } from 'date-fns';
-import { CLIENT_METRICS } from '../../../types/events';
+import { CLIENT_METRICS, CLIENT_REGISTER } from '../../../types/events';
 import ApiUser, { type IApiUser } from '../../../types/api-user';
 import { ALL } from '../../../types/models/api-token';
 import type { IUser } from '../../../types/user';
@@ -157,6 +157,22 @@ export default class ClientMetricsServiceV2 {
             `Got ${toggleNames.length} (${validatedToggleNames.length} valid) metrics from ${clientIp}`,
         );
 
+        if (data.sdkVersion) {
+            const [sdkName, sdkVersion] = data.sdkVersion.split(':');
+            const heartbeatEvent: ISdkHeartbeat = {
+                sdkName,
+                sdkVersion,
+                metadata: {
+                    platformName: data.platformName,
+                    platformVersion: data.platformVersion,
+                    yggdrasilVersion: data.yggdrasilVersion,
+                    specVersion: data.specVersion,
+                },
+            };
+
+            this.config.eventBus.emit(CLIENT_REGISTER, heartbeatEvent);
+        }
+
         if (validatedToggleNames.length > 0) {
             const clientMetrics: IClientMetricsEnv[] = validatedToggleNames.map(
                 (name) => ({

src/lib/features/metrics/instance/instance-service.ts

@@ -12,7 +12,7 @@ import type {
 import type { IFeatureToggleStore } from '../../feature-toggle/types/feature-toggle-store-type';
 import type { IStrategyStore } from '../../../types/stores/strategy-store';
 import type { IClientInstanceStore } from '../../../types/stores/client-instance-store';
-import type { IClientApp } from '../../../types/model';
+import type { IClientApp, ISdkHeartbeat } from '../../../types/model';
 import { clientRegisterSchema } from '../shared/schema';
 
 import type { IClientMetricsStoreV2 } from '../client-metrics/client-metrics-store-v2-type';
@@ -105,7 +105,22 @@ export default class ClientInstanceService {
         value.clientIp = clientIp;
         value.createdBy = SYSTEM_USER.username!;
         this.seenClients[this.clientKey(value)] = value;
-        this.eventStore.emit(CLIENT_REGISTER, value);
+
+        if (value.sdkVersion && value.sdkVersion.indexOf(':') > -1) {
+            const [sdkName, sdkVersion] = value.sdkVersion.split(':');
+            const heartbeatEvent: ISdkHeartbeat = {
+                sdkName,
+                sdkVersion,
+                metadata: {
+                    platformName: data.platformName,
+                    platformVersion: data.platformVersion,
+                    yggdrasilVersion: data.yggdrasilVersion,
+                    specVersion: data.specVersion,
+                },
+            };
+
+            this.eventStore.emit(CLIENT_REGISTER, heartbeatEvent);
+        }
     }
 
     async announceUnannounced(): Promise<void> {

src/lib/metrics.test.ts

@@ -239,40 +239,47 @@ test('Should collect metrics for database', async () => {
 
 test('Should collect metrics for client sdk versions', async () => {
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-node:3.2.5',
+        sdkName: 'unleash-client-node',
+        sdkVersion: '3.2.5',
     });
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-node:3.2.5',
+        sdkName: 'unleash-client-node',
+        sdkVersion: '3.2.5',
     });
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-node:3.2.5',
+        sdkName: 'unleash-client-node',
+        sdkVersion: '3.2.5',
     });
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-java:5.0.0',
+        sdkName: 'unleash-client-java',
+        sdkVersion: '5.0.0',
     });
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-java:5.0.0',
+        sdkName: 'unleash-client-java',
+        sdkVersion: '5.0.0',
     });
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-java:5.0.0',
+        sdkName: 'unleash-client-java',
+        sdkVersion: '5.0.0',
     });
     const metrics = await prometheusRegister.getSingleMetricAsString(
         'client_sdk_versions',
     );
     expect(metrics).toMatch(
-        /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\} 3/,
+        /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\,platformName=\"not-set\",platformVersion=\"not-set\",yggdrasilVersion=\"not-set\",specVersion=\"not-set\"} 3/,
     );
     expect(metrics).toMatch(
-        /client_sdk_versions\{sdk_name="unleash-client-java",sdk_version="5\.0\.0"\} 3/,
+        /client_sdk_versions\{sdk_name="unleash-client-java",sdk_version="5\.0\.0"\,platformName=\"not-set\",platformVersion=\"not-set\",yggdrasilVersion=\"not-set\",specVersion=\"not-set\"} 3/,
     );
     eventStore.emit(CLIENT_REGISTER, {
-        sdkVersion: 'unleash-client-node:3.2.5',
+        sdkName: 'unleash-client-node',
+        sdkVersion: '3.2.5',
     });
     const newmetrics = await prometheusRegister.getSingleMetricAsString(
         'client_sdk_versions',
     );
     expect(newmetrics).toMatch(
-        /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\} 4/,
+        /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\,platformName=\"not-set\",platformVersion=\"not-set\",yggdrasilVersion=\"not-set\",specVersion=\"not-set\"} 4/,
     );
 });
 

src/lib/metrics.ts

@@ -24,7 +24,7 @@ import type { IUnleashConfig } from './types/option';
 import type { ISettingStore, IUnleashStores } from './types/stores';
 import { hoursToMilliseconds, minutesToMilliseconds } from 'date-fns';
 import type { InstanceStatsService } from './features/instance-stats/instance-stats-service';
-import type { IEnvironment } from './types';
+import type { IEnvironment, ISdkHeartbeat } from './types';
 import {
     createCounter,
     createGauge,
@@ -51,6 +51,7 @@ export default class MetricsMonitor {
         }
 
         const { eventStore, environmentStore } = stores;
+        const { flagResolver } = config;
 
         const cachedEnvironments: () => Promise<IEnvironment[]> = memoizee(
             async () => environmentStore.getAll(),
@@ -244,7 +245,14 @@ export default class MetricsMonitor {
         const clientSdkVersionUsage = createCounter({
             name: 'client_sdk_versions',
             help: 'Which sdk versions are being used',
-            labelNames: ['sdk_name', 'sdk_version'],
+            labelNames: [
+                'sdk_name',
+                'sdk_version',
+                'platformName',
+                'platformVersion',
+                'yggdrasilVersion',
+                'specVersion',
+            ],
         });
 
         const productionChanges30 = createGauge({
@@ -784,15 +792,36 @@ export default class MetricsMonitor {
             }
         });
 
-        eventStore.on(CLIENT_REGISTER, (m) => {
-            if (m.sdkVersion && m.sdkVersion.indexOf(':') > -1) {
-                const [sdkName, sdkVersion] = m.sdkVersion.split(':');
+        eventStore.on(CLIENT_REGISTER, (heartbeatEvent: ISdkHeartbeat) => {
+            if (!heartbeatEvent.sdkName || !heartbeatEvent.sdkVersion) {
+                return;
+            }
+
+            if (flagResolver.isEnabled('extendedMetrics')) {
                 clientSdkVersionUsage.increment({
-                    sdk_name: sdkName,
-                    sdk_version: sdkVersion,
+                    sdk_name: heartbeatEvent.sdkName,
+                    sdk_version: heartbeatEvent.sdkVersion,
+                    platformName:
+                        heartbeatEvent.metadata?.platformName ?? 'not-set',
+                    platformVersion:
+                        heartbeatEvent.metadata?.platformVersion ?? 'not-set',
+                    yggdrasilVersion:
+                        heartbeatEvent.metadata?.yggdrasilVersion ?? 'not-set',
+                    specVersion:
+                        heartbeatEvent.metadata?.specVersion ?? 'not-set',
+                });
+            } else {
+                clientSdkVersionUsage.increment({
+                    sdk_name: heartbeatEvent.sdkName,
+                    sdk_version: heartbeatEvent.sdkVersion,
+                    platformName: 'not-set',
+                    platformVersion: 'not-set',
+                    yggdrasilVersion: 'not-set',
+                    specVersion: 'not-set',
                 });
             }
         });
+
         eventStore.on(PROJECT_ENVIRONMENT_REMOVED, ({ project }) => {
             projectEnvironmentsDisabled.increment({ project_id: project });
         });

src/lib/openapi/spec/client-application-schema.ts

@@ -30,6 +30,30 @@ export const clientApplicationSchema = {
             type: 'string',
             example: 'development',
         },
+        platformName: {
+            description:
+                'The platform the application is running on. For languages that compile to binaries, this can be omitted',
+            type: 'string',
+            example: '.NET Core',
+        },
+        platformVersion: {
+            description:
+                'The version of the platform the application is running on. Languages that compile to binaries, this is expected to be the compiler version used to assemble the binary.',
+            type: 'string',
+            example: '3.1',
+        },
+        yggdrasilVersion: {
+            description:
+                'The semantic version of the Yggdrasil engine used by the client. If the client is using a native engine this can be omitted.',
+            type: 'string',
+            example: '1.0.0',
+        },
+        specVersion: {
+            description:
+                'The version of the Unleash client specification the client supports',
+            type: 'string',
+            example: '3.0.0',
+        },
         interval: {
             type: 'number',
             description:

src/lib/openapi/spec/client-metrics-schema.ts

@@ -21,9 +21,41 @@ export const clientMetricsSchema = {
             example: 'application-name-dacb1234',
         },
         environment: {
-            description: 'Which environment the application is running in',
+            description:
+                'Which environment the application is running in. This property was deprecated in v5. This can be determined by the API key calling this endpoint.',
             type: 'string',
             example: 'development',
+            deprecated: true,
+        },
+        sdkVersion: {
+            type: 'string',
+            description:
+                'An SDK version identifier. Usually formatted as "unleash-client-<language>:<version>"',
+            example: 'unleash-client-java:7.0.0',
+        },
+        platformName: {
+            description:
+                'The platform the application is running on. For languages that compile to binaries, this can be omitted',
+            type: 'string',
+            example: '.NET Core',
+        },
+        platformVersion: {
+            description:
+                'The version of the platform the application is running on. Languages that compile to binaries, this is expected to be the compiler version used to assemble the binary.',
+            type: 'string',
+            example: '3.1',
+        },
+        yggdrasilVersion: {
+            description:
+                'The semantic version of the Yggdrasil engine used by the client. If the client is using a native engine this can be omitted.',
+            type: 'string',
+            example: '1.0.0',
+        },
+        specVersion: {
+            description:
+                'The version of the Unleash client specification the client supports',
+            type: 'string',
+            example: '3.0.0',
         },
         bucket: {
             type: 'object',

src/lib/types/experimental.ts

@@ -62,7 +62,8 @@ export type IFlagKey =
     | 'commandBarUI'
     | 'flagCreator'
     | 'anonymizeProjectOwners'
-    | 'resourceLimits';
+    | 'resourceLimits'
+    | 'extendedMetrics';
 
 export type IFlags = Partial<{ [key in IFlagKey]: boolean | Variant }>;
 
@@ -299,6 +300,10 @@ const flags: IFlags = {
         process.env.UNLEASH_EXPERIMENTAL_RESOURCE_LIMITS,
         false,
     ),
+    extendedMetrics: parseEnvVarBoolean(
+        process.env.UNLEASH_EXPERIMENTAL_EXTENDED_METRICS,
+        false,
+    ),
 };
 
 export const defaultExperimentalOptions: IExperimentalOptions = {

src/lib/types/model.ts

@@ -451,6 +451,10 @@ export interface IClientApp {
     icon?: string;
     description?: string;
     color?: string;
+    platformName?: string;
+    platformVersion?: string;
+    yggdrasilVersion?: string;
+    specVersion?: string;
 }
 
 export interface IAppFeature {
@@ -598,3 +602,16 @@ export interface IUserAccessOverview {
     rootRole: string;
     groupProjects: string[];
 }
+
+export interface ISdkHeartbeat {
+    sdkVersion: string;
+    sdkName: string;
+    metadata: ISdkHeartbeatMetadata;
+}
+
+export interface ISdkHeartbeatMetadata {
+    platformName?: string;
+    platformVersion?: string;
+    yggdrasilVersion?: string;
+    specVersion?: string;
+}

src/server-dev.ts

@@ -55,6 +55,7 @@ process.nextTick(async () => {
                         commandBarUI: true,
                         flagCreator: true,
                         resourceLimits: true,
+                        extendedMetrics: true,
                     },
                 },
                 authentication: {