feat: start collecting prometheus metrics for onboarding events (#8012)

We start collecting prometheus metrics for onboarding events.

Co-authored-by: @kwasniew

src/lib/db/index.ts

@@ -52,6 +52,7 @@ import { LargestResourcesReadModel } from '../features/metrics/sizes/largest-res
 import { IntegrationEventsStore } from '../features/integration-events/integration-events-store';
 import { FeatureCollaboratorsReadModel } from '../features/feature-toggle/feature-collaborators-read-model';
 import { createProjectReadModel } from '../features/project/createProjectReadModel';
+import { OnboardingReadModel } from '../features/onboarding/onboarding-read-model';
 
 export const createStores = (
     config: IUnleashConfig,
@@ -171,6 +172,7 @@ export const createStores = (
         projectFlagCreatorsReadModel: new ProjectFlagCreatorsReadModel(db),
         featureLifecycleStore: new FeatureLifecycleStore(db),
         featureStrategiesReadModel: new FeatureStrategiesReadModel(db),
+        onboardingReadModel: new OnboardingReadModel(db),
         featureLifecycleReadModel: new FeatureLifecycleReadModel(
             db,
             config.flagResolver,

src/lib/features/feature-lifecycle/feature-lifecycle-store.ts

@@ -63,6 +63,7 @@ export class FeatureLifecycleStore implements IFeatureLifecycleStore {
                     stage: stage.stage,
                     status: stage.status,
                     status_value: stage.statusValue,
+                    created_at: new Date(),
                 })),
             )
             .returning('*')

src/lib/features/onboarding/fake-onboarding-read-model.ts

@@ -0,0 +1,14 @@
+import type { IOnboardingReadModel } from '../../types';
+import type { InstanceOnboarding } from './onboarding-read-model-type';
+
+export class FakeOnboardingReadModel implements IOnboardingReadModel {
+    getInstanceOnboardingMetrics(): Promise<InstanceOnboarding> {
+        return Promise.resolve({
+            firstLogin: null,
+            secondLogin: null,
+            firstFeatureFlag: null,
+            firstPreLive: null,
+            firstLive: null,
+        });
+    }
+}

src/lib/features/onboarding/onboarding-read-model-type.ts

@@ -0,0 +1,14 @@
+/**
+ * All the values are in minutes
+ */
+export type InstanceOnboarding = {
+    firstLogin: number | null;
+    secondLogin: number | null;
+    firstFeatureFlag: number | null;
+    firstPreLive: number | null;
+    firstLive: number | null;
+};
+
+export interface IOnboardingReadModel {
+    getInstanceOnboardingMetrics(): Promise<InstanceOnboarding>;
+}

src/lib/features/onboarding/onboarding-read-model.test.ts

@@ -0,0 +1,106 @@
+import dbInit, { type ITestDb } from '../../../test/e2e/helpers/database-init';
+import getLogger from '../../../test/fixtures/no-logger';
+import type {
+    IFeatureLifecycleStore,
+    IFeatureToggleStore,
+    IUserStore,
+} from '../../types';
+import { OnboardingReadModel } from './onboarding-read-model';
+import type { IOnboardingReadModel } from './onboarding-read-model-type';
+import { minutesToMilliseconds } from 'date-fns';
+
+let db: ITestDb;
+let onboardingReadModel: IOnboardingReadModel;
+let userStore: IUserStore;
+let lifecycleStore: IFeatureLifecycleStore;
+let featureToggleStore: IFeatureToggleStore;
+
+beforeAll(async () => {
+    db = await dbInit('onboarding_read_model', getLogger, {
+        experimental: { flags: { onboardingMetrics: true } },
+    });
+    onboardingReadModel = new OnboardingReadModel(db.rawDatabase);
+    userStore = db.stores.userStore;
+    lifecycleStore = db.stores.featureLifecycleStore;
+    featureToggleStore = db.stores.featureToggleStore;
+});
+
+afterAll(async () => {
+    if (db) {
+        await db.destroy();
+    }
+});
+
+beforeEach(async () => {
+    await userStore.deleteAll();
+    jest.useRealTimers();
+});
+
+test('can get onboarding durations', async () => {
+    const initialResult =
+        await onboardingReadModel.getInstanceOnboardingMetrics();
+    expect(initialResult).toMatchObject({
+        firstLogin: null,
+        secondLogin: null,
+        firstFeatureFlag: null,
+        firstPreLive: null,
+        firstLive: null,
+    });
+
+    const firstUser = await userStore.insert({});
+    await userStore.successfullyLogin(firstUser);
+
+    const firstLoginResult =
+        await onboardingReadModel.getInstanceOnboardingMetrics();
+    expect(firstLoginResult).toMatchObject({
+        firstLogin: 0,
+        secondLogin: null,
+    });
+    jest.useFakeTimers();
+    jest.advanceTimersByTime(minutesToMilliseconds(10));
+
+    const secondUser = await userStore.insert({});
+    await userStore.successfullyLogin(secondUser);
+
+    jest.advanceTimersByTime(minutesToMilliseconds(10));
+
+    await featureToggleStore.create('default', {
+        name: 'test',
+        createdByUserId: secondUser.id,
+    });
+
+    await lifecycleStore.insert([
+        {
+            feature: 'test',
+            stage: 'initial',
+        },
+    ]);
+
+    jest.advanceTimersByTime(minutesToMilliseconds(10));
+
+    await lifecycleStore.insert([
+        {
+            feature: 'test',
+            stage: 'pre-live',
+        },
+    ]);
+
+    jest.advanceTimersByTime(minutesToMilliseconds(10));
+
+    await lifecycleStore.insert([
+        {
+            feature: 'test',
+            stage: 'live',
+        },
+    ]);
+
+    const secondLoginResult =
+        await onboardingReadModel.getInstanceOnboardingMetrics();
+    expect(secondLoginResult).toMatchObject({
+        firstLogin: 0,
+        secondLogin: 10,
+        firstFeatureFlag: 20,
+        firstPreLive: 30,
+        firstLive: 40,
+    });
+});

src/lib/features/onboarding/onboarding-read-model.ts

@@ -0,0 +1,92 @@
+import type { Db } from '../../db/db';
+import type {
+    IOnboardingReadModel,
+    InstanceOnboarding,
+} from './onboarding-read-model-type';
+import { millisecondsToMinutes } from 'date-fns';
+
+interface IOnboardingUser {
+    first_login: string;
+}
+const parseStringToNumber = (value: string): number | null => {
+    return Number.isNaN(Number(value)) ? null : Number(value);
+};
+
+const calculateTimeDifferenceInMinutes = (date1?: Date, date2?: Date) => {
+    if (date1 && date2) {
+        const diffInMilliseconds = date2.getTime() - date1.getTime();
+        return millisecondsToMinutes(diffInMilliseconds);
+    }
+    return null;
+};
+
+export class OnboardingReadModel implements IOnboardingReadModel {
+    private db: Db;
+
+    constructor(db: Db) {
+        this.db = db;
+    }
+
+    async getInstanceOnboardingMetrics(): Promise<InstanceOnboarding> {
+        const firstUserCreatedResult = await this.db('users')
+            .select('created_at')
+            .orderBy('created_at')
+            .first();
+        const firstLoginResult = await this.db('users')
+            .select('first_seen_at')
+            .orderBy('first_seen_at')
+            .limit(2);
+
+        const firstInitialResult = await this.db('feature_lifecycles')
+            .select('created_at')
+            .where('stage', 'initial')
+            .orderBy('created_at')
+            .first();
+        const firstPreLiveResult = await this.db('feature_lifecycles')
+            .select('created_at')
+            .where('stage', 'pre-live')
+            .orderBy('created_at')
+            .first();
+        const firstLiveResult = await this.db('feature_lifecycles')
+            .select('created_at')
+            .where('stage', 'live')
+            .orderBy('created_at')
+            .first();
+
+        const createdAt = firstUserCreatedResult?.created_at;
+        const firstLogin = firstLoginResult[0]?.first_seen_at;
+        const secondLogin = firstLoginResult[1]?.first_seen_at;
+        const firstInitial = firstInitialResult?.created_at;
+        const firstPreLive = firstPreLiveResult?.created_at;
+        const firstLive = firstLiveResult?.created_at;
+
+        const firstLoginDiff = calculateTimeDifferenceInMinutes(
+            createdAt,
+            firstLogin,
+        );
+        const secondLoginDiff = calculateTimeDifferenceInMinutes(
+            createdAt,
+            secondLogin,
+        );
+        const firstFlagDiff = calculateTimeDifferenceInMinutes(
+            createdAt,
+            firstInitial,
+        );
+        const firstPreLiveDiff = calculateTimeDifferenceInMinutes(
+            createdAt,
+            firstPreLive,
+        );
+        const firstLiveDiff = calculateTimeDifferenceInMinutes(
+            createdAt,
+            firstLive,
+        );
+
+        return {
+            firstLogin: firstLoginDiff,
+            secondLogin: secondLoginDiff,
+            firstFeatureFlag: firstFlagDiff,
+            firstPreLive: firstPreLiveDiff,
+            firstLive: firstLiveDiff,
+        };
+    }
+}

src/lib/metrics.ts

@@ -307,6 +307,12 @@ export default class MetricsMonitor {
             help: 'Duration of feature lifecycle stages',
         });
 
+        const onboardingDuration = createGauge({
+            name: 'onboarding_duration',
+            labelNames: ['event'],
+            help: 'firstLogin, secondLogin, firstFeatureFlag, firstPreLive, firstLive from first user creation',
+        });
+
         const featureLifecycleStageCountByProject = createGauge({
             name: 'feature_lifecycle_stage_count_by_project',
             help: 'Count features in a given stage by project id',
@@ -388,6 +394,7 @@ export default class MetricsMonitor {
                     largestProjectEnvironments,
                     largestFeatureEnvironments,
                     deprecatedTokens,
+                    onboardingMetrics,
                 ] = await Promise.all([
                     stores.featureStrategiesReadModel.getMaxFeatureStrategies(),
                     stores.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(),
@@ -402,6 +409,9 @@ export default class MetricsMonitor {
                         1,
                     ),
                     stores.apiTokenStore.countDeprecatedTokens(),
+                    flagResolver.isEnabled('onboardingMetrics')
+                        ? stores.onboardingReadModel.getInstanceOnboardingMetrics()
+                        : Promise.resolve({}),
                 ]);
 
                 featureFlagsTotal.reset();
@@ -529,6 +539,16 @@ export default class MetricsMonitor {
                         .set(featureEnvironment.size);
                 }
 
+                Object.keys(onboardingMetrics).forEach((key) => {
+                    if (Number.isInteger(onboardingMetrics[key])) {
+                        onboardingDuration
+                            .labels({
+                                event: key,
+                            })
+                            .set(onboardingMetrics[key]);
+                    }
+                });
+
                 for (const [resource, limit] of Object.entries(
                     config.resourceLimits,
                 )) {

src/lib/types/stores.ts

@@ -49,6 +49,7 @@ import { ILargestResourcesReadModel } from '../features/metrics/sizes/largest-re
 import type { IntegrationEventsStore } from '../features/integration-events/integration-events-store';
 import { IFeatureCollaboratorsReadModel } from '../features/feature-toggle/types/feature-collaborators-read-model-type';
 import type { IProjectReadModel } from '../features/project/project-read-model-type';
+import { IOnboardingReadModel } from '../features/onboarding/onboarding-read-model-type';
 
 export interface IUnleashStores {
     accessStore: IAccessStore;
@@ -102,6 +103,7 @@ export interface IUnleashStores {
     integrationEventsStore: IntegrationEventsStore;
     featureCollaboratorsReadModel: IFeatureCollaboratorsReadModel;
     projectReadModel: IProjectReadModel;
+    onboardingReadModel: IOnboardingReadModel;
 }
 
 export {
@@ -152,6 +154,7 @@ export {
     IFeatureLifecycleReadModel,
     ILargestResourcesReadModel,
     IFeatureCollaboratorsReadModel,
+    IOnboardingReadModel,
     type IntegrationEventsStore,
     type IProjectReadModel,
 };

src/test/fixtures/store.ts

@@ -52,6 +52,7 @@ import { FakeFeatureLifecycleReadModel } from '../../lib/features/feature-lifecy
 import { FakeLargestResourcesReadModel } from '../../lib/features/metrics/sizes/fake-largest-resources-read-model';
 import { FakeFeatureCollaboratorsReadModel } from '../../lib/features/feature-toggle/fake-feature-collaborators-read-model';
 import { createFakeProjectReadModel } from '../../lib/features/project/createProjectReadModel';
+import { FakeOnboardingReadModel } from '../../lib/features/onboarding/fake-onboarding-read-model';
 
 const db = {
     select: () => ({
@@ -109,6 +110,7 @@ const createStores: () => IUnleashStores = () => {
         featureLifecycleStore: new FakeFeatureLifecycleStore(),
         featureStrategiesReadModel: new FakeFeatureStrategiesReadModel(),
         featureLifecycleReadModel: new FakeFeatureLifecycleReadModel(),
+        onboardingReadModel: new FakeOnboardingReadModel(),
         largestResourcesReadModel: new FakeLargestResourcesReadModel(),
         integrationEventsStore: {} as IntegrationEventsStore,
         featureCollaboratorsReadModel: new FakeFeatureCollaboratorsReadModel(),