feat: track timeToFirstToken in LDAIConfigTracker

launchdarkly · Jan 22, 2025 · 1fc9e76 · 1fc9e76
1 parent 1ff96ce
commit 1fc9e76
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 0 deletions.
diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts
@@ -46,6 +46,18 @@ it('tracks duration of async function', async () => {
   );
 });
 
+it('tracks time to first token', () => {
+  const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
+  tracker.trackTimeToFirstToken(1000);
+
+  expect(mockTrack).toHaveBeenCalledWith(
+    '$ld:ai:tokens:ttf',
+    testContext,
+    { configKey, variationKey },
+    1000,
+  );
+});
+
 it('tracks positive feedback', () => {
   const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
   tracker.trackFeedback({ kind: LDFeedbackKind.Positive });

diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts
@@ -41,6 +41,16 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
     }
   }
 
+  trackTimeToFirstToken(timeToFirstTokenMs: number) {
+    this._trackedMetrics.timeToFirstTokenMs = timeToFirstTokenMs;
+    this._ldClient.track(
+      '$ld:ai:tokens:ttf',
+      this._context,
+      this._getTrackData(),
+      timeToFirstTokenMs,
+    );
+  }
+
   trackFeedback(feedback: { kind: LDFeedbackKind }): void {
     this._trackedMetrics.feedback = feedback;
     if (feedback.kind === LDFeedbackKind.Positive) {

diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts
@@ -23,6 +23,11 @@ export interface LDAIMetricSummary {
    * Any sentiment about the generation.
    */
   feedback?: { kind: LDFeedbackKind };
+
+  /**
+   * Time to first token for this generation.
+   */
+  timeToFirstTokenMs?: number;
 }
 
 /**
@@ -62,6 +67,13 @@ export interface LDAIConfigTracker {
    */
   trackFeedback(feedback: { kind: LDFeedbackKind }): void;
 
+  /**
+   * Track the time to first token for this generation.
+   *
+   * @param timeToFirstTokenMs The duration in milliseconds.
+   */
+  trackTimeToFirstToken(timeToFirstTokenMs: number): void;
+
   /**
    * Track the duration of execution of the provided function.
    *