feat: support passing metadata to prompts

* This allows more advanced custom ratings that leverage metadata that
  is tied to specific app prompts. e.g. golden response comparisons.

Author: devversion

examples/environments/remote_env/config.js

@@ -4,15 +4,51 @@
  * @import {EnvironmentConfig} from 'web-codegen-scorer';
  */
 
-import {getBuiltInRatings} from 'web-codegen-scorer';
+import {
+  EvalPromptWithMetadata,
+  getBuiltInRatings,
+  RatingKind,
+  RatingCategory,
+  RatingState,
+} from 'web-codegen-scorer';
 import {FakeRemoteExecutor} from './fake-executor';
 
 /** @type {EnvironmentConfig} */
 export default {
   displayName: 'Remote Env (example)',
   clientSideFramework: 'angular',
-  ratings: getBuiltInRatings(),
+  ratings: [
+    ...getBuiltInRatings(),
+    {
+      name: 'Test Metadata Rating',
+      id: 'test-metadata-rating',
+      kind: RatingKind.PER_BUILD,
+      category: RatingCategory.MEDIUM_IMPACT,
+      description: 'Testing the metadata of prompts',
+      scoreReduction: '100%',
+      rate: ctx => {
+        const metadata = /** @type {{goldenURL: string}} */ (ctx.prompt.metadata);
+        const found = ctx.generatedFiles.some(f => f.code.includes(metadata.goldenURL));
+
+        return {
+          state: RatingState.EXECUTED,
+          coefficient: found ? 1 : 0,
+          message: found ? `${metadata.goldenURL} found!` : `${metadata.goldenURL} not found!`,
+        };
+      },
+    },
+  ],
   generationSystemPrompt: './system-instructions.md',
-  executablePrompts: ['../../prompts/**/*.md'],
+  executablePrompts: [
+    new EvalPromptWithMetadata(
+      'test-app',
+      `Create the Angular documentation website. Make sure you add a link to \`angular.dev\` in there.`,
+      {
+        metadata: {
+          goldenURL: 'angular.dev',
+        },
+      },
+    ),
+  ],
   executor: new FakeRemoteExecutor(),
 };

examples/environments/remote_env/fake-executor.ts

@@ -26,7 +26,7 @@ export class FakeRemoteExecutor implements Executor {
   async performFakeLlmRequest(): Promise<LlmResponse> {
     return {
       success: true,
-      outputFiles: [{code: 'Works!', filePath: 'main.ts'}],
+      outputFiles: [{code: 'angular.dev Works', filePath: 'main.ts'}],
       reasoning: '',
       errors: [],
       usage: {inputTokens: 0, totalTokens: 0, outputTokens: 0},

runner/configuration/environment-config.ts

@@ -2,7 +2,7 @@ import z from 'zod';
 import {createMessageBuilder, fromError} from 'zod-validation-error/v3';
 import {UserFacingError} from '../utils/errors.js';
 import {ratingSchema} from '../ratings/rating-types.js';
-import {MultiStepPrompt} from './multi-step-prompt.js';
+import {EvalPrompt, EvalPromptWithMetadata, MultiStepPrompt} from './prompts.js';
 import {executorSchema} from '../orchestration/executors/executor.js';
 import {
   LocalExecutorConfig,
@@ -46,6 +46,8 @@ export const environmentConfigSchema = z.object({
         ratings: z.array(ratingSchema).optional(),
       }),
       z.custom<MultiStepPrompt>(data => data instanceof MultiStepPrompt),
+      z.custom<EvalPrompt>(data => data instanceof EvalPrompt),
+      z.custom<EvalPromptWithMetadata<unknown>>(data => data instanceof EvalPromptWithMetadata),
     ]),
   ),
   /**

runner/configuration/environment.ts

@@ -13,7 +13,7 @@ import {UserFacingError} from '../utils/errors.js';
 import {generateId} from '../utils/id-generation.js';
 import {lazy} from '../utils/lazy-creation.js';
 import {EnvironmentConfig} from './environment-config.js';
-import {MultiStepPrompt} from './multi-step-prompt.js';
+import {EvalPromptWithMetadata, MultiStepPrompt} from './prompts.js';
 import {renderPromptTemplate} from './prompt-templating.js';
 
 /** Represents a single prompt evaluation environment. */
@@ -176,6 +176,18 @@ export class Environment {
     for (const def of prompts) {
       if (def instanceof MultiStepPrompt) {
         result.push(this.getMultiStepPrompt(def, envRatings));
+      } else if (def instanceof EvalPromptWithMetadata) {
+        result.push(
+          Promise.resolve({
+            name: def.name,
+            kind: 'single',
+            prompt: def.text,
+            ratings: [...envRatings, ...(def.opts.extraRatings ?? [])],
+            systemPromptType: 'generation',
+            contextFilePatterns: def.opts.contextFilePatterns ?? [],
+            metadata: def.opts.metadata,
+          } satisfies PromptDefinition),
+        );
       } else {
         let path: string;
         let ratings: Rating[];
@@ -198,6 +210,7 @@ export class Environment {
                 relativePath,
                 ratings,
                 /* isEditing */ false,
+                undefined,
               ),
           ),
         );
@@ -216,12 +229,13 @@ export class Environment {
    * @param ratings Ratings to run against the definition.
    * @param isEditing Whether this is an editing or generation step.
    */
-  private async getStepPromptDefinition(
+  private async getStepPromptDefinition<Metadata>(
     name: string,
     relativePath: string,
     ratings: Rating[],
     isEditing: boolean,
-  ): Promise<PromptDefinition> {
+    metadata: Metadata,
+  ): Promise<PromptDefinition<Metadata>> {
     const {result, contextFiles} = await this.renderEnvironmentPrompt(relativePath);
 
     return {
@@ -231,7 +245,8 @@ export class Environment {
       ratings,
       systemPromptType: isEditing ? 'editing' : 'generation',
       contextFilePatterns: contextFiles,
-    } satisfies PromptDefinition;
+      metadata,
+    } satisfies PromptDefinition<Metadata>;
   }
 
   /**
@@ -284,6 +299,7 @@ export class Environment {
         ratings.unshift(...def.stepRatings[current.name]);
       }
 
+      const stepMetadata = def.stepMetadata[current.name];
       const stepNum = parseInt(match[1]);
       if (stepNum === 0) {
         throw new UserFacingError('Multi-step prompts start with `step-1`.');
@@ -293,6 +309,7 @@ export class Environment {
         join(def.directoryPath, current.name),
         ratings,
         /*isEditing */ stepNum !== 1,
+        stepMetadata,
       );
 
       stepValues[step.name] = stepNum;

runner/configuration/multi-step-prompt.ts

@@ -0,0 +1,36 @@
+import {Rating} from '../ratings/rating-types.js';
+
+export interface EvalPromptOptions<M> {
+  metadata: M;
+  contextFilePatterns?: string[];
+  extraRatings?: Rating[];
+}
+
+/** Definition of a single-step prompt with metadata. */
+export class EvalPromptWithMetadata<Metadata> {
+  constructor(
+    readonly name: string,
+    readonly text: string,
+    readonly opts: EvalPromptOptions<Metadata>,
+  ) {}
+}
+
+/** Definition of a single-step prompt. */
+export class EvalPrompt extends EvalPromptWithMetadata<undefined> {
+  constructor(
+    name: string,
+    text: string,
+    opts: Omit<EvalPromptOptions<undefined>, 'metadata'> = {},
+  ) {
+    super(name, text, {...opts, metadata: undefined});
+  }
+}
+
+/** Definition of a multi-step prompt. */
+export class MultiStepPrompt {
+  constructor(
+    readonly directoryPath: string,
+    readonly stepRatings: Record<string, Rating[]> = {},
+    readonly stepMetadata: Record<string, unknown> = {},
+  ) {}
+}

runner/configuration/prompts.ts

@@ -9,7 +9,7 @@ export * from './ratings/built-in.js';
 export * from './ratings/rating-types.js';
 export * from './ratings/built-in-ratings/index.js';
 export {calculateBuildAndCheckStats, isPositiveScore} from './ratings/stats.js';
-export {MultiStepPrompt} from './configuration/multi-step-prompt.js';
+export {MultiStepPrompt, EvalPrompt, EvalPromptWithMetadata} from './configuration/prompts.js';
 export {
   BuildErrorType,
   BuildResultStatus,

runner/index.ts

@@ -8,8 +8,8 @@ export const sufficientGeneratedFilesRating: PerBuildRating = {
   id: 'common-generated-file-count',
   scoreReduction: '100%',
   kind: RatingKind.PER_BUILD,
-  rate: ({generatedFileCount}) => ({
+  rate: ({generatedFiles}) => ({
     state: RatingState.EXECUTED,
-    coefficient: generatedFileCount > 0 ? 1 : 0,
+    coefficient: generatedFiles.length > 0 ? 1 : 0,
   }),
 };

runner/ratings/built-in-ratings/sufficient-generated-files-rating.ts

@@ -92,19 +92,20 @@ export async function rateGeneratedCode(
     try {
       if (current.kind === RatingKind.PER_BUILD) {
         result = runPerBuildRating(
+          currentPromptDef,
           current,
           buildResult,
           serveTestingResult,
           repairAttempts,
           testResult,
           testRepairAttempts,
-          outputFiles.length,
+          outputFiles,
           axeRepairAttempts,
           ratingsResult,
         );
       } else if (current.kind === RatingKind.PER_FILE) {
         categorizedFiles ??= splitFilesIntoCategories(outputFiles);
-        result = await runPerFileRating(current, categorizedFiles, ratingsResult);
+        result = await runPerFileRating(currentPromptDef, current, categorizedFiles, ratingsResult);
       } else if (current.kind === RatingKind.LLM_BASED) {
         result = await runLlmBasedRating(
           environment,
@@ -174,25 +175,27 @@ export async function rateGeneratedCode(
 }
 
 function runPerBuildRating(
+  prompt: PromptDefinition,
   rating: PerBuildRating,
   buildResult: BuildResult,
   serveResult: ServeTestingResult | null,
   repairAttempts: number,
   testResult: TestExecutionResult | null,
   testRepairAttempts: number,
-  generatedFileCount: number,
+  generatedFiles: LlmResponseFile[],
   axeRepairAttempts: number,
   ratingsResult: RatingsResult,
 ): IndividualAssessment | SkippedIndividualAssessment {
   const rateResult = rating.rate({
     buildResult,
     serveResult,
     repairAttempts,
-    generatedFileCount,
+    generatedFiles,
     axeRepairAttempts,
-    ratingsResult,
     testResult,
     testRepairAttempts,
+    ratingsResult,
+    prompt,
   });
 
   // If the rating was skipped (e.g., Axe test wasn't run), create a skipped assessment.
@@ -208,6 +211,7 @@ function runPerBuildRating(
 }
 
 async function runPerFileRating(
+  prompt: PromptDefinition,
   rating: PerFileRating,
   categorizedFiles: CategorizedFiles,
   ratingsResult: RatingsResult,
@@ -240,7 +244,7 @@ async function runPerFileRating(
       // Remove comments from the code to avoid false-detection of bad patterns.
       // Some keywords like `NgModule` can be used in code comments.
       const code = removeComments(file.code, contentType);
-      const result = await rating.rate(code, file.filePath, ratingsResult);
+      const result = await rating.rate(code, file.filePath, {prompt, ratingsResult});
       let coeff: number;
 
       if (typeof result === 'number') {

runner/ratings/rate-code.ts

@@ -46,6 +46,11 @@ export const CATEGORY_NAMES = {
   [RatingCategory.LOW_IMPACT]: 'Low Impact',
 };
 
+const ratingCommonContextFields = {
+  ratingsResult: z.record(z.custom<IndividualAssessment | SkippedIndividualAssessment>()),
+  prompt: z.custom<PromptDefinition>(),
+};
+
 const ratingSchemaCommonFields = {
   category: z.custom<RatingCategory>(),
   scoreReduction: z.custom<`${number}%`>(),
@@ -63,13 +68,13 @@ const perBuildRatingSchema = z
       .args(
         z.strictObject({
           buildResult: z.custom<BuildResult>(),
+          generatedFiles: z.custom<LlmResponseFile[]>(),
           serveResult: z.custom<ServeTestingResult | null>(),
           repairAttempts: z.number(),
           testResult: z.custom<TestExecutionResult | null>(),
           testRepairAttempts: z.number(),
           axeRepairAttempts: z.number(),
-          generatedFileCount: z.number(),
-          ratingsResult: z.record(z.custom<IndividualAssessment | SkippedIndividualAssessment>()),
+          ...ratingCommonContextFields,
         }),
       )
       .returns(z.custom<PerBuildRatingResult>()),
@@ -83,9 +88,9 @@ const perFileRatingSchema = z
     rate: z
       .function()
       .args(
-        z.string(),
-        z.string().optional(),
-        z.record(z.custom<IndividualAssessment | SkippedIndividualAssessment>()),
+        z.string().describe('Code'),
+        z.string().optional().describe('File path'),
+        z.object(ratingCommonContextFields).describe('Context'),
       )
       .returns(z.custom<PerFileRatingResult>()),
     filter: z.union([