fix(js): Fix multiple datasets in same test file for Vitest, improve reporter output

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [

js/src/index.ts

@@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.3.3";
+export const __version__ = "0.3.4";

js/src/jest/reporter.ts

@@ -5,6 +5,9 @@ import { printReporterTable } from "../utils/jestlike/reporter.js";
 
 class LangSmithEvalReporter extends DefaultReporter {
   async onTestResult(test: any, testResult: any, aggregatedResults: any) {
+    if (testResult.failureMessage) {
+      console.log(testResult.failureMessage);
+    }
     const groupedTestResults = testResult.testResults.reduce(
       (groups: Record<string, any>, testResult: any) => {
         const ancestorTitle = testResult.ancestorTitles.join(" > ");
@@ -19,7 +22,16 @@ class LangSmithEvalReporter extends DefaultReporter {
     try {
       for (const testGroupName of Object.keys(groupedTestResults)) {
         const resultGroup = groupedTestResults[testGroupName];
-        await printReporterTable(resultGroup, testResult.failureMessage);
+        const unskippedTests = resultGroup.filter(
+          (result: any) => result.status !== "pending"
+        );
+        const overallResult =
+          unskippedTests.length === 0
+            ? "skip"
+            : unskippedTests.every((result: any) => result.status === "passed")
+            ? "pass"
+            : "fail";
+        await printReporterTable(testGroupName, resultGroup, overallResult);
       }
     } catch (e: any) {
       console.log("Failed to display LangSmith eval results:", e.message);

js/src/tests/jestlike/vitest_separate_file.vitesteval.ts

@@ -49,3 +49,43 @@ ls.describe(
     },
   }
 );
+
+ls.describe(
+  "js vitest 3",
+  () => {
+    ls.test.each(
+      [
+        {
+          inputs: {
+            one: "uno",
+          },
+          referenceOutputs: {
+            ein: "un",
+          },
+        },
+        {
+          inputs: {
+            two: "dos",
+          },
+          referenceOutputs: {
+            zwei: "deux",
+          },
+        },
+      ],
+      { iterations: 3, metadata: { something: "cool" } }
+    )("Does the thing", async ({ inputs, referenceOutputs }) => {
+      const myApp = () => {
+        return { bar: "bad" };
+      };
+      const res = myApp();
+      const evaluator = ls.wrapEvaluator(myEvaluator);
+      await evaluator({ inputs, referenceOutputs, outputs: res });
+      return res;
+    });
+  },
+  {
+    metadata: {
+      model: "test-model",
+    },
+  }
+);

js/src/utils/jestlike/reporter.ts

@@ -30,11 +30,11 @@ function formatTestName(name: string, duration: number) {
 
 function getFormattedStatus(status: string) {
   const s = status.toLowerCase();
-  if (s === "pending") {
+  if (s === "pending" || s === "skipped") {
     return chalk.yellow("○ Skipped");
-  } else if (s === "passed") {
+  } else if (s.includes("pass")) {
     return chalk.green("✓ Passed");
-  } else if (s === "failed") {
+  } else if (s.includes("fail")) {
     return chalk.red("✕ Failed");
   } else {
     return status;
@@ -43,11 +43,11 @@ function getFormattedStatus(status: string) {
 
 function getColorParam(status: string) {
   const s = status.toLowerCase();
-  if (s === "pending") {
+  if (s === "pending" || s === "skipped") {
     return { color: "yellow" };
-  } else if (s === "passed") {
+  } else if (s.includes("pass")) {
     return { color: "grey" };
-  } else if (s === "failed") {
+  } else if (s.includes("fail")) {
     return { color: "red" };
   } else {
     return {};
@@ -75,7 +75,13 @@ function formatValue(value: unknown) {
 }
 
 export async function printReporterTable(
-  results: { title: string; duration: number; status: string }[],
+  testSuiteName: string,
+  results: {
+    title: string;
+    duration: number;
+    status: "pass" | "passed" | "fail" | "failed" | "pending" | "skipped";
+  }[],
+  testStatus: "pass" | "skip" | "fail",
   failureMessage?: string
 ) {
   const rows = [];
@@ -101,7 +107,7 @@ export async function printReporterTable(
         },
         getColorParam(status),
       ]);
-    } else if (status === "pending") {
+    } else if (status === "pending" || status === "skipped") {
       // Skipped
       rows.push([
         {
@@ -265,6 +271,14 @@ export async function printReporterTable(
   for (const row of rows) {
     table.addRow(row[0], row[1]);
   }
+  const testStatusColor = testStatus.includes("pass")
+    ? chalk.green
+    : testStatus.includes("fail")
+    ? chalk.red
+    : chalk.yellow;
+  if (testSuiteName) {
+    console.log(testStatusColor(`› ${testSuiteName}`));
+  }
   if (failureMessage) {
     console.log(failureMessage);
   }

js/src/vitest/reporter.ts

@@ -10,15 +10,22 @@ class LangSmithEvalReporter extends DefaultReporter {
   async onFinished(files: RunnerTestFile[], errors: unknown[]) {
     super.onFinished(files, errors);
     for (const file of files) {
-      const testModule = this.ctx.state.getReportedEntity(file) as TestModule;
-      const tests = [...testModule.children.allTests()].map((test) => {
-        return {
-          title: test.name,
-          status: test.result()?.state ?? "skipped",
-          duration: Math.round(test.diagnostic()?.duration ?? 0),
-        };
-      });
-      await printReporterTable(tests);
+      for (const task of file.tasks) {
+        const testModule = this.ctx.state.getReportedEntity(task) as TestModule;
+        const tests = [...testModule.children.allTests()].map((test) => {
+          return {
+            title: test.name,
+            status: test.result()?.state ?? "skipped",
+            duration: Math.round(test.diagnostic()?.duration ?? 0),
+          };
+        });
+        const result = ["pass", "fail", "skip"].includes(
+          task.result?.state ?? ""
+        )
+          ? (task.result?.state as "pass" | "fail" | "skip")
+          : "skip";
+        await printReporterTable(task.name, tests, result);
+      }
     }
   }
 }