From c1631fc04aab61100ce89b765cdc8cbfe673185f Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Wed, 29 Jan 2025 21:22:20 -0800 Subject: [PATCH 1/2] Fix multiple datasets in same test file for Vitest, improve reporter output --- js/package.json | 2 +- js/src/index.ts | 2 +- js/src/jest/reporter.ts | 11 ++++- .../vitest_separate_file.vitesteval.ts | 40 +++++++++++++++++++ js/src/utils/jestlike/reporter.ts | 30 ++++++++++---- js/src/vitest/reporter.ts | 25 +++++++----- 6 files changed, 90 insertions(+), 20 deletions(-) diff --git a/js/package.json b/js/package.json index 7c9300862..dae2fd525 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.3.3", + "version": "0.3.4", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ diff --git a/js/src/index.ts b/js/src/index.ts index 1d05820cb..7e92141da 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js"; export { overrideFetchImplementation } from "./singletons/fetch.js"; // Update using yarn bump-version -export const __version__ = "0.3.3"; +export const __version__ = "0.3.4"; diff --git a/js/src/jest/reporter.ts b/js/src/jest/reporter.ts index ad9d57243..32dc57872 100644 --- a/js/src/jest/reporter.ts +++ b/js/src/jest/reporter.ts @@ -19,7 +19,16 @@ class LangSmithEvalReporter extends DefaultReporter { try { for (const testGroupName of Object.keys(groupedTestResults)) { const resultGroup = groupedTestResults[testGroupName]; - await printReporterTable(resultGroup, testResult.failureMessage); + const unskippedTests = resultGroup.filter( + (result: any) => result.status !== "pending" + ); + const overallResult = + unskippedTests.length === 0 + ? "skip" + : unskippedTests.every((result: any) => result.status === "passed") + ? "pass" + : "fail"; + await printReporterTable(testGroupName, resultGroup, overallResult); } } catch (e: any) { console.log("Failed to display LangSmith eval results:", e.message); diff --git a/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts b/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts index 0bd3cfef6..15c3c7c27 100644 --- a/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts +++ b/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts @@ -49,3 +49,43 @@ ls.describe( }, } ); + +ls.describe( + "js vitest 3", + () => { + ls.test.each( + [ + { + inputs: { + one: "uno", + }, + referenceOutputs: { + ein: "un", + }, + }, + { + inputs: { + two: "dos", + }, + referenceOutputs: { + zwei: "deux", + }, + }, + ], + { iterations: 3, metadata: { something: "cool" } } + )("Does the thing", async ({ inputs, referenceOutputs }) => { + const myApp = () => { + return { bar: "bad" }; + }; + const res = myApp(); + const evaluator = ls.wrapEvaluator(myEvaluator); + await evaluator({ inputs, referenceOutputs, outputs: res }); + return res; + }); + }, + { + metadata: { + model: "test-model", + }, + } +); diff --git a/js/src/utils/jestlike/reporter.ts b/js/src/utils/jestlike/reporter.ts index 649977950..2ef6c83b2 100644 --- a/js/src/utils/jestlike/reporter.ts +++ b/js/src/utils/jestlike/reporter.ts @@ -30,11 +30,11 @@ function formatTestName(name: string, duration: number) { function getFormattedStatus(status: string) { const s = status.toLowerCase(); - if (s === "pending") { + if (s === "pending" || s === "skipped") { return chalk.yellow("○ Skipped"); - } else if (s === "passed") { + } else if (s.includes("pass")) { return chalk.green("✓ Passed"); - } else if (s === "failed") { + } else if (s.includes("fail")) { return chalk.red("✕ Failed"); } else { return status; @@ -43,11 +43,11 @@ function getFormattedStatus(status: string) { function getColorParam(status: string) { const s = status.toLowerCase(); - if (s === "pending") { + if (s === "pending" || s === "skipped") { return { color: "yellow" }; - } else if (s === "passed") { + } else if (s.includes("pass")) { return { color: "grey" }; - } else if (s === "failed") { + } else if (s.includes("fail")) { return { color: "red" }; } else { return {}; @@ -75,7 +75,13 @@ function formatValue(value: unknown) { } export async function printReporterTable( - results: { title: string; duration: number; status: string }[], + testSuiteName: string, + results: { + title: string; + duration: number; + status: "pass" | "passed" | "fail" | "failed" | "pending" | "skipped"; + }[], + testStatus: "pass" | "skip" | "fail", failureMessage?: string ) { const rows = []; @@ -101,7 +107,7 @@ export async function printReporterTable( }, getColorParam(status), ]); - } else if (status === "pending") { + } else if (status === "pending" || status === "skipped") { // Skipped rows.push([ { @@ -265,6 +271,14 @@ export async function printReporterTable( for (const row of rows) { table.addRow(row[0], row[1]); } + const testStatusColor = testStatus.includes("pass") + ? chalk.green + : testStatus.includes("fail") + ? chalk.red + : chalk.yellow; + if (testSuiteName) { + console.log(testStatusColor(`› ${testSuiteName}`)); + } if (failureMessage) { console.log(failureMessage); } diff --git a/js/src/vitest/reporter.ts b/js/src/vitest/reporter.ts index 7f43187d5..796e717e4 100644 --- a/js/src/vitest/reporter.ts +++ b/js/src/vitest/reporter.ts @@ -10,15 +10,22 @@ class LangSmithEvalReporter extends DefaultReporter { async onFinished(files: RunnerTestFile[], errors: unknown[]) { super.onFinished(files, errors); for (const file of files) { - const testModule = this.ctx.state.getReportedEntity(file) as TestModule; - const tests = [...testModule.children.allTests()].map((test) => { - return { - title: test.name, - status: test.result()?.state ?? "skipped", - duration: Math.round(test.diagnostic()?.duration ?? 0), - }; - }); - await printReporterTable(tests); + for (const task of file.tasks) { + const testModule = this.ctx.state.getReportedEntity(task) as TestModule; + const tests = [...testModule.children.allTests()].map((test) => { + return { + title: test.name, + status: test.result()?.state ?? "skipped", + duration: Math.round(test.diagnostic()?.duration ?? 0), + }; + }); + const result = ["pass", "fail", "skip"].includes( + task.result?.state ?? "" + ) + ? (task.result?.state as "pass" | "fail" | "skip") + : "skip"; + await printReporterTable(task.name, tests, result); + } } } } From 9aff444e415265a8863da48db0ff88a28dac02c6 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Wed, 29 Jan 2025 21:25:17 -0800 Subject: [PATCH 2/2] Log failure message in Jest --- js/src/jest/reporter.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/js/src/jest/reporter.ts b/js/src/jest/reporter.ts index 32dc57872..31bc62973 100644 --- a/js/src/jest/reporter.ts +++ b/js/src/jest/reporter.ts @@ -5,6 +5,9 @@ import { printReporterTable } from "../utils/jestlike/reporter.js"; class LangSmithEvalReporter extends DefaultReporter { async onTestResult(test: any, testResult: any, aggregatedResults: any) { + if (testResult.failureMessage) { + console.log(testResult.failureMessage); + } const groupedTestResults = testResult.testResults.reduce( (groups: Record, testResult: any) => { const ancestorTitle = testResult.ancestorTitles.join(" > ");