Allow filtering samples by compound expressions including multiple scorers

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Inspect View: display tool error messages in transcript when tool errors occur.
 - Inspect View: display any completed samples even if the task fails because of an error
 - Inspect View: don't display the 'input' column heading if there isn't an input
+- Inspect View: allow filtering samples by compound expressions including multiple scorers.
 - Open AI: Handle additional bad request status codes (mapping them to appropriate `StopReason`)
 - Open AI: Use new `max_completion_tokens` option for o1 full.
 - Web Browser: raise error when both `error` and `web_at` fields are present in response.

src/inspect_ai/_view/www/App.css

@@ -9,6 +9,12 @@
   --inspect-input-border: var(--bs-light-border-subtle);
   --inspect-diff-add-color: #dafbe1;
   --inspect-diff-remove-color: #ffebe9;
+  --inspect-inactive-selection-background: var(--vscode-editor-inactiveSelectionBackground, #d9d9d9);
+  --inspect-active-selection-background: var(--vscode-editor-selectionBackground, #d7d4f0);
+  --inspect-focus-border-color: #86b7fe;
+  --inspect-focus-border-shadow: 0 0 0 0.25rem rgba(var(--bs-primary-rgb), 0.25);
+  --inspect-focus-border-gray-color: #808080;
+  --inspect-focus-border-gray-shadow: 0 0 0 0.25rem rgba(48, 48, 48, 0.25);
 }
 
 body:not([class^="vscode-"]) button {
@@ -650,6 +656,30 @@ table.table.table-sm td {
   height: auto !important;
 }
 
+[data-tooltip] {
+  position: relative;
+}
+[data-tooltip]:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  line-height: 1.25;
+  background: var(--bs-light);
+  color: var(--bs-body-color);
+  opacity: 1;
+  padding: 4px 8px;
+  border-radius: 4px;
+  border: 1px solid var(--bs-border-color);
+  box-shadow: 0 2px 10px 0 rgba(0, 0, 0, 0.25);
+  white-space: pre-wrap;
+  width: max-content;
+  max-width: 400px;
+  z-index: 1000;
+}
+[data-tooltip][data-tooltip-position="bottom-left"]:hover::after {
+  right: 0%;
+  top: 100%;
+}
+
 /* ANSI Coloring */
 .ansi-display {
   font-family: monospace;
@@ -725,7 +755,7 @@ pre[class*="language-"].tool-output {
   background: none !important;
   border: none !important;
   box-shadow: none !important;
-  border-radius: var(--bs-border-radius) !important;  
+  border-radius: var(--bs-border-radius) !important;
 }
 
 /* lightbox styles */

src/inspect_ai/_view/www/dist/assets/index.css

@@ -14282,6 +14282,12 @@ pre[class*="language-"] {
   --inspect-input-border: var(--bs-light-border-subtle);
   --inspect-diff-add-color: #dafbe1;
   --inspect-diff-remove-color: #ffebe9;
+  --inspect-inactive-selection-background: var(--vscode-editor-inactiveSelectionBackground, #d9d9d9);
+  --inspect-active-selection-background: var(--vscode-editor-selectionBackground, #d7d4f0);
+  --inspect-focus-border-color: #86b7fe;
+  --inspect-focus-border-shadow: 0 0 0 0.25rem rgba(var(--bs-primary-rgb), 0.25);
+  --inspect-focus-border-gray-color: #808080;
+  --inspect-focus-border-gray-shadow: 0 0 0 0.25rem rgba(48, 48, 48, 0.25);
 }
 
 body:not([class^="vscode-"]) button {
@@ -14923,6 +14929,30 @@ table.table.table-sm td {
   height: auto !important;
 }
 
+[data-tooltip] {
+  position: relative;
+}
+[data-tooltip]:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  line-height: 1.25;
+  background: var(--bs-light);
+  color: var(--bs-body-color);
+  opacity: 1;
+  padding: 4px 8px;
+  border-radius: 4px;
+  border: 1px solid var(--bs-border-color);
+  box-shadow: 0 2px 10px 0 rgba(0, 0, 0, 0.25);
+  white-space: pre-wrap;
+  width: max-content;
+  max-width: 400px;
+  z-index: 1000;
+}
+[data-tooltip][data-tooltip-position="bottom-left"]:hover::after {
+  right: 0%;
+  top: 100%;
+}
+
 /* ANSI Coloring */
 .ansi-display {
   font-family: monospace;
@@ -14998,7 +15028,7 @@ pre[class*="language-"].tool-output {
   background: none !important;
   border: none !important;
   box-shadow: none !important;
-  border-radius: var(--bs-border-radius) !important;  
+  border-radius: var(--bs-border-radius) !important;
 }
 
 /* lightbox styles */

src/inspect_ai/_view/www/package.json

@@ -30,8 +30,10 @@
     "bootstrap": "^5.3.3",
     "bootstrap-icons": "^1.11.3",
     "clipboard": "^2.0.11",
+    "codemirror": "^6.0.1",
     "fast-json-patch": "^3.1.1",
     "fflate": "^0.8.2",
+    "filtrex": "^3.1.0",
     "htm": "^3.1.1",
     "json": "^11.0.0",
     "json5": "^2.2.3",

src/inspect_ai/_view/www/src/App.mjs

@@ -38,7 +38,7 @@ import {
 } from "./samples/SamplesDescriptor.mjs";
 import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs";
 import { resolveAttachments } from "./utils/attachments.mjs";
-import { filterFnForType } from "./samples/tools/filters.mjs";
+import { filterSamples } from "./samples/tools/filters.mjs";
 
 import {
   kEvalWorkspaceTabId,
@@ -308,21 +308,19 @@ export function App({
 
   useEffect(() => {
     const samples = selectedLog?.contents?.sampleSummaries || [];
-    const filtered = samples.filter((sample) => {
+    const { result: prefiltered } = filterSamples(
+      evalDescriptor,
+      samples,
+      filter?.value,
+    );
+    const filtered = prefiltered.filter((sample) => {
       // Filter by epoch if specified
       if (epoch && epoch !== "all") {
         if (epoch !== sample.epoch + "") {
           return false;
         }
       }
-
-      // Apply the filter
-      const filterFn = filterFnForType(filter);
-      if (filterFn && filter.value) {
-        return filterFn(samplesDescriptor, sample, filter.value);
-      } else {
-        return true;
-      }
+      return true;
     });
 
     // Sort the samples

src/inspect_ai/_view/www/src/Types.mjs

@@ -25,7 +25,6 @@
 /**
  * @typedef {Object} ScoreFilter
  * @property {string} [value]
- * @property {string} [type]
  */
 
 /**

src/inspect_ai/_view/www/src/navbar/Navbar.mjs

@@ -19,6 +19,7 @@ import { SecondaryBar } from "./SecondaryBar.mjs";
  * @param {import("../types/log").EvalPlan} [props.evalPlan] - The EvalSpec
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats
  * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor
  * @param {string} [props.status] - the status
  * @param {boolean} props.offcanvas - Are we in offcanvas mode?
  * @param {boolean} props.showToggle - Should we show the toggle?
@@ -32,6 +33,7 @@ export const Navbar = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   showToggle,
   offcanvas,
   status,
@@ -182,6 +184,7 @@ export const Navbar = ({
           evalResults=${evalResults}
           evalStats=${evalStats}
           samples=${samples}
+          evalDescriptor=${evalDescriptor}
           status=${status}
           style=${{ gridColumn: "1/-1" }}
         />

src/inspect_ai/_view/www/src/navbar/SecondaryBar.mjs

@@ -3,6 +3,7 @@ import { html } from "htm/preact";
 import { LabeledValue } from "../components/LabeledValue.mjs";
 import { formatDataset, formatDuration } from "../utils/Format.mjs";
 import { ExpandablePanel } from "../components/ExpandablePanel.mjs";
+import { scoreFilterItems } from "../samples/tools/filters.mjs";
 
 /**
  * Renders the Navbar
@@ -13,6 +14,7 @@ import { ExpandablePanel } from "../components/ExpandablePanel.mjs";
  * @param {import("../types/log").EvalResults} [props.evalResults] - The EvalResults
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats
  * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor
  * @param {string} [props.status] - the status
  * @param {Map<string, string>} [props.style] - is this off canvas
  *
@@ -24,6 +26,7 @@ export const SecondaryBar = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   status,
   style,
 }) => {
@@ -60,8 +63,8 @@ export const SecondaryBar = ({
   values.push({
     size: "minmax(12%, auto)",
     value: html`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: hasConfig ? "left" : "center" }}>
-    <${ScorerSummary} 
-      scorers=${evalResults?.scores} />
+    <${ScorerSummary}
+      evalDescriptor=${evalDescriptor} />
   </${LabeledValue}>`,
   });
 
@@ -124,17 +127,23 @@ const DatasetSummary = ({ dataset, samples, epochs, style }) => {
   `;
 };
 
-const ScorerSummary = ({ scorers }) => {
-  if (!scorers) {
+const ScorerSummary = ({ evalDescriptor }) => {
+  if (!evalDescriptor) {
     return "";
   }
 
-  const uniqScorers = new Set();
-  scorers.forEach((scorer) => {
-    uniqScorers.add(scorer.name);
-  });
+  const items = scoreFilterItems(evalDescriptor);
 
-  return Array.from(uniqScorers).join(", ");
+  return html`
+    <span style=${{ position: "relative" }}>
+      ${Array.from(items).map(
+        (item, index) => html`
+          ${index > 0 ? ", " : ""}
+          <span title=${item.tooltip}>${item.canonicalName}</span>
+        `,
+      )}
+    </span>
+  `;
 };
 
 /**

src/inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs

@@ -83,6 +83,29 @@ import {
  * @property {number} normalized.limit - Normalized size of the limit message.
  */
 
+/**
+ * @param {import("../Types.mjs").ScoreLabel | undefined} scoreLabel
+ * @returns {string}
+ */
+export const scoreLabelKey = (scoreLabel) => {
+  if (!scoreLabel) {
+    return "No score key";
+  }
+  return `${scoreLabel.scorer}.${scoreLabel.name}`;
+};
+
+/**
+ * @param {string} key
+ * @returns {import("../Types.mjs").ScoreLabel | undefined}
+ */
+export const parseScoreLabelKey = (key) => {
+  if (key == "No score key") {
+    return undefined;
+  }
+  const [scorer, name] = key.split(".");
+  return { scorer, name };
+};
+
 /**
  * @param {import("../Types.mjs").ScoreLabel[]} scores - the list of available scores
  * @param {import("../api/Types.mjs").SampleSummary[]} samples - the list of sample summaries
@@ -165,17 +188,6 @@ export const createEvalDescriptor = (scores, samples, epochs) => {
     return undefined;
   };
 
-  /**
-   * @param {import("../Types.mjs").ScoreLabel} [scoreLabel]
-   * @returns {string}
-   */
-  const scoreLabelKey = (scoreLabel) => {
-    if (!scoreLabel) {
-      return "No score key";
-    }
-    return `${scoreLabel.scorer}.${scoreLabel.name}`;
-  };
-
   /**
    * The EvalDescriptor is memoized. Compute all descriptors now to avoid duplicate work.
    * @type {Map<string, ScoreDescriptor>}
@@ -462,7 +474,7 @@ const scoreCategorizers = [
      * @returns {ScoreDescriptor} a ScoreDescriptor
      */
     describe: (values, types) => {
-      if (values.length === 2 && types.length === 1 && types[0] === "boolean") {
+      if (types.length === 1 && types[0] === "boolean") {
         return booleanScoreCategorizer();
       }
     },

src/inspect_ai/_view/www/src/samples/SamplesTools.mjs

@@ -23,6 +23,14 @@ export const SampleTools = (props) => {
   const hasEpochs = epochs > 1;
   const tools = [];
 
+  tools.push(
+    html`<${SampleFilter}
+      evalDescriptor=${sampleDescriptor.evalDescriptor}
+      filter=${filter}
+      filterChanged=${filterChanged}
+    />`,
+  );
+
   if (scores.length > 1) {
     tools.push(
       html`<${SelectScorer}
@@ -43,14 +51,6 @@ export const SampleTools = (props) => {
     );
   }
 
-  tools.push(
-    html`<${SampleFilter}
-      filter=${filter}
-      filterChanged=${filterChanged}
-      descriptor=${sampleDescriptor}
-    />`,
-  );
-
   tools.push(
     html`<${SortFilter}
       sampleDescriptor=${sampleDescriptor}