Merge branch 'main' into feature/vscode-0333

docs/agents-api.qmd

@@ -216,14 +216,6 @@ Note that we don't `await` the subtasks when building up our list of `searches`.
 
 ### Forking {#sec-forking}
 
-::: {.callout-note appearance="simple"}
-Note that the `fork()` feature described below is available only in the development version of Inspect. You can install the development version with:
-
-``` bash
-pip install git+https://github.com/ukgovernmentbeis/inspect_ai
-```
-:::
-
 Inspect's `fork()` function provids a convenient wrapper around a very common use of subtasks: running a `TaskState` against a set of solvers in parallel to explore different trajectories. 
 
 For example, let's say you have a solver named `explore()` that takes `temperature` as a parameter. You might want to try the solver out with multiple temperature values and then continue on with the best result:

docs/log-viewer.qmd

@@ -123,7 +123,21 @@ You can see all of these log entries in the **Logging** tab:
 
 ![](images/inspect-view-logging.png){.border .lightbox fig-alt="The Logging panel of the Inspect log viewer, displaying several info log messages from the web search tool indicating what queries were executed by the tool."}
 
-It is important to note that the Inspect View will show all log entries level `info` or higher. However, printing every `info` message to the console during an eval might be too distracting, so the default log level for printing is `warning`. If you change it to `info` then you'll also see these log messages in the console:
+### Log Levels
+
+The log levels and their applicability are described below (in increasing order of severity):
+
+| Level | Description |
+|------------------------------------|------------------------------------|
+| `debug` | Detailed information, typically of interest only when diagnosing problems. |
+| `http` | HTTP diagnostics including requests and response statuses |
+| `sandbox` | Show commands sent to manage and execute code in sandboxes. |
+| `info` | Confirmation that things are working as expected. |
+| `warning` | or indicative of some problem in the near future (e.g. ‘disk space low’). The software is still working as expected. |
+| `error` | Due to a more serious problem, the software has not been able to perform some function |
+| `critical` | A serious error, indicating that the program itself may be unable to continue running. |
+
+: Inspect View will show all log entries level `info` or higher. However, printing every `info` message to the console during an eval might be too distracting, so the default log level for the console is `warning`. If you change it to `info` then you'll also see these log messages in the console:
 
 ``` bash
 $ inspect eval biology_qa.py --log-level info
@@ -181,7 +195,6 @@ By default, an existing output dir will NOT be overwritten. Specify the `--overw
 $ inspect view bundle --output-dir logs-www --overwrite
 ```
 
-
 Bundling the viewer and logs will produce an output directory with the following structure:
 
 ``` bash

src/inspect_ai/_display/rich.py

@@ -512,7 +512,7 @@ def task_config(profile: TaskProfile, generate_config: bool = True) -> str:
         config = config | dict(profile.generate_config.model_dump(exclude_none=True))
     config_print: list[str] = []
     for name, value in config.items():
-        if name not in ["limit", "epochs", "model"]:
+        if name not in ["limit", "model"]:
             config_print.append(f"{name}: {value}")
     values = ", ".join(config_print)
     if values:

src/inspect_ai/_util/constants.py

@@ -22,6 +22,7 @@
     "INFO",
     "WARNING",
     "ERROR",
+    "CRITICAL",
 ]
 DEFAULT_LOG_LEVEL = "warning"
 DEFAULT_LOG_BUFFER_LOCAL = 10

src/inspect_ai/_view/www/dist/assets/index.js

@@ -16327,9 +16327,22 @@ const JSONPanel = ({ data, style }) => {
   </div>`;
 };
 const InfoEventView = ({ id, event, style }) => {
+  const panels = [];
+  if (typeof event.data === "string") {
+    panels.push(
+      m$1`<${MarkdownDiv}
+        markdown=${event.data}
+        style=${{ margin: "0.5em 0" }}
+      />`
+    );
+  } else {
+    panels.push(
+      m$1`<${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }} />`
+    );
+  }
   return m$1`
   <${EventPanel} id=${id} title="Info" icon=${ApplicationIcons.info} style=${style}>
-    <${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }}/>
+    ${panels}
   </${EventPanel}>`;
 };
 const ScoreEventView = ({ id, event, style }) => {
@@ -17354,7 +17367,7 @@ const resolveValue = (value, evalEvents) => {
   }
   return value;
 };
-const SampleError = ({ align, style }) => {
+const SampleError = ({ message, align, style }) => {
   align = align || "center";
   return m$1`<div
     style=${{
@@ -17374,10 +17387,10 @@ const SampleError = ({ align, style }) => {
     height: FontSize.small
   }}
     />
-    <div>Error</div>
+    <div>${errorType(message)}</div>
   </div>`;
 };
-const FlatSampleError = ({ style }) => {
+const FlatSampleError = ({ message, style }) => {
   return m$1`<div
     style=${{
     color: "var(--bs-danger)",
@@ -17402,10 +17415,19 @@ const FlatSampleError = ({ style }) => {
     height: FontSize.base
   }}
     >
-      Error
+      ${errorType(message)}
     </div>
   </div>`;
 };
+const errorType = (message) => {
+  if (!message) {
+    return "Error";
+  }
+  if (message.includes("(")) {
+    return message.split("(")[0];
+  }
+  return "Error";
+};
 const printHtml = (html2, css) => {
   const printWindow = window.open("", "", "height=600,width=800");
   printWindow.document.write("<html><head><title>Print</title>");
@@ -17727,7 +17749,10 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
   }
   columns.push({
     label: "Score",
-    value: sample.error ? m$1`<${FlatSampleError} style=${{ marginTop: "0.4rem" }} />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render(),
+    value: sample.error ? m$1`<${FlatSampleError}
+          message=${sample.error.message}
+          style=${{ marginTop: "0.4rem" }}
+        />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render(),
     size: "minmax(2em, auto)",
     center: true
   });
@@ -18206,7 +18231,7 @@ const SampleRow = ({
     display: "flex"
   }}
       >
-        ${sample.error ? m$1`<${SampleError} />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render()}
+        ${sample.error ? m$1`<${SampleError} message=${sample.error.message} />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render()}
       </div>
     </div>
   `;
@@ -21499,7 +21524,6 @@ function App({ api: api2, pollForLogs = true }) {
     setHeadersLoading(false);
   }, [logs, setStatus, setLogHeaders, setHeadersLoading]);
   y(async () => {
-    console.log({ logs });
     const targetLog = logs.files[selected];
     if (targetLog && (!currentLog || currentLog.name !== targetLog.name)) {
       try {
@@ -21673,7 +21697,6 @@ function App({ api: api2, pollForLogs = true }) {
         const index = result.files.findIndex((val) => {
           return log_file.endsWith(val.name);
         });
-        console.log({ result, log_file, index });
         if (index > -1) {
           setSelected(index);
         }

src/inspect_ai/_view/www/src/App.mjs

@@ -94,7 +94,6 @@ export function App({ api, pollForLogs = true }) {
 
   // Load a specific log
   useEffect(async () => {
-    console.log({ logs });
     const targetLog = logs.files[selected];
     if (targetLog && (!currentLog || currentLog.name !== targetLog.name)) {
       try {
@@ -308,7 +307,6 @@ export function App({ api, pollForLogs = true }) {
         const index = result.files.findIndex((val) => {
           return log_file.endsWith(val.name);
         });
-        console.log({ result, log_file, index });
         if (index > -1) {
           setSelected(index);
         }

src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs

@@ -368,7 +368,10 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
   columns.push({
     label: "Score",
     value: sample.error
-      ? html`<${FlatSampleError} style=${{ marginTop: "0.4rem" }} />`
+      ? html`<${FlatSampleError}
+          message=${sample.error.message}
+          style=${{ marginTop: "0.4rem" }}
+        />`
       : sampleDescriptor?.selectedScore(sample).render(),
     size: "minmax(2em, auto)",
     center: true,

src/inspect_ai/_view/www/src/samples/SampleError.mjs

@@ -8,12 +8,14 @@ import { ApplicationIcons } from "../appearance/Icons.mjs";
  * Component to display a styled error message.
  *
  * @param {Object} props - The component properties.
+ * @param {string} [props.message] - The error message
  * @param {string} [props.align="center"] - The alignment for the error message. Defaults to "center".
  * @param {Object} [props.style] - Styles to add for this component
  * @returns {import("preact").JSX.Element} The error component.
  */
-export const SampleError = ({ align, style }) => {
+export const SampleError = ({ message, align, style }) => {
   align = align || "center";
+
   return html`<div
     style=${{
       color: "var(--bs-danger)",
@@ -32,18 +34,19 @@ export const SampleError = ({ align, style }) => {
         height: FontSize.small,
       }}
     />
-    <div>Error</div>
+    <div>${errorType(message)}</div>
   </div>`;
 };
 
 /**
  * Component to display a styled error message.
  *
  * @param {Object} props - The component properties.
+ * @param {string} [props.message] - The message to display
  * @param {Object} [props.style] - Styles to add for this component
  * @returns {import("preact").JSX.Element} The error component.
  */
-export const FlatSampleError = ({ style }) => {
+export const FlatSampleError = ({ message, style }) => {
   return html`<div
     style=${{
       color: "var(--bs-danger)",
@@ -68,7 +71,26 @@ export const FlatSampleError = ({ style }) => {
         height: FontSize.base,
       }}
     >
-      Error
+      ${errorType(message)}
     </div>
   </div>`;
 };
+
+/**
+ * Extracts the error type from a given message.
+ * If the message contains parentheses, it returns the substring before the first parenthesis.
+ * Otherwise, it returns "Error".
+ *
+ * @param {string | undefined} message - The error message from which to extract the type.
+ * @returns {string} The extracted error type or "Error" if not found.
+ */
+const errorType = (message) => {
+  if (!message) {
+    return "Error";
+  }
+
+  if (message.includes("(")) {
+    return message.split("(")[0];
+  }
+  return "Error";
+};

src/inspect_ai/_view/www/src/samples/SampleList.mjs

@@ -306,7 +306,7 @@ const SampleRow = ({
         }}
       >
         ${sample.error
-          ? html`<${SampleError} />`
+          ? html`<${SampleError} message=${sample.error.message} />`
           : sampleDescriptor?.selectedScore(sample).render()}
       </div>
     </div>

src/inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs

@@ -3,6 +3,7 @@ import { html } from "htm/preact";
 import { ApplicationIcons } from "../../appearance/Icons.mjs";
 import { EventPanel } from "./EventPanel.mjs";
 import { JSONPanel } from "../../components/JsonPanel.mjs";
+import { MarkdownDiv } from "../../components/MarkdownDiv.mjs";
 
 /**
  * Renders the InfoEventView component.
@@ -14,8 +15,22 @@ import { JSONPanel } from "../../components/JsonPanel.mjs";
  * @returns {import("preact").JSX.Element} The component.
  */
 export const InfoEventView = ({ id, event, style }) => {
+  const panels = [];
+  if (typeof event.data === "string") {
+    panels.push(
+      html`<${MarkdownDiv}
+        markdown=${event.data}
+        style=${{ margin: "0.5em 0" }}
+      />`,
+    );
+  } else {
+    panels.push(
+      html`<${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }} />`,
+    );
+  }
+
   return html`
   <${EventPanel} id=${id} title="Info" icon=${ApplicationIcons.info} style=${style}>
-    <${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }}/>
+    ${panels}
   </${EventPanel}>`;
 };

src/inspect_ai/model/_providers/providers.py

@@ -1,3 +1,5 @@
+import os
+
 from inspect_ai._util.error import pip_dependency_error
 from inspect_ai._util.version import verify_required_version
 
@@ -69,6 +71,10 @@ def vertex() -> type[ModelAPI]:
     PACKAGE = "google-cloud-aiplatform"
     MIN_VERSION = "1.59.0"
 
+    # workaround log spam
+    # https://github.com/ray-project/ray/issues/24917
+    os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "0"
+
     # verify we have the package
     try:
         import vertexai  # type: ignore  # noqa: F401
@@ -90,6 +96,10 @@ def google() -> type[ModelAPI]:
     PACKAGE = "google-generativeai"
     MIN_VERSION = "0.8.1"
 
+    # workaround log spam
+    # https://github.com/ray-project/ray/issues/24917
+    os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "0"
+
     # verify we have the package
     try:
         import google.generativeai  # type: ignore  # noqa: F401