diff --git a/docs/agents-api.qmd b/docs/agents-api.qmd
index 49de9c07e..502f59fbe 100644
--- a/docs/agents-api.qmd
+++ b/docs/agents-api.qmd
@@ -216,14 +216,6 @@ Note that we don't `await` the subtasks when building up our list of `searches`.
### Forking {#sec-forking}
-::: {.callout-note appearance="simple"}
-Note that the `fork()` feature described below is available only in the development version of Inspect. You can install the development version with:
-
-``` bash
-pip install git+https://github.com/ukgovernmentbeis/inspect_ai
-```
-:::
-
Inspect's `fork()` function provids a convenient wrapper around a very common use of subtasks: running a `TaskState` against a set of solvers in parallel to explore different trajectories.
For example, let's say you have a solver named `explore()` that takes `temperature` as a parameter. You might want to try the solver out with multiple temperature values and then continue on with the best result:
diff --git a/docs/log-viewer.qmd b/docs/log-viewer.qmd
index d324d1eb0..09c2f5643 100644
--- a/docs/log-viewer.qmd
+++ b/docs/log-viewer.qmd
@@ -123,7 +123,21 @@ You can see all of these log entries in the **Logging** tab:
{.border .lightbox fig-alt="The Logging panel of the Inspect log viewer, displaying several info log messages from the web search tool indicating what queries were executed by the tool."}
-It is important to note that the Inspect View will show all log entries level `info` or higher. However, printing every `info` message to the console during an eval might be too distracting, so the default log level for printing is `warning`. If you change it to `info` then you'll also see these log messages in the console:
+### Log Levels
+
+The log levels and their applicability are described below (in increasing order of severity):
+
+| Level | Description |
+|------------------------------------|------------------------------------|
+| `debug` | Detailed information, typically of interest only when diagnosing problems. |
+| `http` | HTTP diagnostics including requests and response statuses |
+| `sandbox` | Show commands sent to manage and execute code in sandboxes. |
+| `info` | Confirmation that things are working as expected. |
+| `warning` | or indicative of some problem in the near future (e.g. ‘disk space low’). The software is still working as expected. |
+| `error` | Due to a more serious problem, the software has not been able to perform some function |
+| `critical` | A serious error, indicating that the program itself may be unable to continue running. |
+
+: Inspect View will show all log entries level `info` or higher. However, printing every `info` message to the console during an eval might be too distracting, so the default log level for the console is `warning`. If you change it to `info` then you'll also see these log messages in the console:
``` bash
$ inspect eval biology_qa.py --log-level info
@@ -181,7 +195,6 @@ By default, an existing output dir will NOT be overwritten. Specify the `--overw
$ inspect view bundle --output-dir logs-www --overwrite
```
-
Bundling the viewer and logs will produce an output directory with the following structure:
``` bash
diff --git a/src/inspect_ai/_display/rich.py b/src/inspect_ai/_display/rich.py
index 03f23a7cf..d788e3d35 100644
--- a/src/inspect_ai/_display/rich.py
+++ b/src/inspect_ai/_display/rich.py
@@ -512,7 +512,7 @@ def task_config(profile: TaskProfile, generate_config: bool = True) -> str:
config = config | dict(profile.generate_config.model_dump(exclude_none=True))
config_print: list[str] = []
for name, value in config.items():
- if name not in ["limit", "epochs", "model"]:
+ if name not in ["limit", "model"]:
config_print.append(f"{name}: {value}")
values = ", ".join(config_print)
if values:
diff --git a/src/inspect_ai/_util/constants.py b/src/inspect_ai/_util/constants.py
index 2c737d71e..88e9d7d39 100644
--- a/src/inspect_ai/_util/constants.py
+++ b/src/inspect_ai/_util/constants.py
@@ -22,6 +22,7 @@
"INFO",
"WARNING",
"ERROR",
+ "CRITICAL",
]
DEFAULT_LOG_LEVEL = "warning"
DEFAULT_LOG_BUFFER_LOCAL = 10
diff --git a/src/inspect_ai/_view/www/dist/assets/index.js b/src/inspect_ai/_view/www/dist/assets/index.js
index 9fa13d25c..9260ca941 100644
--- a/src/inspect_ai/_view/www/dist/assets/index.js
+++ b/src/inspect_ai/_view/www/dist/assets/index.js
@@ -16327,9 +16327,22 @@ const JSONPanel = ({ data, style }) => {
`;
};
const InfoEventView = ({ id, event, style }) => {
+ const panels = [];
+ if (typeof event.data === "string") {
+ panels.push(
+ m$1`<${MarkdownDiv}
+ markdown=${event.data}
+ style=${{ margin: "0.5em 0" }}
+ />`
+ );
+ } else {
+ panels.push(
+ m$1`<${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }} />`
+ );
+ }
return m$1`
<${EventPanel} id=${id} title="Info" icon=${ApplicationIcons.info} style=${style}>
- <${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }}/>
+ ${panels}
`;
};
const ScoreEventView = ({ id, event, style }) => {
@@ -17354,7 +17367,7 @@ const resolveValue = (value, evalEvents) => {
}
return value;
};
-const SampleError = ({ align, style }) => {
+const SampleError = ({ message, align, style }) => {
align = align || "center";
return m$1` {
height: FontSize.small
}}
/>
-
Error
+
${errorType(message)}
{
height: FontSize.base
}}
>
- Error
+ ${errorType(message)}
`;
};
+const errorType = (message) => {
+ if (!message) {
+ return "Error";
+ }
+ if (message.includes("(")) {
+ return message.split("(")[0];
+ }
+ return "Error";
+};
const printHtml = (html2, css) => {
const printWindow = window.open("", "", "height=600,width=800");
printWindow.document.write("Print");
@@ -17727,7 +17749,10 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
}
columns.push({
label: "Score",
- value: sample.error ? m$1`<${FlatSampleError} style=${{ marginTop: "0.4rem" }} />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render(),
+ value: sample.error ? m$1`<${FlatSampleError}
+ message=${sample.error.message}
+ style=${{ marginTop: "0.4rem" }}
+ />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render(),
size: "minmax(2em, auto)",
center: true
});
@@ -18206,7 +18231,7 @@ const SampleRow = ({
display: "flex"
}}
>
- ${sample.error ? m$1`<${SampleError} />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render()}
+ ${sample.error ? m$1`<${SampleError} message=${sample.error.message} />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render()}
`;
@@ -21499,7 +21524,6 @@ function App({ api: api2, pollForLogs = true }) {
setHeadersLoading(false);
}, [logs, setStatus, setLogHeaders, setHeadersLoading]);
y(async () => {
- console.log({ logs });
const targetLog = logs.files[selected];
if (targetLog && (!currentLog || currentLog.name !== targetLog.name)) {
try {
@@ -21673,7 +21697,6 @@ function App({ api: api2, pollForLogs = true }) {
const index = result.files.findIndex((val) => {
return log_file.endsWith(val.name);
});
- console.log({ result, log_file, index });
if (index > -1) {
setSelected(index);
}
diff --git a/src/inspect_ai/_view/www/src/App.mjs b/src/inspect_ai/_view/www/src/App.mjs
index 12d4df4a8..1e5590547 100644
--- a/src/inspect_ai/_view/www/src/App.mjs
+++ b/src/inspect_ai/_view/www/src/App.mjs
@@ -94,7 +94,6 @@ export function App({ api, pollForLogs = true }) {
// Load a specific log
useEffect(async () => {
- console.log({ logs });
const targetLog = logs.files[selected];
if (targetLog && (!currentLog || currentLog.name !== targetLog.name)) {
try {
@@ -308,7 +307,6 @@ export function App({ api, pollForLogs = true }) {
const index = result.files.findIndex((val) => {
return log_file.endsWith(val.name);
});
- console.log({ result, log_file, index });
if (index > -1) {
setSelected(index);
}
diff --git a/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs b/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs
index 0bbbc65e1..d902a5e1e 100644
--- a/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs
+++ b/src/inspect_ai/_view/www/src/samples/SampleDisplay.mjs
@@ -368,7 +368,10 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
columns.push({
label: "Score",
value: sample.error
- ? html`<${FlatSampleError} style=${{ marginTop: "0.4rem" }} />`
+ ? html`<${FlatSampleError}
+ message=${sample.error.message}
+ style=${{ marginTop: "0.4rem" }}
+ />`
: sampleDescriptor?.selectedScore(sample).render(),
size: "minmax(2em, auto)",
center: true,
diff --git a/src/inspect_ai/_view/www/src/samples/SampleError.mjs b/src/inspect_ai/_view/www/src/samples/SampleError.mjs
index 453890819..b145345b5 100644
--- a/src/inspect_ai/_view/www/src/samples/SampleError.mjs
+++ b/src/inspect_ai/_view/www/src/samples/SampleError.mjs
@@ -8,12 +8,14 @@ import { ApplicationIcons } from "../appearance/Icons.mjs";
* Component to display a styled error message.
*
* @param {Object} props - The component properties.
+ * @param {string} [props.message] - The error message
* @param {string} [props.align="center"] - The alignment for the error message. Defaults to "center".
* @param {Object} [props.style] - Styles to add for this component
* @returns {import("preact").JSX.Element} The error component.
*/
-export const SampleError = ({ align, style }) => {
+export const SampleError = ({ message, align, style }) => {
align = align || "center";
+
return html` {
height: FontSize.small,
}}
/>
-
Error
+
${errorType(message)}
{
height: FontSize.base,
}}
>
- Error
+ ${errorType(message)}
`;
};
+
+/**
+ * Extracts the error type from a given message.
+ * If the message contains parentheses, it returns the substring before the first parenthesis.
+ * Otherwise, it returns "Error".
+ *
+ * @param {string | undefined} message - The error message from which to extract the type.
+ * @returns {string} The extracted error type or "Error" if not found.
+ */
+const errorType = (message) => {
+ if (!message) {
+ return "Error";
+ }
+
+ if (message.includes("(")) {
+ return message.split("(")[0];
+ }
+ return "Error";
+};
diff --git a/src/inspect_ai/_view/www/src/samples/SampleList.mjs b/src/inspect_ai/_view/www/src/samples/SampleList.mjs
index a4942d163..6430c5fdc 100644
--- a/src/inspect_ai/_view/www/src/samples/SampleList.mjs
+++ b/src/inspect_ai/_view/www/src/samples/SampleList.mjs
@@ -306,7 +306,7 @@ const SampleRow = ({
}}
>
${sample.error
- ? html`<${SampleError} />`
+ ? html`<${SampleError} message=${sample.error.message} />`
: sampleDescriptor?.selectedScore(sample).render()}
diff --git a/src/inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs b/src/inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs
index 67d347063..a74b70ddd 100644
--- a/src/inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs
+++ b/src/inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs
@@ -3,6 +3,7 @@ import { html } from "htm/preact";
import { ApplicationIcons } from "../../appearance/Icons.mjs";
import { EventPanel } from "./EventPanel.mjs";
import { JSONPanel } from "../../components/JsonPanel.mjs";
+import { MarkdownDiv } from "../../components/MarkdownDiv.mjs";
/**
* Renders the InfoEventView component.
@@ -14,8 +15,22 @@ import { JSONPanel } from "../../components/JsonPanel.mjs";
* @returns {import("preact").JSX.Element} The component.
*/
export const InfoEventView = ({ id, event, style }) => {
+ const panels = [];
+ if (typeof event.data === "string") {
+ panels.push(
+ html`<${MarkdownDiv}
+ markdown=${event.data}
+ style=${{ margin: "0.5em 0" }}
+ />`,
+ );
+ } else {
+ panels.push(
+ html`<${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }} />`,
+ );
+ }
+
return html`
<${EventPanel} id=${id} title="Info" icon=${ApplicationIcons.info} style=${style}>
- <${JSONPanel} data=${event.data} style=${{ margin: "0.5em 0" }}/>
+ ${panels}
`;
};
diff --git a/src/inspect_ai/model/_providers/providers.py b/src/inspect_ai/model/_providers/providers.py
index 4f191c755..be232faf4 100644
--- a/src/inspect_ai/model/_providers/providers.py
+++ b/src/inspect_ai/model/_providers/providers.py
@@ -1,3 +1,5 @@
+import os
+
from inspect_ai._util.error import pip_dependency_error
from inspect_ai._util.version import verify_required_version
@@ -69,6 +71,10 @@ def vertex() -> type[ModelAPI]:
PACKAGE = "google-cloud-aiplatform"
MIN_VERSION = "1.59.0"
+ # workaround log spam
+ # https://github.com/ray-project/ray/issues/24917
+ os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "0"
+
# verify we have the package
try:
import vertexai # type: ignore # noqa: F401
@@ -90,6 +96,10 @@ def google() -> type[ModelAPI]:
PACKAGE = "google-generativeai"
MIN_VERSION = "0.8.1"
+ # workaround log spam
+ # https://github.com/ray-project/ray/issues/24917
+ os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "0"
+
# verify we have the package
try:
import google.generativeai # type: ignore # noqa: F401