From 2a0b6b972c75464e790a52ac712a95ff8d7867d9 Mon Sep 17 00:00:00 2001 From: aisi-inspect <166920645+aisi-inspect@users.noreply.github.com> Date: Mon, 6 May 2024 20:09:48 +0000 Subject: [PATCH] release v0.3.6 --- CHANGELOG.md | 6 ++++ src/inspect_ai/_cli/info.py | 25 +++++++++++++-- src/inspect_ai/_cli/main.py | 21 ++++++++---- src/inspect_ai/_util/constants.py | 2 +- src/inspect_ai/_view/www/App.mjs | 53 +++++++++++++++---------------- 5 files changed, 70 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a13d59007..774b9f746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.3.6 (06 May 2024) + +- Show first log file immediately (don't wait for fetching metadata for other logs) +- Add `--version` CLI arg and `inspect info version` command for interogating version and runtime source path. +- Fix: exclude `null` config values in output from `inspect info log-file` + ## v0.3.5 (04 May 2024) - Fix issue with logs from S3 buckets in inspect view. diff --git a/src/inspect_ai/_cli/info.py b/src/inspect_ai/_cli/info.py index ba1a90038..9f4ed10a9 100644 --- a/src/inspect_ai/_cli/info.py +++ b/src/inspect_ai/_cli/info.py @@ -1,7 +1,10 @@ +from json import dumps + import click +from inspect_ai import __version__ from inspect_ai._util.constants import PKG_PATH -from inspect_ai.log import read_eval_log +from inspect_ai.log import eval_log_json, read_eval_log @click.group("info") @@ -10,6 +13,22 @@ def info_command() -> None: return None +@info_command.command("version") +@click.option( + "--json", + type=bool, + is_flag=True, + default=False, + help="Output version and path info as JSON", +) +def version(json: bool) -> None: + if json: + print(dumps(dict(version=__version__, path=PKG_PATH.as_posix()), indent=2)) + else: + print(f"version: {__version__}") + print(f"path: {PKG_PATH.as_posix()}") + + @info_command.command("log-file") @click.argument("path") @click.option( @@ -22,7 +41,7 @@ def info_command() -> None: def log(path: str, header_only: bool) -> None: """Print log file contents.""" log = read_eval_log(path, header_only=header_only) - print(log.model_dump_json(indent=2)) + print(eval_log_json(log)) @info_command.command("log-schema") @@ -38,6 +57,6 @@ def log_types() -> None: def view_resource(file: str) -> str: - resource = PKG_PATH / "src" / "inspect_ai" / "_view" / "www" / file + resource = PKG_PATH / "_view" / "www" / file with open(resource, "r", encoding="utf-8") as f: return f.read() diff --git a/src/inspect_ai/_cli/main.py b/src/inspect_ai/_cli/main.py index 40f822ef3..34c984bfa 100644 --- a/src/inspect_ai/_cli/main.py +++ b/src/inspect_ai/_cli/main.py @@ -2,6 +2,7 @@ from inspect_ai._util.dotenv import init_dotenv +from .. import __version__ from .eval import eval_command from .info import info_command from .list import list_command @@ -10,17 +11,25 @@ @click.group(invoke_without_command=True) +@click.option( + "--version", + type=bool, + is_flag=True, + default=False, + help="Print the Inspect version.", +) @click.pass_context -def inspect( - ctx: click.Context, -) -> None: +def inspect(ctx: click.Context, version: bool) -> None: # if this was a subcommand then allow it to execute if ctx.invoked_subcommand is not None: return - # if invoked as plain 'inspect' just print help and exit - click.echo(ctx.get_help()) - ctx.exit() + if version: + print(__version__) + ctx.exit() + else: + click.echo(ctx.get_help()) + ctx.exit() inspect.add_command(eval_command) diff --git a/src/inspect_ai/_util/constants.py b/src/inspect_ai/_util/constants.py index 90116b065..4e525bf36 100644 --- a/src/inspect_ai/_util/constants.py +++ b/src/inspect_ai/_util/constants.py @@ -3,7 +3,7 @@ PKG_AUTHOR = "UK AI Safety Institute" PKG_AUTHOR_DIR = "UK-AISI" PKG_NAME = Path(__file__).parent.parent.stem -PKG_PATH = Path(__file__).parent.parent.parent.parent +PKG_PATH = Path(__file__).parent.parent DEFAULT_EPOCHS = 1 DEFAULT_MAX_RETRIES = 5 DEFAULT_TIMEOUT = 120 diff --git a/src/inspect_ai/_view/www/App.mjs b/src/inspect_ai/_view/www/App.mjs index 3aa285421..58ead8e51 100644 --- a/src/inspect_ai/_view/www/App.mjs +++ b/src/inspect_ai/_view/www/App.mjs @@ -22,11 +22,29 @@ export function App() { useEffect(() => { // Default select the first item let index = 0; - setSelected(index); }, [logs]); - useEffect(() => { + useEffect(async () => { + // Read header information for the logs + // and then update + const headerResults = await Promise.all(logs.files.map((file) => { + return eval_log(file.name, true).then((result) => { + return { file: file.name, result }; + }).catch(() => { return undefined}); + })); + + // Update the headers + const updatedHeaders = logHeaders; + for (const headerResult of headerResults) { + if (headerResult) { + updatedHeaders[headerResult.file] = headerResult.result; + } + } + setLogHeaders({ ...updatedHeaders }); + }, [logs]); + + useEffect(async () => { const urlParams = new URLSearchParams(window.location.search); // Note whether we should default off canvas the sidebar @@ -35,40 +53,21 @@ export function App() { // If the URL provides a task file, load that const logPath = urlParams.get("task_file"); const loadLogs = logPath - ? () => { + ? async () => { setLogs({ log_dir: "", files: [{ name: logPath }], }); } - : () => { - eval_logs().then((logresult) => { - // Set the list of logs - setLogs(logresult); + : async () => { + // Set the list of logs + const logresult = await eval_logs(); + setLogs(logresult); - // Read header information for the logs - // and then update - const updatedHeaders = logHeaders; - Promise.all( - logresult.files.map(async (file) => { - try { - const result = await eval_log(file.name, true); - return { file: file.name, result }; - } catch { } - }) - ).then((headerResults) => { - for (const headerResult of headerResults) { - if (headerResult) { - updatedHeaders[headerResult.file] = headerResult.result; - } - } - setLogHeaders({ ...updatedHeaders }); - }); - }); }; // initial fetch of logs - loadLogs(); + await loadLogs(); // poll every 1s for events setInterval(() => {