From b7c9fff69d71f509591a1a6c2f6c9d245dfed352 Mon Sep 17 00:00:00 2001 From: Ada Bohm Date: Fri, 20 Sep 2024 12:53:55 +0200 Subject: [PATCH] "hq read" renamed to "hq output-log" --- CHANGELOG.md | 4 +- crates/hyperqueue/src/bin/hq.rs | 4 +- crates/hyperqueue/src/client/commands/mod.rs | 2 +- .../commands/{reader.rs => outputlog.rs} | 4 +- crates/hyperqueue/src/common/cli.rs | 4 +- .../hyperqueue/src/stream/reader/streamdir.rs | 2 +- docs/imgs/cheatsheet.svg | 36 ++++++++++--- docs/imgs/streaming.svg | 17 +++--- docs/jobs/streaming.md | 20 +++---- tests/test_placeholders.py | 2 +- tests/test_stream.py | 54 +++++++++---------- 11 files changed, 86 insertions(+), 63 deletions(-) rename crates/hyperqueue/src/client/commands/{reader.rs => outputlog.rs} (95%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1ebeaace..c67d5ea2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ ## Removed * Because worker streaming fully replaces original streaming, the original server streaming was removed. - For most cases, you can rename `--log` to `--stream` and `hq log` to `hq read`. See the docs for more details. + For most cases, you can rename `--log` to `--stream` and `hq log` to `hq output-log`. See the docs for more details. ## Fixes @@ -373,7 +373,7 @@ an old client/worker to a new server (Connecting a new client/worker to an old s ### Stream log -* Basic export of stream log into JSON (`hq read export`) +* Basic export of stream log into JSON (`hq output-log export`) ### Server diff --git a/crates/hyperqueue/src/bin/hq.rs b/crates/hyperqueue/src/bin/hq.rs index deda005a5..581ed7c12 100644 --- a/crates/hyperqueue/src/bin/hq.rs +++ b/crates/hyperqueue/src/bin/hq.rs @@ -14,7 +14,7 @@ use hyperqueue::client::commands::job::{ output_job_summary, JobCancelOpts, JobCatOpts, JobCloseOpts, JobForgetOpts, JobInfoOpts, JobListOpts, JobTaskIdsOpts, }; -use hyperqueue::client::commands::reader::command_reader; +use hyperqueue::client::commands::outputlog::command_reader; use hyperqueue::client::commands::server::command_server; use hyperqueue::client::commands::submit::command::{open_job, SubmitJobConfOpts}; use hyperqueue::client::commands::submit::{ @@ -474,7 +474,7 @@ async fn main() -> hyperqueue::Result<()> { }) => command_task_info(&gsettings, opts).await, #[cfg(feature = "dashboard")] SubCommand::Dashboard(opts) => command_dashboard_start(&gsettings, opts).await, - SubCommand::Read(opts) => command_reader(&gsettings, opts), + SubCommand::OutputLog(opts) => command_reader(&gsettings, opts), SubCommand::AutoAlloc(opts) => command_autoalloc(&gsettings, opts).await, SubCommand::EventLog(opts) => command_event_log(&gsettings, opts).await, SubCommand::GenerateCompletion(opts) => generate_completion(opts), diff --git a/crates/hyperqueue/src/client/commands/mod.rs b/crates/hyperqueue/src/client/commands/mod.rs index 8646d7a8c..ce8f3a900 100644 --- a/crates/hyperqueue/src/client/commands/mod.rs +++ b/crates/hyperqueue/src/client/commands/mod.rs @@ -1,7 +1,7 @@ pub mod autoalloc; pub mod event; pub mod job; -pub mod reader; +pub mod outputlog; pub mod server; pub mod submit; pub mod wait; diff --git a/crates/hyperqueue/src/client/commands/reader.rs b/crates/hyperqueue/src/client/commands/outputlog.rs similarity index 95% rename from crates/hyperqueue/src/client/commands/reader.rs rename to crates/hyperqueue/src/client/commands/outputlog.rs index ca61770f6..55541813b 100644 --- a/crates/hyperqueue/src/client/commands/reader.rs +++ b/crates/hyperqueue/src/client/commands/outputlog.rs @@ -6,7 +6,7 @@ use clap::Parser; use std::path::PathBuf; #[derive(Parser)] -pub struct ReadOpts { +pub struct OutputLogOpts { /// Path of log file path: PathBuf, @@ -85,7 +85,7 @@ pub enum Channel { Stderr, } -pub fn command_reader(gsettings: &GlobalSettings, opts: ReadOpts) -> anyhow::Result<()> { +pub fn command_reader(gsettings: &GlobalSettings, opts: OutputLogOpts) -> anyhow::Result<()> { let mut stream_dir = StreamDir::open(&opts.path, opts.server_uid.as_deref())?; match opts.command { StreamCommand::Summary(_) => { diff --git a/crates/hyperqueue/src/common/cli.rs b/crates/hyperqueue/src/common/cli.rs index 409ee392f..da67153fc 100644 --- a/crates/hyperqueue/src/common/cli.rs +++ b/crates/hyperqueue/src/common/cli.rs @@ -12,7 +12,7 @@ use crate::client::commands::job::{ JobCancelOpts, JobCatOpts, JobCloseOpts, JobForgetOpts, JobInfoOpts, JobListOpts, JobTaskIdsOpts, }; -use crate::client::commands::reader::ReadOpts; +use crate::client::commands::outputlog::OutputLogOpts; use crate::client::commands::server::ServerOpts; use crate::client::commands::submit::command::SubmitJobConfOpts; use crate::client::commands::submit::{JobSubmitFileOpts, JobSubmitOpts}; @@ -199,7 +199,7 @@ pub enum SubCommand { /// Commands for controlling HyperQueue workers Worker(WorkerOpts), /// Operations with log - Read(ReadOpts), + OutputLog(OutputLogOpts), /// Automatic allocation management #[command(name = "alloc")] AutoAlloc(AutoAllocOpts), diff --git a/crates/hyperqueue/src/stream/reader/streamdir.rs b/crates/hyperqueue/src/stream/reader/streamdir.rs index 4f45a2aa4..fca42234a 100644 --- a/crates/hyperqueue/src/stream/reader/streamdir.rs +++ b/crates/hyperqueue/src/stream/reader/streamdir.rs @@ -1,4 +1,4 @@ -use crate::client::commands::reader::{CatOpts, Channel, ExportOpts, ShowOpts}; +use crate::client::commands::outputlog::{CatOpts, Channel, ExportOpts, ShowOpts}; use crate::common::arraydef::IntArray; use crate::common::error::HqError; use crate::server::event::bincode_config; diff --git a/docs/imgs/cheatsheet.svg b/docs/imgs/cheatsheet.svg index 185c454eb..64785d6a5 100644 --- a/docs/imgs/cheatsheet.svg +++ b/docs/imgs/cheatsheet.svg @@ -91,7 +91,7 @@ inkscape:pageshadow="2" inkscape:zoom="0.98994949" inkscape:cx="307.59145" - inkscape:cy="346.9874" + inkscape:cy="539.92654" inkscape:document-units="mm" inkscape:current-layer="layer1" inkscape:document-rotation="0" @@ -850,7 +850,7 @@ style="color:#000000;font-variation-settings:normal;overflow:visible;vector-effect:none;fill:#4d4d4d;fill-opacity:1;stroke:none;stroke-width:1.02965;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.419608;stop-color:#000000" id="rect2143-1-3" width="64.327179" - height="35.142365" + height="44.250179" x="137.02899" y="159.59457" ry="0" @@ -929,8 +929,8 @@ read <path> <command> + style="letter-spacing:-0.015749px;fill:#ffffff;stroke-width:0.207596px">output-log <path> <cmd> + jobs + show job ids diff --git a/docs/imgs/streaming.svg b/docs/imgs/streaming.svg index 31faecf89..4c6752e9b 100644 --- a/docs/imgs/streaming.svg +++ b/docs/imgs/streaming.svg @@ -47,7 +47,7 @@ showgrid="false" inkscape:zoom="0.61899712" inkscape:cx="533.92817" - inkscape:cy="425.68857" + inkscape:cy="424.07306" inkscape:window-width="1920" inkscape:window-height="971" inkscape:window-x="0" @@ -323,9 +323,10 @@ x="762.73413" y="466.72525">Log file + id="path7362" + sodipodi:nodetypes="sssssssss" /> hq read + style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'DejaVu Sans Mono';-inkscape-font-specification:'DejaVu Sans Mono';fill:#000000;stroke-width:0.747153">hq output-log ... directory HQ + id="tspan4">HQ Reader + id="tspan5">output log reader diff --git a/docs/jobs/streaming.md b/docs/jobs/streaming.md index caa8d1596..7c8be3224 100644 --- a/docs/jobs/streaming.md +++ b/docs/jobs/streaming.md @@ -77,7 +77,7 @@ HyperQueue lets you inspect the data stored inside the stream file using various the following structure: ```bash -$ hq read +$ hq output-log ``` ### Stream summary @@ -85,7 +85,7 @@ $ hq read You can display a summary of a log file using the `summary` subcommand: ```bash -$ hq read summary +$ hq output-log summary ``` ### Stream jobs @@ -93,7 +93,7 @@ $ hq read summary To print all job IDs that streaming in the stream path, you can run the following command: ```bash -$ hq read jobs +$ hq output-log jobs ``` ### Printing stream content @@ -102,7 +102,7 @@ If you want to simply print the (textual) content of the log file, without any a `cat` subcommand: ```bash -$ hq read cat +$ hq output-log cat ``` It will print the raw content of either `stdout` or `stderr`, ordered by task id. All outputs will be concatenated one @@ -119,7 +119,7 @@ If you want to inspect the contents of the log, along with its inner metadata th has produced which part of the data, you can use the `show` subcommand: ```commandline -$ hq read show +$ hq output-log show ``` The output will have the form `J.T:C> DATA` where `J` is a job id, `T` is a task id and `C` is `0` for `stdout` channel @@ -132,7 +132,7 @@ You can filter a specific channel with the `--channel=stdout/stderr` flag. Log can be exported into JSON by the following command: ```commandline -$ hq read export +$ hq output-log export ``` This prints the log file into a JSON format on standard output. @@ -145,14 +145,14 @@ hence HyperQueue streaming is able to avoid mixing outputs from different executions of the same task, when a task is restarted. HyperQueue automatically marks all output from previous instance of a task except the last instance as *superseded*. -You can see statistics about superseded data via `hq read summary` command. +You can see statistics about superseded data via `hq output-log summary` command. In the current version, superseded data is ignored by all other commands. ## More server instances HyperQueue supports writing streams from the different server instances into the same directory. -If you run `hq read` commands over such directory then it will detect the situation and prints all server uids -that writes into the directory. You have to specify the server instance via `hq read --server-uid= ...` +If you run `hq output-log` commands over such directory then it will detect the situation and prints all server uids +that writes into the directory. You have to specify the server instance via `hq output-log --server-uid= ...` when working with such a stream directory. !!! note @@ -163,4 +163,4 @@ when working with such a stream directory. ## Working with non-shared file system You do not need to have a shared file system when working with streaming path. It is just your responsibility to -collect all generated files into one directory before using `hq read` commands. \ No newline at end of file +collect all generated files into one directory before using `hq output-log` commands. \ No newline at end of file diff --git a/tests/test_placeholders.py b/tests/test_placeholders.py index 00f413b5a..ce52c787c 100644 --- a/tests/test_placeholders.py +++ b/tests/test_placeholders.py @@ -138,7 +138,7 @@ def test_stream_submit_placeholder(hq_env: HqEnv): hq_env.start_workers(1) wait_for_job_state(hq_env, 1, "FINISHED") - lines = set(hq_env.command(["read", "log-1", "show"], as_lines=True)) + lines = set(hq_env.command(["output-log", "log-1", "show"], as_lines=True)) assert "1.0:0> Hello" in lines diff --git a/tests/test_stream.py b/tests/test_stream.py index 53d26b752..b7697f52b 100644 --- a/tests/test_stream.py +++ b/tests/test_stream.py @@ -29,7 +29,7 @@ def test_stream_collision(hq_env: HqEnv): hq_env.stop_server() time.sleep(1) - print(hq_env.command(["read", "mylog", "show"], expect_fail="Found streams from multiple server instances")) + print(hq_env.command(["output-log", "mylog", "show"], expect_fail="Found streams from multiple server instances")) def test_stream_submit(hq_env: HqEnv): @@ -50,30 +50,30 @@ def test_stream_submit(hq_env: HqEnv): hq_env.start_workers(2, cpus="2") wait_for_job_state(hq_env, 1, "FINISHED") - lines = set(hq_env.command(["read", "mylog", "show"], as_lines=True)) + lines = set(hq_env.command(["output-log", "mylog", "show"], as_lines=True)) for i in range(1, 21): assert "1.{0:02}:0> Hello from {0}".format(i) in lines assert len(lines) == 20 - result = hq_env.command(["read", "mylog", "show", "--channel=stderr"]) + result = hq_env.command(["output-log", "mylog", "show", "--channel=stderr"]) assert result == "" - table = hq_env.command(["read", "mylog", "summary"], as_table=True) + table = hq_env.command(["output-log", "mylog", "summary"], as_table=True) table.check_row_value("Tasks", "20") table.check_row_value("Opened streams", "0") table.check_row_value("Stdout/stderr size", "271 B / 0 B") table.check_row_value("Superseded streams", "0") table.check_row_value("Superseded stdout/stderr size", "0 B / 0 B") - result = hq_env.command(["read", "mylog", "cat", "1", "stdout"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout"]) assert result == "".join(["Hello from {}\n".format(i) for i in range(1, 21)]) - result = hq_env.command(["read", "mylog", "cat", "1", "stdout", "--task=3-4,2"]).splitlines() + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout", "--task=3-4,2"]).splitlines() assert result[0] == "Hello from 3" assert result[1] == "Hello from 4" assert result[2] == "Hello from 2" - result = hq_env.command(["read", "mylog", "export", "1", "--task=3-4,2"], as_json=True) + result = hq_env.command(["output-log", "mylog", "export", "1", "--task=3-4,2"], as_json=True) assert result == [ {"finished": True, "id": 3, "stdout": "Hello from 3\n"}, {"finished": True, "id": 4, "stdout": "Hello from 4\n"}, @@ -98,13 +98,13 @@ def test_stream_overlap(hq_env: HqEnv): ) hq_env.start_workers(2) wait_for_job_state(hq_env, 1, "FINISHED") - result = hq_env.command(["read", "mylog", "show"], as_lines=True) + result = hq_env.command(["output-log", "mylog", "show"], as_lines=True) chunks = [set(result[i * 2 : i * 2 + 2]) for i in range(3)] assert chunks[0] == {"1.1:0> A", "1.2:0> A"} assert chunks[1] == {"1.2:0> B", "1.1:0> B"} - result = hq_env.command(["read", "mylog", "cat", "1", "stdout"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout"]) assert result == "A\nB\nA\nB\n" @@ -126,7 +126,7 @@ def test_stream_big_output(hq_env: HqEnv): hq_env.start_workers(2) wait_for_job_state(hq_env, 1, "FINISHED") - result = hq_env.command(["read", "mylog", "show"], as_lines=True) + result = hq_env.command(["output-log", "mylog", "show"], as_lines=True) # print(result) first = [] second = [] @@ -140,7 +140,7 @@ def test_stream_big_output(hq_env: HqEnv): assert "".join(first) == expected assert "".join(second) == expected - result = hq_env.command(["read", "mylog", "cat", "1", "stdout"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout"]) assert result == 2 * (expected + "\n") @@ -161,19 +161,19 @@ def test_stream_stderr(hq_env: HqEnv): hq_env.start_workers(2) wait_for_job_state(hq_env, 1, "FINISHED") - result = hq_env.command(["read", "mylog", "show"], as_lines=True) + result = hq_env.command(["output-log", "mylog", "show"], as_lines=True) assert set(result[0:2]) == {"1.0:0> Ok", "1.0:1> Error"} - result = hq_env.command(["read", "mylog", "show", "--channel=stdout"], as_lines=True) + result = hq_env.command(["output-log", "mylog", "show", "--channel=stdout"], as_lines=True) assert result[0] == "1.0:0> Ok" - result = hq_env.command(["read", "mylog", "show", "--channel=stderr"], as_lines=True) + result = hq_env.command(["output-log", "mylog", "show", "--channel=stderr"], as_lines=True) assert result[0] == "1.0:1> Error" - result = hq_env.command(["read", "mylog", "cat", "1", "stdout"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout"]) assert result == "Ok\n" - result = hq_env.command(["read", "mylog", "cat", "1", "stderr"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stderr"]) assert result == "Error\n" @@ -200,10 +200,10 @@ def test_stream_restart(hq_env: HqEnv): wait_for_job_state(hq_env, 1, "FINISHED") - result = hq_env.command(["read", "mylog", "cat", "1", "stdout"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout"]) assert result == "Start\nEnd 1\n" - table = hq_env.command(["read", "mylog", "summary"], as_table=True) + table = hq_env.command(["output-log", "mylog", "summary"], as_table=True) print(table) assert table[1] == ["Files", "2"] assert table[2] == ["Jobs", "1"] @@ -213,7 +213,7 @@ def test_stream_restart(hq_env: HqEnv): assert table[6] == ["Superseded streams", "1"] assert table[7] == ["Superseded stdout/stderr size", "6 B / 0 B"] - result = hq_env.command(["read", "mylog", "show"]) + result = hq_env.command(["output-log", "mylog", "show"]) assert result == "1.0:0> Start\n1.0:0> End 1\n" @@ -235,7 +235,7 @@ def test_stream_partial(hq_env: HqEnv): hq_env.start_workers(2) wait_for_job_state(hq_env, 1, "FINISHED") - result = hq_env.command(["read", "mylog", "show"], as_lines=True) + result = hq_env.command(["output-log", "mylog", "show"], as_lines=True) assert result[0] == "1.0:1> Error" check_file_contents("task-1-0-0.out", "Ok\n") @@ -258,7 +258,7 @@ def test_stream_unfinished_small(hq_env: HqEnv): ) hq_env.start_workers(2) wait_for_job_state(hq_env, 1, "RUNNING") - hq_env.command(["read", "mylog", "cat", "1", "stdout", "--allow-unfinished"], expect_fail="Job 1 not found") + hq_env.command(["output-log", "mylog", "cat", "1", "stdout", "--allow-unfinished"], expect_fail="Job 1 not found") wait_for_job_state(hq_env, 1, "FINISHED") @@ -283,15 +283,15 @@ def test_stream_unfinished_large(hq_env: HqEnv): wait_for_job_state(hq_env, 1, "RUNNING") time.sleep(1.0) hq_env.command( - ["read", "mylog", "cat", "1", "stdout"], + ["output-log", "mylog", "cat", "1", "stdout"], expect_fail="Stream for task 0 is not finished", ) - result = hq_env.command(["read", "mylog", "cat", "1", "stdout", "--allow-unfinished"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout", "--allow-unfinished"]) assert ("ab" * (16 * 1024 + 3)).startswith(result) assert len(result) >= 8 * 1024 wait_for_job_state(hq_env, 1, "FINISHED") - result = hq_env.command(["read", "mylog", "cat", "1", "stdout"]) + result = hq_env.command(["output-log", "mylog", "cat", "1", "stdout"]) assert "ab" * (16 * 1024 + 3) + "end\n" == result @@ -313,7 +313,7 @@ def test_stream_task_fail(hq_env: HqEnv): ) wait_for_job_state(hq_env, 1, "FAILED") - wait_until(lambda: hq_env.command(["read", "mylog", "show"]) == "1.0:0> Start\n") + wait_until(lambda: hq_env.command(["output-log", "mylog", "show"]) == "1.0:0> Start\n") def test_stream_task_cancel(hq_env: HqEnv): @@ -339,7 +339,7 @@ def test_stream_task_cancel(hq_env: HqEnv): hq_env.command(["job", "cancel", "1"]) wait_for_job_state(hq_env, 1, "CANCELED") - wait_until(lambda: hq_env.command(["read", "mylog", "show"]) == "1.0:0> Start\n") + wait_until(lambda: hq_env.command(["output-log", "mylog", "show"]) == "1.0:0> Start\n") def test_stream_timeout(hq_env: HqEnv): @@ -362,4 +362,4 @@ def test_stream_timeout(hq_env: HqEnv): wait_for_job_state(hq_env, 1, "FAILED") - wait_until(lambda: hq_env.command(["read", "mylog", "show"]) == "1.0:0> Start\n") + wait_until(lambda: hq_env.command(["output-log", "mylog", "show"]) == "1.0:0> Start\n")