include backtrace from omdb panic from within omdb tests

[davepacheco]

This is basically to help debug issues like #6505.

[davepacheco]

With just the first commit, tests fail with:

$ cargo nextest run -p omicron-omdb
info: experimental features enabled: setup-scripts
   Compiling oximeter-db v0.1.0 (/home/dap/omicron-work/oximeter/db)
   Compiling omicron-test-utils v0.1.0 (/home/dap/omicron-work/test-utils)
   Compiling nexus-test-utils-macros v0.1.0 (/home/dap/omicron-work/nexus/test-utils-macros)
   Compiling gateway-test-utils v0.1.0 (/home/dap/omicron-work/gateway-test-utils)
   Compiling oximeter-collector v0.1.0 (/home/dap/omicron-work/oximeter/collector)
   Compiling omicron-nexus v0.1.0 (/home/dap/omicron-work/nexus)
   Compiling omicron-omdb v0.1.0 (/home/dap/omicron-work/dev-tools/omdb)
   Compiling nexus-test-utils v0.1.0 (/home/dap/omicron-work/nexus/test-utils)
    Finished `test` profile [unoptimized + debuginfo] target(s) in 4m 57s
------------
 Nextest run ID cf368dc4-dd83-4f02-a559-f4dbac6046ab with nextest profile: default
    Starting 3 tests across 2 binaries
       SETUP [      1/1] crdb-seed: cargo run -p crdb-seed --profile test
             [ 00:00:00] [                                               ] 0/3:         Compiling crdb-seed v0.1.0 (/home/dap/omicron-work/dev-tools/crdb-seed)
    Finished `test` profile [unoptimized + debuginfo] target(s) in 6.27s
     Running `target/debug/crdb-seed`
Sep 04 00:14:40.146 INFO Using existing CRDB seed tarball: `/dangerzone/omicron_tmp/crdb-base-dap/e995388eb7cee33d3fe65ee2692558445557501398f2dc426cf155ebc72e89d1.tar`
  SETUP PASS [      1/1] crdb-seed: cargo run -p crdb-seed --profile test
        FAIL [   3.990s] omicron-omdb::test_all_output test_omdb_usage_errors

--- STDOUT:              omicron-omdb::test_all_output test_omdb_usage_errors ---

running 1 test
@@ -597,10 +597,75 @@
 stdout:
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://[::1]:111
 Error: This command is potentially destructive. Pass the `-w` / `--destructive` flag to allow it.
+
+Stack backtrace:
+   0: anyhow::error::<impl anyhow::Error>::msg
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
+   1: anyhow::__private::format_err
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/lib.rs:689:13
+   2: omdb::check_allow_destructive::<impl omdb::Omdb>::check_allow_destructive
+             at ./src/bin/omdb/main.rs:127:13
+   3: {async_fn#0}
+             at ./src/bin/omdb/nexus.rs:477:37
+   4: {async_block#0}
+             at ./src/bin/omdb/main.rs:69:66
+   5: poll<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/future/future.rs:123:9
+   6: {closure#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:63
+   7: with_budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5
+   8: budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5
+   9: block_on<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:31
+  10: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/blocking.rs:66:9
+  11: tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:87:13
+  12: tokio::runtime::context::runtime::enter_runtime
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16
+  13: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:86:9
+  14: tokio::runtime::runtime::Runtime::block_on_inner
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:363:45
+  15: block_on<omdb::main::{async_block_env#0}>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:333:13
+  16: omdb::main
+             at ./src/bin/omdb/main.rs:66:5
+  17: core::ops::function::FnOnce::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:250:5
+  18: __rust_begin_short_backtrace<fn() -> core::result::Result<(), anyhow::Error>, core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:155:18
+  19: {closure#0}<core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:159:18
+  20: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:284:13
+  21: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  22: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  23: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  24: std::rt::lang_start_internal::{{closure}}
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:48
+  25: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  26: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  27: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  28: std::rt::lang_start_internal
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:20
+  29: std::rt::lang_start
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:158:17
+  30: main
+  31: _start_crt
+  32: _start
 =============================================
 EXECUTING COMMAND: omdb ["nexus", "sleds"]
 termination: Exited(2)
 ---------------------------------------------
 stdout:

test test_omdb_usage_errors ... FAILED

failures:

failures:
    test_omdb_usage_errors

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 2 filtered out; finished in 3.95s


--- STDERR:              omicron-omdb::test_all_output test_omdb_usage_errors ---
thread 'test_omdb_usage_errors' panicked at dev-tools/omdb/tests/test_all_output.rs:113:5:
assertion failed: string doesn't match the contents of file: "tests/usage_errors.out" see diffset above
                set EXPECTORATE=overwrite if these changes are intentional
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

   Canceling due to test failure: 2 tests still running
        FAIL [  12.949s] omicron-omdb::test_all_output test_omdb_env_settings

--- STDOUT:              omicron-omdb::test_all_output test_omdb_env_settings ---

running 1 test
@@ -184,10 +184,75 @@
 
 Caused by:
     0: Communication Error: builder error: relative URL without a base
     1: builder error: relative URL without a base
     2: relative URL without a base
+
+Stack backtrace:
+   0: <E as anyhow::context::ext::StdError>::ext_context
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
+   1: anyhow::context::<impl anyhow::Context<T,E> for core::result::Result<T,E>>::context
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/context.rs:54:31
+   2: {async_fn#0}
+             at ./src/bin/omdb/nexus.rs:519:9
+   3: {async_fn#0}
+             at ./src/bin/omdb/nexus.rs:379:59
+   4: {async_block#0}
+             at ./src/bin/omdb/main.rs:69:66
+   5: poll<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/future/future.rs:123:9
+   6: {closure#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:63
+   7: with_budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5
+   8: budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5
+   9: block_on<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:31
+  10: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/blocking.rs:66:9
+  11: tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:87:13
+  12: tokio::runtime::context::runtime::enter_runtime
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16
+  13: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:86:9
+  14: tokio::runtime::runtime::Runtime::block_on_inner
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:363:45
+  15: block_on<omdb::main::{async_block_env#0}>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:333:13
+  16: omdb::main
+             at ./src/bin/omdb/main.rs:66:5
+  17: core::ops::function::FnOnce::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:250:5
+  18: __rust_begin_short_backtrace<fn() -> core::result::Result<(), anyhow::Error>, core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:155:18
+  19: {closure#0}<core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:159:18
+  20: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:284:13
+  21: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  22: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  23: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  24: std::rt::lang_start_internal::{{closure}}
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:48
+  25: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  26: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  27: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  28: std::rt::lang_start_internal
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:20
+  29: std::rt::lang_start
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:158:17
+  30: main
+  31: _start_crt
+  32: _start
 =============================================
 EXECUTING COMMAND: omdb ["nexus", "background-tasks", "doc"]
 termination: Exited(0)
 ---------------------------------------------
 stdout:
@@ -550,10 +615,75 @@
 
 Caused by:
     0: Communication Error: builder error: relative URL without a base
     1: builder error: relative URL without a base
     2: relative URL without a base
+
+Stack backtrace:
+   0: <E as anyhow::context::ext::StdError>::ext_context
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
+   1: anyhow::context::<impl anyhow::Context<T,E> for core::result::Result<T,E>>::context
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/context.rs:54:31
+   2: {async_fn#0}
+             at ./src/bin/omdb/oximeter.rs:90:20
+   3: {async_fn#0}
+             at ./src/bin/omdb/oximeter.rs:84:45
+   4: {async_block#0}
+             at ./src/bin/omdb/main.rs:70:75
+   5: poll<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/future/future.rs:123:9
+   6: {closure#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:63
+   7: with_budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5
+   8: budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5
+   9: block_on<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:31
+  10: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/blocking.rs:66:9
+  11: tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:87:13
+  12: tokio::runtime::context::runtime::enter_runtime
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16
+  13: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:86:9
+  14: tokio::runtime::runtime::Runtime::block_on_inner
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:363:45
+  15: block_on<omdb::main::{async_block_env#0}>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:333:13
+  16: omdb::main
+             at ./src/bin/omdb/main.rs:66:5
+  17: core::ops::function::FnOnce::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:250:5
+  18: __rust_begin_short_backtrace<fn() -> core::result::Result<(), anyhow::Error>, core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:155:18
+  19: {closure#0}<core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:159:18
+  20: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:284:13
+  21: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  22: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  23: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  24: std::rt::lang_start_internal::{{closure}}
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:48
+  25: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  26: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  27: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  28: std::rt::lang_start_internal
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:20
+  29: std::rt::lang_start
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:158:17
+  30: main
+  31: _start_crt
+  32: _start
 =============================================
 EXECUTING COMMAND: omdb ["oxql", "--clickhouse-url", "junk"]
 termination: Exited(1)
 ---------------------------------------------
 stdout:
@@ -561,6 +691,71 @@
 stderr:
 Error: failed parsing URL from command-line or environment variable
 
 Caused by:
     relative URL without a base
+
+Stack backtrace:
+   0: <E as anyhow::context::ext::StdError>::ext_context
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
+   1: anyhow::context::<impl anyhow::Context<T,E> for core::result::Result<T,E>>::context
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/context.rs:54:31
+   2: {async_fn#0}
+             at ./src/bin/omdb/oxql.rs:72:37
+   3: {async_fn#0}
+             at ./src/bin/omdb/oxql.rs:49:41
+   4: {async_block#0}
+             at ./src/bin/omdb/main.rs:71:63
+   5: poll<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/future/future.rs:123:9
+   6: {closure#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:63
+   7: with_budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5
+   8: budget<core::task::poll::Poll<core::result::Result<(), anyhow::Error>>, tokio::runtime::park::{impl#4}::block_on::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5
+   9: block_on<core::pin::Pin<alloc::boxed::Box<omdb::main::{async_block_env#0}, alloc::alloc::Global>>>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/park.rs:281:31
+  10: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/blocking.rs:66:9
+  11: tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:87:13
+  12: tokio::runtime::context::runtime::enter_runtime
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16
+  13: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/mod.rs:86:9
+  14: tokio::runtime::runtime::Runtime::block_on_inner
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:363:45
+  15: block_on<omdb::main::{async_block_env#0}>
+             at /home/dap/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/runtime.rs:333:13
+  16: omdb::main
+             at ./src/bin/omdb/main.rs:66:5
+  17: core::ops::function::FnOnce::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:250:5
+  18: __rust_begin_short_backtrace<fn() -> core::result::Result<(), anyhow::Error>, core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/sys_common/backtrace.rs:155:18
+  19: {closure#0}<core::result::Result<(), anyhow::Error>>
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:159:18
+  20: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/core/src/ops/function.rs:284:13
+  21: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  22: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  23: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  24: std::rt::lang_start_internal::{{closure}}
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:48
+  25: std::panicking::try::do_call
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:559:40
+  26: std::panicking::try
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panicking.rs:523:19
+  27: std::panic::catch_unwind
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/panic.rs:149:14
+  28: std::rt::lang_start_internal
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:141:20
+  29: std::rt::lang_start
+             at /rustc/3f5fd8dd41153bc5fdca9427e9e05be2c767ba23/library/std/src/rt.rs:158:17
+  30: main
+  31: _start_crt
+  32: _start
 =============================================

test test_omdb_env_settings ... FAILED

failures:

failures:
    test_omdb_env_settings

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 2 filtered out; finished in 12.86s


--- STDERR:              omicron-omdb::test_all_output test_omdb_env_settings ---
log file: /dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.0.log
note: configured to log to "/dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.0.log"
DB URL: postgresql://root@[::1]:46970/omicron?sslmode=disable
DB address: [::1]:46970
log file: /dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.2.log
note: configured to log to "/dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.2.log"
log file: /dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.3.log
note: configured to log to "/dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.3.log"
thread 'test_omdb_env_settings' panicked at dev-tools/omdb/tests/test_all_output.rs:418:5:
assertion failed: string doesn't match the contents of file: "tests/env.out" see diffset above
                set EXPECTORATE=overwrite if these changes are intentional
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: temporary directory leaked: "/dangerzone/omicron_tmp/.tmppnrwq6"
        If you would like to access the database for debugging, run the following:

        # Run the database
        cargo xtask db-dev run --no-populate --store-dir "/dangerzone/omicron_tmp/.tmppnrwq6/data"
        # Access the database. Note the port may change if you run multiple databases.
        cockroach sql --host=localhost:32221 --insecure
WARN: dropped ClickHouseInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
failed to clean up ClickHouse data dir:
- /dangerzone/omicron_tmp/test_all_output-87caec0fc25c3d50-test_omdb_env_settings.15206.1-clickhouse-uO5kj4: File exists (os error 17)
WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: dendrite temporary directory leaked: /dangerzone/omicron_tmp/.tmpOyMWEz
WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: dendrite temporary directory leaked: /dangerzone/omicron_tmp/.tmpprtKil
WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: mgd temporary directory leaked: /dangerzone/omicron_tmp/.tmp4wdgRl
WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: mgd temporary directory leaked: /dangerzone/omicron_tmp/.tmpfH6JZ1

        PASS [  20.496s] omicron-omdb::test_all_output test_omdb_success_cases
------------
     Summary [  26.978s] 3 tests run: 1 passed, 2 failed, 0 skipped
        FAIL [  12.949s] omicron-omdb::test_all_output test_omdb_env_settings
        FAIL [   3.990s] omicron-omdb::test_all_output test_omdb_usage_errors
error: test run failed

This is where I learned about RUST_LIB_BACKTRACE, and the surprising (to me) behavior that with RUST_BACKTRACE=1, you get Rust traces dumped to stderr even for errors that are generated and cleanly handled (like the error messages above). So I followed the guidance in the anyhow crate's README:

If you want panics and errors to both have backtraces, set RUST_BACKTRACE=1;
If you want only errors to have backtraces, set RUST_LIB_BACKTRACE=1;
If you want only panics to have backtraces, set RUST_BACKTRACE=1 and RUST_LIB_BACKTRACE=0.

So I've set RUST_BACKTRACE=1 and RUST_LIB_BACKTRACE=0.

[davepacheco]

It was pointed out in oxide-rust that the reason I was getting the backtraces for handled errors wasn't that they were being printed synchronously when the error was created. Rather, they were being captured in the Error that main() was returning, and when main() returns Err, its debug output is printed out, and that includes the stack trace. So a different solution to that problem could be to have omdb explicitly format errors the way we want it to rather than using the fn main() -> Result<(), ...> syntax. I could make that change here, but I think we'd still want to set these env variables consistently, so I wasn't going to bother in this PR unless folks would prefer it.

[sunshowers]

I think this is fine. Thanks for doing this!