thread 'test_ingest_sql' panicked at /__w/mobilecoin/mobilecoin/fog/test_infra/src/lib.rs:173:17:
users did not converge to ledger_db.num_blocks before we ran out of retry attempts
stack backtrace:
   0: rust_begin_unwind
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/std/src/panicking.rs:597:5
   1: core::panicking::panic_fmt
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/core/src/panicking.rs:72:14
   2: tx_recovery::test_ingest_polling_integration
   3: mc_util_test_helper::run_with_several_seeds
   4: tx_recovery::__wrapped_test_ingest_sql
   5: core::ops::function::FnOnce::call_once
   6: core::ops::function::FnOnce::call_once
             at /rustc/ca62d2c445628587660ae48013f460b08b1f5552/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
2024-01-12 06:50:45.709609742 UTC DEBG Ingesting block #6: 10/10 txs ingested, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:587
2024-01-12 06:50:45.709688359 UTC INFO add_block_data, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:596
2024-01-12 06:50:46.696134345 UTC DEBG Checking up on peers..., mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-01-12 06:50:51.696788721 UTC DEBG Checking up on peers..., mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-01-12 06:50:52.856077749 UTC INFO Succeeded writing block 6 to the database, invocation id Some(IngestInvocationId(2)) ingress key CompressedRistrettoPublic(a29af9b673bbe389f0b0ea031da3eee9d4b5c6124d854e03f08d2631cb017722), mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:636
2024-01-12 06:50:52.856131679 UTC DEBG Incremented next block index: 7, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller_state, mc.src: fog/ingest/server/src/controller_state.rs:155
2024-01-12 06:50:52.856174620 UTC DEBG Controller state: { mode: Active, next_block_index: 7, pubkey_expiry_window: 10, ingest_invocation_id: 2, peers: [insecure-igp://0.0.0.0:3141/] }, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:647
2024-01-12 06:50:52.856254590 UTC INFO Finished ingesting block #6, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:659
2024-01-12 06:50:52.856307589 UTC DEBG Writing state file to StateFile { file_path: "/tmp/.tmpZyDnSt/mc-fog-ingest-state-1" }, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::controller, mc.src: fog/ingest/server/src/controller.rs:1314
2024-01-12 06:50:56.697324854 UTC DEBG Checking up on peers..., mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:235
2024-01-12 06:51:03.166474642 UTC INFO Stop Requested: Polling loop stopped at block number 7, is_idle false, mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:75
2024-01-12 06:51:04.166612387 UTC DEBG Peer checkup thread stop requested., mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:229
2024-01-12 06:51:04.971146144 UTC DEBG Report cache thread stop requested., mc.node_id: 1, mc.test_name: tx_recovery::test_ingest_sql, mc.module: mc_fog_ingest_server::worker, mc.src: fog/ingest/server/src/worker.rs:289