Add simple unexpectedly crashy testcase

crates/cnidarium/src/snapshot.rs

@@ -82,11 +82,13 @@ impl Snapshot {
             db: db.clone(),
         };
 
+        println!("HERE_");
         let (substore_value, substore_commitment_proof) = tokio::task::spawn_blocking({
             let span = span.clone();
             move || span.in_scope(|| substore.get_with_proof(substore_key_bytes))
         })
         .await??;
+        println!("HERE2");
 
         proofs.push(substore_commitment_proof);
 
@@ -103,15 +105,18 @@ impl Snapshot {
                 db,
             };
 
+            println!("HERE3");
             let (_, main_commitment_proof) = tokio::task::spawn_blocking({
                 let span = span.clone();
                 move || span.in_scope(|| mainstore.get_with_proof(key_to_substore_root.into()))
             })
             .await??;
+            println!("HERE4");
 
             proofs.push(main_commitment_proof);
         }
 
+        println!("HERE5, substore_value: {:?}", substore_value);
         Ok((
             substore_value,
             MerkleProof {

crates/cnidarium/src/store/substore.rs

@@ -229,6 +229,13 @@ impl SubstoreSnapshot {
     ) -> Result<(Option<Vec<u8>>, ics23::CommitmentProof)> {
         let version = self.version();
         let tree = jmt::Sha256Jmt::new(self);
+        let rh = tree.get_root_hash(version)?;
+        println!(
+            "HERE in SubstoreSnapshot get_with_proof, key: {}, version: {}, root hash: {}",
+            hex::encode(key.clone()),
+            version,
+            hex::encode(rh.0)
+        );
         tree.get_with_ics23_proof(key, version)
     }
 

crates/core/app/tests/common/ibc_tests/node.rs

@@ -59,6 +59,8 @@ pub struct TestNodeWithIBC {
 }
 
 #[allow(unused)]
+/// This interacts with a node similarly to how a relayer would. We intentionally call
+/// against the external gRPC interfaces to get the most comprehensive test coverage.
 impl TestNodeWithIBC {
     pub async fn new(
         suffix: &str,
@@ -191,6 +193,7 @@ impl TestNodeWithIBC {
         client_id: &ClientId,
         height: &Height,
     ) -> Result<Option<(Height, ConsensusState)>> {
+        // TODO: this probably shouldn't reach into storage directly huh
         self.storage
             .clone()
             .latest_snapshot()

crates/core/app/tests/common/temp_storage_ext.rs

@@ -14,6 +14,7 @@ pub trait TempStorageExt: Sized {
 
 #[async_trait]
 impl TempStorageExt for TempStorage {
+    // TODO: move this to an Ext trait for Storage instead of TempStorage
     async fn new_with_penumbra_prefixes() -> anyhow::Result<TempStorage> {
         TempStorage::new_with_prefixes(SUBSTORE_PREFIXES.to_vec()).await
     }