feat(pd): support archives for migrate and join

Enables opt-in archive generation when performing:

  * pd export
  * pd migrate

The goal is to provide a standardized bottling-up of pd state,
specifically the rocksdb directory. In the context of upgrades,
the `pd migrate` functionality is what's important:
we want the archived dir to contain both rocksdb data and the modified
genesis file and updated validator state.

Accordingly, `pd testnet join` is modified to support an optional
archive URL. If set, the remote tar.gz archive will be downloaded
and extracted, clobbering the cometbft config files that were fetched.
A remote bootstrap node is still contacted, to learn about peers,
otherwise the newly created node wouldn't be able to talk to the network.

crates/bin/pd/Cargo.toml

@@ -50,6 +50,7 @@ decaf377                         = { workspace = true, features = ["parallel"],
 decaf377-rdsa                    = { workspace = true }
 directories                      = { workspace = true }
 ed25519-consensus                = { workspace = true }
+flate2                           = "1.0.28"
 fs_extra                         = "1.3.0"
 futures                          = { workspace = true }
 hex                              = { workspace = true }
@@ -91,12 +92,13 @@ rand                             = { workspace = true }
 rand_chacha                      = { workspace = true }
 rand_core                        = { workspace = true, features = ["getrandom"] }
 regex                            = { workspace = true }
-reqwest                          = { version = "0.11", features = ["json"] }
+reqwest                          = { version = "0.11", features = ["json", "stream"] }
 rocksdb                          = { workspace = true }
 serde                            = { workspace = true, features = ["derive"] }
 serde_json                       = { workspace = true }
 serde_with                       = { workspace = true, features = ["hex"] }
 sha2                             = { workspace = true }
+tar                              = "0.4.40"
 tempfile                         = { workspace = true }
 tendermint                       = { workspace = true }
 tendermint-config                = { workspace = true }

crates/bin/pd/src/cli.rs

@@ -113,23 +113,32 @@ pub enum RootCommand {
         /// The home directory of the full node.
         #[clap(long, env = "PENUMBRA_PD_HOME", display_order = 100)]
         home: PathBuf,
-        /// The directory that the exported state will be written to.
+        /// The directory where the exported node state will be written.
+        #[clap(long, display_order = 200, alias = "export-path")]
+        export_directory: PathBuf,
+        /// An optional filepath for a compressed archive containing the exported
+        /// node state, e.g. ~/pd-backup.tar.gz.
         #[clap(long, display_order = 200)]
-        export_path: PathBuf,
+        export_archive: Option<PathBuf>,
         /// Whether to prune the JMT tree.
         #[clap(long, display_order = 300)]
         prune: bool,
     },
     /// Run a migration on the exported storage state of the full node,
     /// and create a genesis file.
     Migrate {
-        /// The directory containing exported state to which the upgrade will be applied.
-        #[clap(long, display_order = 200)]
-        target_dir: PathBuf,
+        /// The directory containing exported state, created via `pd export`, to be modified
+        /// in-place. This should be a pd home directory, with a subdirectory called "rocksdb".
+        #[clap(long, display_order = 200, alias = "target-dir")]
+        target_directory: PathBuf,
         #[clap(long, display_order = 300)]
         /// Timestamp of the genesis file in RFC3339 format. If unset, defaults to the current time,
         /// unless the migration logic overrides it.
         genesis_start: Option<tendermint::time::Time>,
+        /// An optional filepath for a compressed archive containing the migrated node state,
+        /// e.g. ~/pd-state-post-upgrade.tar.gz.
+        #[clap(long, display_order = 400)]
+        migrate_archive: Option<PathBuf>,
     },
 }
 
@@ -197,6 +206,13 @@ pub enum TestnetCommand {
             default_value = "https://rpc.testnet.penumbra.zone"
         )]
         node: Url,
+
+        /// Optional URL of archived node state in .tar.gz format. The archive will be
+        /// downloaded and extracted locally, allowing the node to join a network at a block height
+        /// higher than 0.
+        #[clap(long)]
+        archive_url: Option<Url>,
+
         /// Human-readable name to identify node on network
         // Default: 'node-#'
         #[clap(long, env = "PENUMBRA_PD_TM_MONIKER")]

crates/bin/pd/src/main.rs

@@ -255,6 +255,7 @@ async fn main() -> anyhow::Result<()> {
             tn_cmd:
                 TestnetCommand::Join {
                     node,
+                    archive_url,
                     moniker,
                     external_address,
                     tendermint_rpc_bind,
@@ -290,14 +291,19 @@ async fn main() -> anyhow::Result<()> {
             // Join the target testnet, looking up network info and writing
             // local configs for pd and tendermint.
             testnet_join(
-                output_dir,
+                output_dir.clone(),
                 node,
                 &node_name,
                 external_address,
                 tendermint_rpc_bind,
                 tendermint_p2p_bind,
             )
             .await?;
+
+            // Download and extract archive URL, if set.
+            if let Some(archive_url) = archive_url {
+                pd::testnet::join::unpack_state_archive(archive_url, output_dir).await?;
+            }
         }
 
         RootCommand::Testnet {
@@ -379,44 +385,79 @@ async fn main() -> anyhow::Result<()> {
             t.write_configs()?;
         }
         RootCommand::Export {
-            mut home,
-            mut export_path,
+            home,
+            export_directory,
+            export_archive,
             prune,
         } => {
             use fs_extra;
 
-            tracing::info!("exporting state to {}", export_path.display());
+            // Export state as directory.
+            let src_rocksdb_dir = home.join("rocksdb");
+            tracing::info!(
+                "copying node state {} -> {}",
+                src_rocksdb_dir.display(),
+                export_directory.display()
+            );
+            std::fs::create_dir_all(&export_directory)?;
             let copy_opts = fs_extra::dir::CopyOptions::new();
-            home.push("rocksdb");
-            let from = [home.as_path()];
-            tracing::info!(?home, ?export_path, "copying from data dir to export dir",);
-            std::fs::create_dir_all(&export_path)?;
-            fs_extra::copy_items(&from, export_path.as_path(), &copy_opts)?;
-
-            tracing::info!("done copying");
-            if !prune {
-                return Ok(());
+            fs_extra::copy_items(
+                &[src_rocksdb_dir.as_path()],
+                export_directory.as_path(),
+                &copy_opts,
+            )?;
+            tracing::info!("finished copying node state");
+
+            let dst_rocksdb_dir = export_directory.join("rocksdb");
+            // If prune=true, then export-directory is required, because we must munge state prior
+            // to compressing. So we'll just mandate the presence of the --export-directory arg
+            // always.
+            if prune {
+                tracing::info!("pruning JMT tree");
+                let export = Storage::load(dst_rocksdb_dir, SUBSTORE_PREFIXES.to_vec()).await?;
+                let _ = StateDelta::new(export.latest_snapshot());
+                // TODO:
+                // - add utilities in `cnidarium` to prune a tree
+                // - apply the delta to the exported storage
+                // - apply checks: root hash, size, etc.
+                todo!()
             }
 
-            tracing::info!("pruning JMT tree");
-            export_path.push("rocksdb");
-            let export = Storage::load(export_path, SUBSTORE_PREFIXES.to_vec()).await?;
-            let _ = StateDelta::new(export.latest_snapshot());
-            // TODO:
-            // - add utilities in `cnidarium` to prune a tree
-            // - apply the delta to the exported storage
-            // - apply checks: root hash, size, etc.
-            todo!()
+            // Compress to tarball if requested.
+            if let Some(archive_filepath) = export_archive {
+                pd::migrate::archive_directory(
+                    dst_rocksdb_dir.clone(),
+                    archive_filepath.clone(),
+                    Some("rocksdb".to_owned()),
+                )?;
+                tracing::info!("export complete: {}", archive_filepath.display());
+            } else {
+                // Provide friendly "OK" message that's still accurate without archiving.
+                tracing::info!("export complete: {}", export_directory.display());
+            }
         }
         RootCommand::Migrate {
-            target_dir,
+            target_directory,
             genesis_start,
+            migrate_archive,
         } => {
-            tracing::info!("migrating state from {}", target_dir.display());
+            tracing::info!("migrating state in {}", target_directory.display());
             SimpleMigration
-                .migrate(target_dir.clone(), genesis_start)
+                .migrate(target_directory.clone(), genesis_start)
                 .await
                 .context("failed to upgrade state")?;
+            // Compress to tarball if requested.
+            if let Some(archive_filepath) = migrate_archive {
+                pd::migrate::archive_directory(
+                    target_directory.clone(),
+                    archive_filepath.clone(),
+                    None,
+                )?;
+                tracing::info!("migration complete: {}", archive_filepath.display());
+            } else {
+                // Provide friendly "OK" message that's still accurate without archiving.
+                tracing::info!("migration complete: {}", target_directory.display());
+            }
         }
     }
     Ok(())

crates/bin/pd/src/migrate.rs

@@ -4,6 +4,7 @@
 //! node operators must coordinate to perform a chain upgrade.
 //! This module declares how local `pd` state should be altered, if at all,
 //! in order to be compatible with the network post-chain-upgrade.
+use anyhow::Context;
 use std::path::PathBuf;
 
 use cnidarium::{StateDelta, StateWrite, Storage};
@@ -16,6 +17,10 @@ use penumbra_stake::{
 
 use crate::testnet::generate::TestnetConfig;
 
+use flate2::write::GzEncoder;
+use flate2::Compression;
+use std::fs::File;
+
 /// The kind of migration that should be performed.
 pub enum Migration {
     /// No-op migration.
@@ -36,9 +41,8 @@ impl Migration {
         match self {
             Migration::Noop => (),
             Migration::SimpleMigration => {
-                let mut db_path = path_to_export.clone();
-                db_path.push("rocksdb");
-                let storage = Storage::load(db_path, SUBSTORE_PREFIXES.to_vec()).await?;
+                let rocksdb_dir = path_to_export.join("rocksdb");
+                let storage = Storage::load(rocksdb_dir, SUBSTORE_PREFIXES.to_vec()).await?;
                 let export_state = storage.latest_snapshot();
                 let root_hash = export_state.root_hash().await.expect("can get root hash");
                 let app_hash_pre_migration: RootHash = root_hash.into();
@@ -97,12 +101,10 @@ impl Migration {
 
                 let genesis_json = serde_json::to_string(&genesis).expect("can serialize genesis");
                 tracing::info!("genesis: {}", genesis_json);
-                let mut genesis_path = path_to_export.clone();
-                genesis_path.push("genesis.json");
+                let genesis_path = path_to_export.join("genesis.json");
                 std::fs::write(genesis_path, genesis_json).expect("can write genesis");
 
-                let mut validator_state_path = path_to_export.clone();
-                validator_state_path.push("priv_validator_state.json");
+                let validator_state_path = path_to_export.join("priv_validator_state.json");
                 let fresh_validator_state =
                     crate::testnet::generate::TestnetValidator::initial_state();
                 std::fs::write(validator_state_path, fresh_validator_state)
@@ -113,3 +115,35 @@ impl Migration {
         Ok(())
     }
 }
+
+/// Compress single directory to gzipped tar archive. Accepts an Option for naming
+/// the subdir within the tar archive, which defaults to ".", meaning no nesting.
+pub fn archive_directory(
+    src_directory: PathBuf,
+    archive_filepath: PathBuf,
+    subdir_within_archive: Option<String>,
+) -> anyhow::Result<()> {
+    // Don't clobber an existing target archive.
+    if archive_filepath.exists() {
+        tracing::error!(
+            "export archive filepath already exists: {}",
+            archive_filepath.display()
+        );
+        anyhow::bail!("refusing to overwrite existing archive");
+    }
+
+    tracing::info!(
+        "creating archive {} -> {}",
+        src_directory.display(),
+        archive_filepath.display()
+    );
+    let tarball_file = File::create(&archive_filepath)
+        .context("failed to create file for archive: check parent directory and permissions")?;
+    let enc = GzEncoder::new(tarball_file, Compression::default());
+    let mut tarball = tar::Builder::new(enc);
+    let subdir_within_archive = subdir_within_archive.unwrap_or(String::from("."));
+    tarball
+        .append_dir_all(subdir_within_archive, src_directory.as_path())
+        .context("failed to package archive contents")?;
+    Ok(())
+}