can parse transaction archive, and detect transfer type transactions

tools/storage/src/read_tx_chunk.rs

@@ -1,16 +1,15 @@
-
 use std::path::Path;
 
 use anyhow::{anyhow, Result};
 
 use diem_backup_cli::backup_types::transaction::manifest::TransactionBackup;
 use diem_backup_cli::backup_types::transaction::manifest::TransactionChunk;
+use diem_backup_cli::utils::read_record_bytes::ReadRecordBytes;
+use diem_types::contract_event::ContractEvent;
 use diem_types::transaction::Transaction;
 use diem_types::transaction::TransactionInfo;
-use diem_types::contract_event::ContractEvent;
 use diem_types::write_set::WriteSet;
 use libra_backwards_compatibility::version_five::state_snapshot_v5::open_for_read;
-use diem_backup_cli::utils::read_record_bytes::ReadRecordBytes;
 
 /// read snapshot manifest file into object
 pub fn load_tx_chunk_manifest(path: &Path) -> anyhow::Result<TransactionBackup> {
@@ -34,12 +33,11 @@ pub struct TransactionArchiveChunk {
     pub write_sets: Vec<WriteSet>,
 }
 
-
 pub async fn load_chunk(
     archive_path: &Path,
     manifest: TransactionChunk,
 ) -> Result<TransactionArchiveChunk> {
-        let full_handle = archive_path
+    let full_handle = archive_path
         .parent()
         .expect("could not read archive path")
         .join(&manifest.transactions);
@@ -66,7 +64,6 @@ pub async fn load_chunk(
 
     // TODO: for purposes of explorer/warehouse do we want to do the full tx restore controller verifications
 
-
     Ok(TransactionArchiveChunk {
         manifest,
         txns,

warehouse/Cargo.toml

@@ -12,12 +12,14 @@ version.workspace = true
 [dependencies]
 anyhow = { workspace = true }
 clap = { workspace = true }
-glob = { workspace = true }
 diem-types = { workspace = true }
+glob = { workspace = true }
 libra-backwards-compatibility = { workspace = true }
+libra-cached-packages = { workspace = true }
 libra-storage = { workspace =  true }
 libra-types = { workspace =  true }
 once_cell = { workspace =  true }
+serde_json = { workspace =  true }
 sqlx = { workspace = true }
 tokio = { workspace = true }
 

warehouse/src/extract_transactions.rs

@@ -1,7 +1,12 @@
 use anyhow::Result;
+use diem_types::transaction::SignedTransaction;
+use libra_cached_packages::libra_stdlib::EntryFunctionCall;
 use libra_storage::read_tx_chunk::{load_chunk, load_tx_chunk_manifest};
+use serde_json::json;
 use std::path::Path;
 
+use crate::table_structs::{WarehouseDepositTx, WarehouseGenericTx, WarehouseTxMeta};
+
 pub async fn extract_current_transactions(archive_path: &Path) -> Result<()> {
     let manifest_file = archive_path.join("transaction.manifest");
     assert!(
@@ -10,13 +15,69 @@ pub async fn extract_current_transactions(archive_path: &Path) -> Result<()> {
         &format!("transaction.manifest file not found at {:?}", archive_path)
     );
     let manifest = load_tx_chunk_manifest(&manifest_file)?;
+
+    let mut user_txs = 0;
+
     for each_chunk_manifest in manifest.chunks {
-      let deserialized = load_chunk(archive_path, each_chunk_manifest).await?;
+        let deserialized = load_chunk(archive_path, each_chunk_manifest).await?;
 
-      for tx in deserialized.txns {
-        dbg!(&tx);
-      }
+        for tx in deserialized.txns {
+            match tx {
+                diem_types::transaction::Transaction::UserTransaction(signed_transaction) => {
+                    // maybe_decode_entry_function(&signed_transaction);
+                    if let Ok(t) = try_decode_deposit_tx(&signed_transaction) {
+                        dbg!(&t);
+                    }
+                    user_txs += 1;
+                }
+                diem_types::transaction::Transaction::GenesisTransaction(_write_set_payload) => {}
+                diem_types::transaction::Transaction::BlockMetadata(_block_metadata) => {}
+                diem_types::transaction::Transaction::StateCheckpoint(_hash_value) => {}
+            }
+        }
     }
-
+    // TODO: use logging
+    println!("user transactions found in chunk: {}", user_txs);
     Ok(())
 }
+
+pub fn decode_tx_meta(user_tx: &SignedTransaction) -> Result<WarehouseTxMeta> {
+    let timestamp = user_tx.expiration_timestamp_secs();
+    let sender = user_tx.sender();
+
+    let raw = user_tx.raw_transaction_ref();
+    let p = raw.clone().into_payload().clone();
+    let ef = p.into_entry_function();
+    let module = ef.module().to_string();
+    let function = ef.function().to_string();
+
+    Ok(WarehouseTxMeta {
+        sender,
+        module,
+        function,
+        timestamp,
+    })
+}
+
+pub fn maybe_decode_generic_entry_function(
+    user_tx: &SignedTransaction,
+) -> Result<WarehouseGenericTx> {
+    Ok(WarehouseGenericTx {
+        meta: decode_tx_meta(user_tx)?,
+        args: json!(""),
+    })
+}
+
+pub fn try_decode_deposit_tx(user_tx: &SignedTransaction) -> Result<WarehouseDepositTx> {
+    let (to, amount) = match EntryFunctionCall::decode(user_tx.payload()) {
+        Some(EntryFunctionCall::OlAccountTransfer { to, amount }) => (to, amount),
+        // many variants
+        _ => anyhow::bail!("not a deposit tx"),
+    };
+
+    Ok(WarehouseDepositTx {
+        meta: decode_tx_meta(user_tx)?,
+        to,
+        amount,
+    })
+}

warehouse/src/lib.rs

@@ -1,10 +1,10 @@
 pub mod age_init;
 pub mod extract;
+pub mod extract_transactions;
 pub mod load_account;
 pub mod load_coin;
 pub mod migrate;
 pub mod query_balance;
 pub mod scan;
 pub mod table_structs;
 pub mod warehouse_cli;
-pub mod extract_transactions;

warehouse/src/table_structs.rs

@@ -24,7 +24,6 @@ impl WarehouseRecord {
         self.time.epoch = epoch;
     }
 }
-
 // holds timestamp, chain height, and epoch
 #[derive(Debug, Clone, Default)]
 pub struct WarehouseTime {
@@ -44,12 +43,23 @@ pub struct WarehouseBalance {
     pub balance: u64,
 }
 
-// #[derive(Debug, Default, Clone, FromRow)]
-// pub struct WarehouseBalanceAlt {
-//     // balances in v6+ terms
-//     #[sqlx(try_from = "i64")]
-//     pub balance: u64,
-//     // the balance pre v6 recast
-//     #[sqlx(default, try_from = "i64")]
-//     pub legacy_balance: Option<u64>,
-// }
+#[derive(Debug, Clone, FromRow)]
+pub struct WarehouseTxMeta {
+    pub sender: AccountAddress,
+    pub module: String,
+    pub function: String,
+    // TODO: we can only seem to access the tx expiration timestamp which the user defined.
+    pub timestamp: u64,
+}
+
+#[derive(Debug, Clone, FromRow)]
+pub struct WarehouseDepositTx {
+    pub meta: WarehouseTxMeta,
+    pub to: AccountAddress,
+    pub amount: u64,
+}
+#[derive(Debug, Clone, FromRow)]
+pub struct WarehouseGenericTx {
+    pub meta: WarehouseTxMeta,
+    pub args: serde_json::Value,
+}

warehouse/tests/support/fixtures.rs

@@ -29,7 +29,6 @@ pub fn v7_state_manifest_fixtures_path() -> PathBuf {
     dir
 }
 
-
 pub fn v7_tx_manifest_fixtures_path() -> PathBuf {
     let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .canonicalize()

warehouse/tests/test_extract_transactions.rs

@@ -3,8 +3,8 @@ use libra_warehouse::extract_transactions::extract_current_transactions;
 mod support;
 
 #[tokio::test]
-async fn test_extract_tx_from_archive() -> anyhow::Result<()>{
-  let archive_path = support::fixtures::v7_tx_manifest_fixtures_path();
-  let _tx = extract_current_transactions(&archive_path).await?;
-  Ok(())
+async fn test_extract_tx_from_archive() -> anyhow::Result<()> {
+    let archive_path = support::fixtures::v7_tx_manifest_fixtures_path();
+    extract_current_transactions(&archive_path).await?;
+    Ok(())
 }