Skip to content

Commit

Permalink
Optimize and refactor lookup_txos()
Browse files Browse the repository at this point in the history
- Avoid unnecessary copying of prev outpoints

- When looking for both mempool and on-chain txos, accumulate the set of
  outpoints that remain to be looked up to avoid re-checking for them
  later again in the found set

- Refactored lookup_txos() to use lookup_txo() internally rather than
  the other way around, which was less efficient

- Lookup txos in mempool first, then on-chain

- ChainQuery::lookup_txos() now returns a Result instead of panicking
  when outpoints are missing

- Removed ChainQuery::lookup_avail_txos() and allow_missing, which are
  no longer neceesary
  • Loading branch information
shesek committed May 22, 2024
1 parent 38c9576 commit a0d034d
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 44 deletions.
4 changes: 2 additions & 2 deletions src/bin/tx-fingerprint-stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,12 @@ fn main() {
//info!("{:?},{:?}", txid, blockid);

let prevouts = chain.lookup_txos(
&tx.input
tx.input
.iter()
.filter(|txin| has_prevout(txin))
.map(|txin| txin.previous_output)
.collect(),
);
).unwrap();

let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum();
let small_out = tx
Expand Down
37 changes: 16 additions & 21 deletions src/new_index/mempool.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use arraydeque::{ArrayDeque, Wrapping};
use itertools::Itertools;
use itertools::{Either, Itertools};

#[cfg(not(feature = "liquid"))]
use bitcoin::consensus::encode::serialize;
Expand Down Expand Up @@ -310,7 +310,7 @@ impl Mempool {
self.txstore.insert(txid, tx);
}
// Phase 2: index history and spend edges (can fail if some txos cannot be found)
let txos = match self.lookup_txos(&self.get_prevouts(&txids)) {
let txos = match self.lookup_txos(self.get_prevouts(&txids)) {
Ok(txos) => txos,
Err(err) => {
warn!("lookup txouts failed: {}", err);
Expand Down Expand Up @@ -397,34 +397,29 @@ impl Mempool {
}
}

pub fn lookup_txo(&self, outpoint: &OutPoint) -> Result<TxOut> {
let mut outpoints = BTreeSet::new();
outpoints.insert(*outpoint);
Ok(self.lookup_txos(&outpoints)?.remove(outpoint).unwrap())
fn lookup_txo(&self, outpoint: &OutPoint) -> Option<TxOut> {
self.txstore
.get(&outpoint.txid)
.and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
}

pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
pub fn lookup_txos(&self, outpoints: BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
let _timer = self
.latency
.with_label_values(&["lookup_txos"])
.start_timer();

let confirmed_txos = self.chain.lookup_avail_txos(outpoints);
// Get the txos available in the mempool, skipping over (and collecting) missing ones
let (mut txos, remain_outpoints): (HashMap<_, _>, _) = outpoints
.into_iter()
.partition_map(|outpoint| match self.lookup_txo(&outpoint) {
Some(txout) => Either::Left((outpoint, txout)),
None => Either::Right(outpoint),
});

let mempool_txos = outpoints
.iter()
.filter(|outpoint| !confirmed_txos.contains_key(outpoint))
.map(|outpoint| {
self.txstore
.get(&outpoint.txid)
.and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
.map(|txout| (*outpoint, txout))
.chain_err(|| format!("missing outpoint {:?}", outpoint))
})
.collect::<Result<HashMap<OutPoint, TxOut>>>()?;
// Get the remaining txos from the chain (fails if any are missing)
txos.extend(self.chain.lookup_txos(remain_outpoints)?);

let mut txos = confirmed_txos;
txos.extend(mempool_txos);
Ok(txos)
}

Expand Down
2 changes: 1 addition & 1 deletion src/new_index/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ impl Query {
.or_else(|| self.mempool().lookup_raw_txn(txid))
}

pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
pub fn lookup_txos(&self, outpoints: BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
// the mempool lookup_txos() internally looks up confirmed txos as well
self.mempool()
.lookup_txos(outpoints)
Expand Down
29 changes: 10 additions & 19 deletions src/new_index/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ impl Indexer {
fn index(&self, blocks: &[BlockEntry]) {
let previous_txos_map = {
let _timer = self.start_timer("index_lookup");
lookup_txos(&self.store.txstore_db, &get_previous_txos(blocks), false)
lookup_txos(&self.store.txstore_db, get_previous_txos(blocks)).unwrap()
};
let rows = {
let _timer = self.start_timer("index_process");
Expand Down Expand Up @@ -859,14 +859,9 @@ impl ChainQuery {
lookup_txo(&self.store.txstore_db, outpoint)
}

pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
pub fn lookup_txos(&self, outpoints: BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
let _timer = self.start_timer("lookup_txos");
lookup_txos(&self.store.txstore_db, outpoints, false)
}

pub fn lookup_avail_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
let _timer = self.start_timer("lookup_available_txos");
lookup_txos(&self.store.txstore_db, outpoints, true)
lookup_txos(&self.store.txstore_db, outpoints)
}

pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
Expand Down Expand Up @@ -1033,21 +1028,17 @@ fn get_previous_txos(block_entries: &[BlockEntry]) -> BTreeSet<OutPoint> {
.collect()
}

fn lookup_txos(
txstore_db: &DB,
outpoints: &BTreeSet<OutPoint>,
allow_missing: bool,
) -> HashMap<OutPoint, TxOut> {
let mut remain_outpoints = outpoints.iter();
fn lookup_txos(txstore_db: &DB, outpoints: BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
let keys = outpoints.iter().map(TxOutRow::key).collect::<Vec<_>>();
let mut remain_outpoints = outpoints.into_iter();
txstore_db
.multi_get(outpoints.iter().map(TxOutRow::key))
.multi_get(keys)
.into_iter()
.filter_map(|res| {
.map(|res| {
let outpoint = remain_outpoints.next().unwrap();
match res.unwrap() {
Some(txo) => Some((*outpoint, deserialize(&txo).expect("failed to parse TxOut"))),
None if allow_missing => None,
None => panic!("missing txo {}", outpoint),
Some(txo) => Ok((outpoint, deserialize(&txo).expect("failed to parse TxOut"))),
None => Err(format!("missing txo {}", outpoint).into()),
}
})
.collect()
Expand Down
2 changes: 1 addition & 1 deletion src/rest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ fn prepare_txs(
})
.collect();

let prevouts = query.lookup_txos(&outpoints);
let prevouts = query.lookup_txos(outpoints);

txs.into_iter()
.map(|(tx, blockid)| TransactionValue::new(tx, blockid, &prevouts, config))
Expand Down

0 comments on commit a0d034d

Please sign in to comment.