diff --git a/crates/chain/src/indexer/keychain_txout.rs b/crates/chain/src/indexer/keychain_txout.rs index 4c40ce7344..ccbaf635f9 100644 --- a/crates/chain/src/indexer/keychain_txout.rs +++ b/crates/chain/src/indexer/keychain_txout.rs @@ -5,7 +5,7 @@ use crate::{ alloc::boxed::Box, collections::*, miniscript::{Descriptor, DescriptorPublicKey}, - spk_client::{FullScanRequestBuilder, SyncRequestBuilder}, + spk_client::{FullScanRequestBuilder, ScanRequestBuilder, SyncRequestBuilder}, spk_iter::BIP32_MAX_INDEX, spk_txout::SpkTxOutIndex, DescriptorExt, DescriptorId, Indexed, Indexer, KeychainIndexed, SpkIterator, @@ -1134,6 +1134,55 @@ impl FullScanRequestBuilderExt for FullSca } } +/// Trait to extend [`ScanRequestBuilder`]. +pub trait ScanRequestBuilderExt { + /// Add keychains from `indexer` for discovery, starting after the last revealed + /// index of each keychain. + /// + /// Discovery starts at `last_revealed + 1` for each keychain. For wallets with + /// no revealed scripts, discovery starts at 0. Combine with + /// [`revealed_spks_from_indexer`](Self::revealed_spks_from_indexer) to cover the + /// full revealed and undiscovered range without overlap. + fn discover_from_indexer(self, indexer: &KeychainTxOutIndex) -> Self; + + /// Add already-revealed [`Script`]s from `indexer` for explicit sync. + fn revealed_spks_from_indexer(self, indexer: &KeychainTxOutIndex, spk_range: R) -> Self + where + R: core::ops::RangeBounds; + + /// Add unused [`Script`]s from `indexer` for explicit sync. + fn unused_spks_from_indexer(self, indexer: &KeychainTxOutIndex) -> Self; +} + +impl ScanRequestBuilderExt + for ScanRequestBuilder +{ + fn discover_from_indexer(mut self, indexer: &KeychainTxOutIndex) -> Self { + for (keychain, descriptor) in indexer.keychains() { + // Start discovery AFTER last revealed index — revealed scripts + // are handled by revealed_spks_from_indexer(). No overlap. + let start = indexer + .last_revealed_index(keychain.clone()) + .map(|i| i + 1) + .unwrap_or(0); + let spks = SpkIterator::new_with_range(descriptor.clone(), start..); + self = self.discover_keychain(keychain.clone(), spks); + } + self + } + + fn revealed_spks_from_indexer(self, indexer: &KeychainTxOutIndex, spk_range: R) -> Self + where + R: core::ops::RangeBounds, + { + self.spks_with_indexes(indexer.revealed_spks(spk_range)) + } + + fn unused_spks_from_indexer(self, indexer: &KeychainTxOutIndex) -> Self { + self.spks_with_indexes(indexer.unused_spks()) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/core/src/spk_client.rs b/crates/core/src/spk_client.rs index c9cabc3e6c..2438019be6 100644 --- a/crates/core/src/spk_client.rs +++ b/crates/core/src/spk_client.rs @@ -10,6 +10,8 @@ type InspectSync = dyn FnMut(SyncItem, SyncProgress) + Send + 'static; type InspectFullScan = dyn FnMut(K, u32, &Script) + Send + 'static; +const DEFAULT_STOP_GAP: usize = 20; + /// An item reported to the [`inspect`](SyncRequestBuilder::inspect) closure of [`SyncRequest`]. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SyncItem<'i, I> { @@ -136,9 +138,6 @@ impl SyncRequestBuilder { /// /// # Example /// - /// Sync revealed script pubkeys obtained from a - /// [`KeychainTxOutIndex`](https://docs.rs/bdk_chain/latest/bdk_chain/indexer/keychain_txout/struct.KeychainTxOutIndex.html). - /// /// ```rust /// # use bdk_chain::bitcoin::BlockHash; /// # use bdk_chain::spk_client::SyncRequest; @@ -666,3 +665,634 @@ impl Iterator for SyncIter<'_, I, D, OutPoint> { (remaining, Some(remaining)) } } + +/// An item reported to the [`inspect`](ScanRequestBuilder::inspect) closure of [`ScanRequest`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ScanItem<'i, K, I> { + /// A keychain discovery script being scanned. + Discovery(K, u32, &'i Script), + /// A script being synced. + Spk(I, &'i Script), + /// A txid being synced. + Txid(Txid), + /// An outpoint being synced. + OutPoint(OutPoint), +} + +impl core::fmt::Display + for ScanItem<'_, K, I> +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + ScanItem::Discovery(k, i, spk) => { + if (k as &dyn core::any::Any).is::<()>() { + write!(f, "discovery script #{i} '{spk}'") + } else { + write!(f, "discovery script {k:?}#{i} '{spk}'") + } + } + ScanItem::Spk(i, spk) => { + if (i as &dyn core::any::Any).is::<()>() { + write!(f, "script '{spk}'") + } else { + write!(f, "script {i:?} '{spk}'") + } + } + ScanItem::Txid(txid) => write!(f, "txid '{txid}'"), + ScanItem::OutPoint(op) => write!(f, "outpoint '{op}'"), + } + } +} + +/// Progress of a [`ScanRequest`]. +/// +/// Discovery progress only tracks consumed count (remaining is unknown because they are unbounded). +/// Explicit sync progress tracks both consumed and remaining. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ScanProgress { + /// Scripts consumed by keychain discovery. + pub discovery_consumed: usize, + /// Script pubkeys consumed by explicit sync. + pub spks_consumed: usize, + /// Script pubkeys remaining in explicit sync. + pub spks_remaining: usize, + /// Txids consumed by explicit sync. + pub txids_consumed: usize, + /// Txids remaining in explicit sync. + pub txids_remaining: usize, + /// Outpoints consumed by explicit sync. + pub outpoints_consumed: usize, + /// Outpoints remaining in explicit sync. + pub outpoints_remaining: usize, +} + +impl ScanProgress { + /// Total explicit sync items (consumed + remaining). + pub fn explicit_total(&self) -> usize { + self.explicit_consumed() + self.explicit_remaining() + } + + /// Total explicit sync items consumed. + pub fn explicit_consumed(&self) -> usize { + self.spks_consumed + self.txids_consumed + self.outpoints_consumed + } + + /// Total explicit sync items remaining. + pub fn explicit_remaining(&self) -> usize { + self.spks_remaining + self.txids_remaining + self.outpoints_remaining + } + + /// Number of discovery scripts consumed so far. + /// + /// Unlike explicit progress, discovery has no total or remaining count + /// because discovery iterators are unbounded. + pub fn discovery_consumed(&self) -> usize { + self.discovery_consumed + } +} + +type InspectScan = dyn FnMut(ScanItem, ScanProgress) + Send + 'static; + +struct CountedQueue { + items: VecDeque, + consumed: usize, +} + +impl Default for CountedQueue { + fn default() -> Self { + Self { + items: Default::default(), + consumed: 0, + } + } +} + +impl CountedQueue { + fn extend(&mut self, items: impl IntoIterator) { + self.items.extend(items); + } + + fn len(&self) -> usize { + self.items.len() + } + + fn consumed(&self) -> usize { + self.consumed + } + + fn pop_front(&mut self) -> Option { + let item = self.items.pop_front()?; + self.consumed += 1; + Some(item) + } +} + +struct DiscoveryState { + stop_gap: usize, + spks_by_keychain: BTreeMap> + Send>>, + consumed: usize, +} + +impl Default for DiscoveryState { + fn default() -> Self { + Self { + stop_gap: DEFAULT_STOP_GAP, + spks_by_keychain: Default::default(), + consumed: 0, + } + } +} + +struct ExplicitSyncState { + spks: CountedQueue<(I, ScriptBuf)>, + spk_expected_txids: HashMap>, + txids: CountedQueue, + outpoints: CountedQueue, +} + +impl Default for ExplicitSyncState { + fn default() -> Self { + Self { + spks: Default::default(), + spk_expected_txids: Default::default(), + txids: Default::default(), + outpoints: Default::default(), + } + } +} + +/// Builds a [`ScanRequest`]. +/// +/// Construct with [`ScanRequest::builder`]. +#[must_use] +pub struct ScanRequestBuilder { + inner: ScanRequest, +} + +impl ScanRequestBuilder { + /// Set the stop gap for keychain discovery. Default is 20. + /// + /// The stop gap controls how many consecutive unused scripts must be found before + /// stopping discovery for a keychain. It applies only to the discovery portion of a + /// [`ScanRequest`] and has no effect when no keychains are added with + /// [`discover_keychain`](Self::discover_keychain). + pub fn stop_gap(mut self, stop_gap: usize) -> Self { + self.inner.discovery.stop_gap = stop_gap; + self + } + + /// Add a keychain for discovery. + /// + /// The `spks` iterator provides scripts to scan for the given `keychain`. Discovery + /// continues until `stop_gap` consecutive unused scripts are found. + pub fn discover_keychain( + mut self, + keychain: K, + spks: impl IntoIterator> + Send + 'static>, + ) -> Self { + self.inner + .discovery + .spks_by_keychain + .insert(keychain, Box::new(spks.into_iter())); + self + } +} + +impl ScanRequestBuilder { + /// Add [`Script`]s to scan. + pub fn spks(self, spks: impl IntoIterator) -> Self { + self.spks_with_indexes(spks.into_iter().map(|spk| ((), spk))) + } +} + +impl ScanRequestBuilder { + /// Set the initial chain tip for the scan request. + /// + /// This is used to update [`LocalChain`](../../bdk_chain/local_chain/struct.LocalChain.html). + pub fn chain_tip(mut self, cp: CheckPoint) -> Self { + self.inner.chain_tip = Some(cp); + self + } + + /// Add [`Script`]s with associated indexes to scan. + /// + /// # Example + /// + /// Sync revealed script pubkeys obtained from a + /// [`KeychainTxOutIndex`](https://docs.rs/bdk_chain/latest/bdk_chain/indexer/keychain_txout/struct.KeychainTxOutIndex.html). + /// + /// ```rust + /// # use bdk_chain::bitcoin::BlockHash; + /// # use bdk_chain::spk_client::ScanRequest; + /// # use bdk_chain::indexer::keychain_txout::KeychainTxOutIndex; + /// # use bdk_chain::miniscript::{Descriptor, DescriptorPublicKey}; + /// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only(); + /// # let (descriptor_a,_) = Descriptor::::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap(); + /// # let (descriptor_b,_) = Descriptor::::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap(); + /// let mut indexer = KeychainTxOutIndex::<&'static str>::default(); + /// indexer.insert_descriptor("descriptor_a", descriptor_a)?; + /// indexer.insert_descriptor("descriptor_b", descriptor_b)?; + /// + /// /* Assume that the caller does more mutations to the `indexer` here... */ + /// + /// // Reveal spks for "descriptor_a", then build a scan request. Each spk will be indexed with + /// // `u32`, which represents the derivation index of the associated spk from "descriptor_a". + /// let (newly_revealed_spks, _changeset) = indexer + /// .reveal_to_target("descriptor_a", 21) + /// .expect("keychain must exist"); + /// let _request: ScanRequest<(), u32, BlockHash> = ScanRequest::builder() + /// .spks_with_indexes(newly_revealed_spks) + /// .build(); + /// + /// // Sync all revealed spks in the indexer. This time, spks may be derived from different + /// // keychains. Each spk will be indexed with `(&str, u32)` where `&str` is the keychain + /// // identifier and `u32` is the derivation index. + /// let all_revealed_spks = indexer.revealed_spks(..); + /// let _request: ScanRequest<(), (&str, u32), BlockHash> = ScanRequest::builder() + /// .spks_with_indexes(all_revealed_spks) + /// .build(); + /// # Ok::<_, bdk_chain::keychain_txout::InsertDescriptorError<_>>(()) + /// ``` + pub fn spks_with_indexes(mut self, spks: impl IntoIterator) -> Self { + self.inner.explicit.spks.extend(spks); + self + } + + /// Add transactions that are expected to exist under the given spks. + /// + /// This is useful for detecting a malicious replacement of an incoming transaction. + /// It only affects the explicit script queue added with `spks` or + /// [`spks_with_indexes`](Self::spks_with_indexes). It does not affect discovery scripts added + /// with [`discover_keychain`](Self::discover_keychain). + pub fn expected_spk_txids(mut self, txs: impl IntoIterator) -> Self { + for (spk, txid) in txs { + self.inner + .explicit + .spk_expected_txids + .entry(spk) + .or_default() + .insert(txid); + } + self + } + + /// Add [`Txid`]s to scan. + pub fn txids(mut self, txids: impl IntoIterator) -> Self { + self.inner.explicit.txids.extend(txids); + self + } + + /// Add [`OutPoint`]s to scan. + pub fn outpoints(mut self, outpoints: impl IntoIterator) -> Self { + self.inner.explicit.outpoints.extend(outpoints); + self + } + + /// Set the closure that will inspect every scan item visited. + pub fn inspect(mut self, inspect: F) -> Self + where + F: FnMut(ScanItem, ScanProgress) + Send + 'static, + { + self.inner.inspect = Box::new(inspect); + self + } + + /// Build the [`ScanRequest`]. + pub fn build(self) -> ScanRequest { + self.inner + } +} + +/// Data required to perform a spk-based blockchain client scan. +/// +/// A client scan combines keychain discovery with scanning known scripts, transaction ids, and +/// outpoints. During discovery, the client iterates over the scripts for each keychain and stops +/// after encountering `stop_gap` consecutive scripts with no relevant history. The explicit +/// portion scans a known set of scripts, transaction ids, and outpoints for relevant chain data. +/// The scan can also update the chain from the given [`chain_tip`](ScanRequestBuilder::chain_tip), +/// if provided. +/// +/// # Example +/// +/// ```rust +/// # use bdk_core::spk_client::ScanRequest; +/// # use bdk_chain::{bitcoin::hashes::Hash, local_chain::LocalChain}; +/// # use bdk_core::bitcoin::ScriptBuf; +/// # let (local_chain, _) = LocalChain::from_genesis(Hash::all_zeros()); +/// # let discovery_spk = ScriptBuf::default(); +/// # let explicit_spk = ScriptBuf::default(); +/// // Build a scan request with both keychain discovery and explicit scripts. +/// let _scan_request: ScanRequest = ScanRequest::builder() +/// .chain_tip(local_chain.tip()) +/// .stop_gap(2) +/// .discover_keychain((), [(0, discovery_spk)]) +/// .spks([explicit_spk]) +/// .build(); +/// ``` +#[must_use] +pub struct ScanRequest { + start_time: u64, + chain_tip: Option>, + discovery: DiscoveryState, + explicit: ExplicitSyncState, + inspect: Box>, +} + +impl From> for ScanRequest { + fn from(builder: ScanRequestBuilder) -> Self { + builder.inner + } +} + +impl ScanRequest { + /// Start building a [`ScanRequest`] with a given `start_time`. + /// + /// `start_time` specifies the start time of scan. Chain sources can use this value to set + /// [`TxUpdate::seen_ats`](crate::TxUpdate::seen_ats) for mempool transactions. A transaction + /// without any `seen_ats` is assumed to be unseen in the mempool. + /// + /// Use [`ScanRequest::builder`] to use the current timestamp as `start_time` (this requires + /// `feature = "std"`). + pub fn builder_at(start_time: u64) -> ScanRequestBuilder { + ScanRequestBuilder { + inner: Self { + start_time, + chain_tip: None, + discovery: Default::default(), + explicit: Default::default(), + inspect: Box::new(|_, _| ()), + }, + } + } + + /// Start building a [`ScanRequest`] with the current timestamp as the `start_time`. + /// + /// Use [`ScanRequest::builder_at`] to manually set the `start_time`, or if `feature = "std"` + /// is not available. + #[cfg(feature = "std")] + #[cfg_attr(docsrs, doc(cfg(feature = "std")))] + pub fn builder() -> ScanRequestBuilder { + let start_time = std::time::UNIX_EPOCH + .elapsed() + .expect("failed to get current timestamp") + .as_secs(); + Self::builder_at(start_time) + } + + /// When the scan request was initiated. + pub fn start_time(&self) -> u64 { + self.start_time + } + + /// Get the chain tip [`CheckPoint`] of this request (if any). + pub fn chain_tip(&self) -> Option> { + self.chain_tip.clone() + } + + /// Get the stop gap for keychain discovery. + pub fn stop_gap(&self) -> usize { + self.discovery.stop_gap + } + + /// List all keychains registered for discovery. + pub fn keychains(&self) -> Vec { + self.discovery.spks_by_keychain.keys().cloned().collect() + } + + /// Advances the scan request and returns the next discovered indexed [`ScriptBuf`] of the + /// given `keychain`. + /// + /// Returns [`None`] when there are no more scripts for the keychain. + pub fn next_discovery_spk(&mut self, keychain: K) -> Option> { + self.iter_discovery_spks(keychain).next() + } + + /// Iterate over discovered indexed [`ScriptBuf`]s for the given `keychain`. + pub fn iter_discovery_spks( + &mut self, + keychain: K, + ) -> impl Iterator> + '_ { + let spks_consumed = self.explicit.spks.consumed(); + let spks_remaining = self.explicit.spks.len(); + let txids_consumed = self.explicit.txids.consumed(); + let txids_remaining = self.explicit.txids.len(); + let outpoints_consumed = self.explicit.outpoints.consumed(); + let outpoints_remaining = self.explicit.outpoints.len(); + let spks = self.discovery.spks_by_keychain.get_mut(&keychain); + DiscoveryKeychainSpkIter { + keychain, + spks, + inspect: &mut self.inspect, + discovery_consumed: &mut self.discovery.consumed, + spks_consumed, + spks_remaining, + txids_consumed, + txids_remaining, + outpoints_consumed, + outpoints_remaining, + } + } + + /// Get the [`ScanProgress`] of this request. + pub fn progress(&self) -> ScanProgress { + ScanProgress { + discovery_consumed: self.discovery.consumed, + spks_consumed: self.explicit.spks.consumed(), + spks_remaining: self.explicit.spks.len(), + txids_consumed: self.explicit.txids.consumed(), + txids_remaining: self.explicit.txids.len(), + outpoints_consumed: self.explicit.outpoints.consumed(), + outpoints_remaining: self.explicit.outpoints.len(), + } + } + + /// Advances the scan request and returns the next [`ScriptBuf`] with corresponding [`Txid`] + /// history from the explicit sync queue. + /// + /// Returns [`None`] when there are no more scripts remaining. + pub fn next_spk_with_expected_txids(&mut self) -> Option { + let (i, next_spk) = self.explicit.spks.pop_front()?; + self._call_inspect(ScanItem::Spk(i, next_spk.as_script())); + let spk_history = self + .explicit + .spk_expected_txids + .get(&next_spk) + .cloned() + .unwrap_or_default(); + Some(SpkWithExpectedTxids { + spk: next_spk, + expected_txids: spk_history, + }) + } + + /// Advances the scan request and returns the next [`Txid`]. + /// + /// Returns [`None`] when there are no more txids remaining. + pub fn next_txid(&mut self) -> Option { + let txid = self.explicit.txids.pop_front()?; + self._call_inspect(ScanItem::Txid(txid)); + Some(txid) + } + + /// Advances the scan request and returns the next [`OutPoint`]. + /// + /// Returns [`None`] when there are no more outpoints remaining. + pub fn next_outpoint(&mut self) -> Option { + let outpoint = self.explicit.outpoints.pop_front()?; + self._call_inspect(ScanItem::OutPoint(outpoint)); + Some(outpoint) + } + + /// Iterate over [`ScriptBuf`]s with corresponding [`Txid`] histories in the explicit sync + /// queue. + pub fn iter_spks_with_expected_txids( + &mut self, + ) -> impl ExactSizeIterator + '_ { + ExplicitIter::::new(self) + } + + /// Iterate over [`Txid`]s in the explicit sync queue. + pub fn iter_txids(&mut self) -> impl ExactSizeIterator + '_ { + ExplicitIter::::new(self) + } + + /// Iterate over [`OutPoint`]s in the explicit sync queue. + pub fn iter_outpoints(&mut self) -> impl ExactSizeIterator + '_ { + ExplicitIter::::new(self) + } + + fn _call_inspect(&mut self, item: ScanItem) { + let progress = self.progress(); + (*self.inspect)(item, progress); + } +} + +/// Data returned from a unified spk-based blockchain client scan. +/// +/// See also [`ScanRequest`]. +#[must_use] +#[derive(Debug)] +pub struct ScanResponse { + /// Relevant transaction data discovered during the scan. + pub tx_update: crate::TxUpdate, + /// Last active indices per keychain (`K`) found during discovery. An index is active if its + /// script pubkey had an associated transaction. Explicit-sync hits are not included. + pub last_active_indices: BTreeMap, + /// Changes to the chain discovered during the scan. + pub chain_update: Option>, +} + +impl Default for ScanResponse { + fn default() -> Self { + Self { + tx_update: Default::default(), + last_active_indices: Default::default(), + chain_update: Default::default(), + } + } +} + +impl ScanResponse { + /// Returns true if the `ScanResponse` is empty. + pub fn is_empty(&self) -> bool { + self.tx_update.is_empty() + && self.last_active_indices.is_empty() + && self.chain_update.is_none() + } +} + +/// Iterator over discovery scripts for a single keychain. +struct DiscoveryKeychainSpkIter<'r, K, I> { + keychain: K, + spks: Option<&'r mut Box> + Send>>, + inspect: &'r mut Box>, + discovery_consumed: &'r mut usize, + spks_consumed: usize, + spks_remaining: usize, + txids_consumed: usize, + txids_remaining: usize, + outpoints_consumed: usize, + outpoints_remaining: usize, +} + +impl Iterator for DiscoveryKeychainSpkIter<'_, K, I> { + type Item = Indexed; + + fn next(&mut self) -> Option { + let (i, spk) = self.spks.as_mut()?.next()?; + *self.discovery_consumed += 1; + let progress = ScanProgress { + discovery_consumed: *self.discovery_consumed, + spks_consumed: self.spks_consumed, + spks_remaining: self.spks_remaining, + txids_consumed: self.txids_consumed, + txids_remaining: self.txids_remaining, + outpoints_consumed: self.outpoints_consumed, + outpoints_remaining: self.outpoints_remaining, + }; + (*self.inspect)( + ScanItem::Discovery(self.keychain.clone(), i, &spk), + progress, + ); + Some((i, spk)) + } +} + +/// Iterator over explicit scan items. +struct ExplicitIter<'r, K, I, D, Item> { + request: &'r mut ScanRequest, + marker: core::marker::PhantomData, +} + +impl<'r, K, I, D, Item> ExplicitIter<'r, K, I, D, Item> { + fn new(request: &'r mut ScanRequest) -> Self { + Self { + request, + marker: core::marker::PhantomData, + } + } +} + +impl<'r, K, I, D, Item> ExactSizeIterator for ExplicitIter<'r, K, I, D, Item> where + ExplicitIter<'r, K, I, D, Item>: Iterator +{ +} + +impl Iterator for ExplicitIter<'_, K, I, D, SpkWithExpectedTxids> { + type Item = SpkWithExpectedTxids; + + fn next(&mut self) -> Option { + self.request.next_spk_with_expected_txids() + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.request.explicit.spks.len(); + (remaining, Some(remaining)) + } +} + +impl Iterator for ExplicitIter<'_, K, I, D, Txid> { + type Item = Txid; + + fn next(&mut self) -> Option { + self.request.next_txid() + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.request.explicit.txids.len(); + (remaining, Some(remaining)) + } +} + +impl Iterator for ExplicitIter<'_, K, I, D, OutPoint> { + type Item = OutPoint; + + fn next(&mut self) -> Option { + self.request.next_outpoint() + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.request.explicit.outpoints.len(); + (remaining, Some(remaining)) + } +} diff --git a/crates/core/tests/test_spk_client.rs b/crates/core/tests/test_spk_client.rs index 38766ac31d..f8adc1bb0f 100644 --- a/crates/core/tests/test_spk_client.rs +++ b/crates/core/tests/test_spk_client.rs @@ -1,4 +1,45 @@ -use bdk_core::spk_client::{FullScanResponse, SyncResponse}; +use bdk_core::bitcoin::hashes::Hash; +use bdk_core::bitcoin::{OutPoint, ScriptBuf, Txid}; +use bdk_core::spk_client::{ + FullScanResponse, ScanItem, ScanProgress, ScanRequest, ScanResponse, SyncResponse, +}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum ObservedItem { + Discovery(u32, u32, ScriptBuf), + Spk(u32, ScriptBuf), + Txid(Txid), + OutPoint(OutPoint), +} + +impl From> for ObservedItem { + fn from(item: ScanItem<'_, u32, u32>) -> Self { + match item { + ScanItem::Discovery(keychain, index, spk) => { + Self::Discovery(keychain, index, spk.to_owned()) + } + ScanItem::Spk(index, spk) => Self::Spk(index, spk.to_owned()), + ScanItem::Txid(txid) => Self::Txid(txid), + ScanItem::OutPoint(outpoint) => Self::OutPoint(outpoint), + } + } +} + +type ObservedLog = Arc>>; + +/// Build a shared event log and the matching `inspect` closure for `ScanRequestBuilder`. +fn recorder() -> ( + ObservedLog, + impl FnMut(ScanItem<'_, u32, u32>, ScanProgress) + Send + 'static, +) { + let observed = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&observed); + let inspect = move |item: ScanItem<'_, u32, u32>, progress| { + captured.lock().unwrap().push((item.into(), progress)); + }; + (observed, inspect) +} #[test] fn test_empty() { @@ -10,4 +51,252 @@ fn test_empty() { SyncResponse::<()>::default().is_empty(), "Default `SyncResponse` must be empty" ); + assert!( + ScanResponse::<(), ()>::default().is_empty(), + "Default `ScanResponse` must be empty" + ); +} + +#[test] +fn test_scan_request_with_explicit_items() { + let spks = [ + ScriptBuf::from_bytes(vec![1, 2, 3]), + ScriptBuf::from_bytes(vec![4, 5, 6]), + ]; + let txid = Txid::from_byte_array([0xaa; 32]); + let outpoint = OutPoint::new(txid, 0); + + let mut scan: ScanRequest = ScanRequest::builder() + .spks(spks.clone()) + .txids([txid]) + .outpoints([outpoint]) + .build(); + + assert!(scan.keychains().is_empty()); + assert_eq!(scan.stop_gap(), 20); + + let progress = scan.progress(); + assert_eq!(progress.explicit_total(), spks.len() + 2); + assert_eq!(progress.explicit_consumed(), 0); + assert_eq!(progress.discovery_consumed(), 0); + + assert_eq!(scan.next_spk_with_expected_txids().unwrap().spk, spks[0]); + assert_eq!(scan.next_spk_with_expected_txids().unwrap().spk, spks[1]); + assert!(scan.next_spk_with_expected_txids().is_none()); + + assert_eq!(scan.next_txid().unwrap(), txid); + assert!(scan.next_txid().is_none()); + + assert_eq!(scan.next_outpoint().unwrap(), outpoint); + assert!(scan.next_outpoint().is_none()); + + let progress = scan.progress(); + assert_eq!(progress.explicit_consumed(), spks.len() + 2); + assert_eq!(progress.explicit_remaining(), 0); +} + +#[test] +fn test_scan_request_with_discovery() { + let discovery_spks: Vec<(u32, ScriptBuf)> = (0u32..5) + .map(|i| (i, ScriptBuf::from_bytes(vec![i as u8]))) + .collect(); + + let mut scan = ScanRequest::::builder() + .stop_gap(3) + .discover_keychain(0, discovery_spks.clone()) + .build(); + + assert_eq!(scan.stop_gap(), 3); + assert_eq!(scan.keychains(), vec![0]); + assert_eq!(scan.progress().explicit_total(), 0); + assert_eq!(scan.progress().discovery_consumed(), 0); + + let discovered: Vec<_> = scan.iter_discovery_spks(0).collect(); + assert_eq!(discovered, discovery_spks); + + assert_eq!(scan.progress().discovery_consumed(), discovery_spks.len()); +} + +#[test] +fn test_scan_inspect_explicit_items() { + let spks = [ + (10, ScriptBuf::from_bytes(vec![1, 2, 3])), + (11, ScriptBuf::from_bytes(vec![4, 5, 6])), + ]; + let txid = Txid::from_byte_array([0xaa; 32]); + let outpoint = OutPoint::new(txid, 1); + let (observed, inspect) = recorder(); + + let mut scan = ScanRequest::::builder() + .spks_with_indexes(spks.clone()) + .txids([txid]) + .outpoints([outpoint]) + .inspect(inspect) + .build(); + + assert_eq!(scan.next_spk_with_expected_txids().unwrap().spk, spks[0].1); + assert_eq!(scan.next_spk_with_expected_txids().unwrap().spk, spks[1].1); + assert_eq!(scan.next_txid().unwrap(), txid); + assert_eq!(scan.next_outpoint().unwrap(), outpoint); + + assert_eq!( + *observed.lock().unwrap(), + vec![ + ( + ObservedItem::Spk(spks[0].0, spks[0].1.clone()), + ScanProgress { + spks_consumed: 1, + spks_remaining: 1, + txids_remaining: 1, + outpoints_remaining: 1, + ..Default::default() + }, + ), + ( + ObservedItem::Spk(spks[1].0, spks[1].1.clone()), + ScanProgress { + spks_consumed: 2, + txids_remaining: 1, + outpoints_remaining: 1, + ..Default::default() + }, + ), + ( + ObservedItem::Txid(txid), + ScanProgress { + spks_consumed: 2, + txids_consumed: 1, + outpoints_remaining: 1, + ..Default::default() + }, + ), + ( + ObservedItem::OutPoint(outpoint), + ScanProgress { + spks_consumed: 2, + txids_consumed: 1, + outpoints_consumed: 1, + ..Default::default() + }, + ), + ] + ); +} + +#[test] +fn test_scan_inspect_discovery_items() { + let discovery_spks = [ + (0, ScriptBuf::from_bytes(vec![0])), + (1, ScriptBuf::from_bytes(vec![1])), + (2, ScriptBuf::from_bytes(vec![2])), + ]; + let (observed, inspect) = recorder(); + + let mut scan = ScanRequest::::builder() + .stop_gap(3) + .discover_keychain(7, discovery_spks.clone()) + .inspect(inspect) + .build(); + + for (expected_index, expected_spk) in &discovery_spks { + let (index, spk) = scan.next_discovery_spk(7).unwrap(); + assert_eq!(index, *expected_index); + assert_eq!(spk, *expected_spk); + } + + assert_eq!( + *observed.lock().unwrap(), + vec![ + ( + ObservedItem::Discovery(7, 0, discovery_spks[0].1.clone()), + ScanProgress { + discovery_consumed: 1, + ..Default::default() + }, + ), + ( + ObservedItem::Discovery(7, 1, discovery_spks[1].1.clone()), + ScanProgress { + discovery_consumed: 2, + ..Default::default() + }, + ), + ( + ObservedItem::Discovery(7, 2, discovery_spks[2].1.clone()), + ScanProgress { + discovery_consumed: 3, + ..Default::default() + }, + ), + ] + ); +} + +#[test] +fn test_scan_inspect_mixed_request() { + let discovery_spks = [ + (0, ScriptBuf::from_bytes(vec![0])), + (1, ScriptBuf::from_bytes(vec![1])), + ]; + let explicit_spk = (42, ScriptBuf::from_bytes(vec![9, 9, 9])); + let txid = Txid::from_byte_array([0xbb; 32]); + let (observed, inspect) = recorder(); + + let mut scan = ScanRequest::::builder() + .stop_gap(2) + .discover_keychain(7, discovery_spks.clone()) + .spks_with_indexes([explicit_spk.clone()]) + .txids([txid]) + .inspect(inspect) + .build(); + + assert_eq!(scan.next_discovery_spk(7).unwrap(), discovery_spks[0]); + assert_eq!( + scan.next_spk_with_expected_txids().unwrap().spk, + explicit_spk.1 + ); + assert_eq!(scan.next_txid().unwrap(), txid); + assert_eq!(scan.next_discovery_spk(7).unwrap(), discovery_spks[1]); + + assert_eq!( + *observed.lock().unwrap(), + vec![ + ( + ObservedItem::Discovery(7, 0, discovery_spks[0].1.clone()), + ScanProgress { + discovery_consumed: 1, + spks_remaining: 1, + txids_remaining: 1, + ..Default::default() + }, + ), + ( + ObservedItem::Spk(explicit_spk.0, explicit_spk.1.clone()), + ScanProgress { + discovery_consumed: 1, + spks_consumed: 1, + txids_remaining: 1, + ..Default::default() + }, + ), + ( + ObservedItem::Txid(txid), + ScanProgress { + discovery_consumed: 1, + spks_consumed: 1, + txids_consumed: 1, + ..Default::default() + }, + ), + ( + ObservedItem::Discovery(7, 1, discovery_spks[1].1.clone()), + ScanProgress { + discovery_consumed: 2, + spks_consumed: 1, + txids_consumed: 1, + ..Default::default() + }, + ), + ] + ); } diff --git a/crates/electrum/src/bdk_electrum_client.rs b/crates/electrum/src/bdk_electrum_client.rs index 25da3998ad..de834eb8b7 100644 --- a/crates/electrum/src/bdk_electrum_client.rs +++ b/crates/electrum/src/bdk_electrum_client.rs @@ -2,7 +2,8 @@ use bdk_core::{ bitcoin::{block::Header, BlockHash, OutPoint, Transaction, Txid}, collections::{BTreeMap, HashMap, HashSet}, spk_client::{ - FullScanRequest, FullScanResponse, SpkWithExpectedTxids, SyncRequest, SyncResponse, + FullScanRequest, FullScanResponse, ScanRequest, ScanResponse, SpkWithExpectedTxids, + SyncRequest, SyncResponse, }, BlockId, CheckPoint, ConfirmationBlockTime, TxUpdate, }; @@ -262,6 +263,115 @@ impl BdkElectrumClient { }) } + /// Scan keychain scripts, explicit scripts, txids, and/or outpoints with the blockchain (via + /// an Electrum client) and returns updates for [`bdk_chain`] data structures. + /// + /// - `request`: struct with data required to perform a spk-based blockchain client scan, see + /// [`ScanRequest`]. Keychain discovery stops after a gap of `request.stop_gap()` script + /// pubkeys with no associated transactions. + /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch + /// request. + /// - `fetch_prev_txouts`: specifies whether we want previous `TxOut`s for fee calculation. Note + /// that this requires additional calls to the Electrum server, but is necessary for + /// calculating the fee on a transaction if your wallet does not own the inputs. Methods like + /// [`Wallet.calculate_fee`] and [`Wallet.calculate_fee_rate`] will return a + /// [`CalculateFeeError::MissingTxOut`] error if those `TxOut`s are not present in the + /// transaction graph. + /// + /// [`bdk_chain`]: ../bdk_chain/index.html + /// [`CalculateFeeError::MissingTxOut`]: ../bdk_chain/tx_graph/enum.CalculateFeeError.html#variant.MissingTxOut + /// [`Wallet.calculate_fee`]: ../bdk_wallet/struct.Wallet.html#method.calculate_fee + /// [`Wallet.calculate_fee_rate`]: ../bdk_wallet/struct.Wallet.html#method.calculate_fee_rate + pub fn scan( + &self, + request: impl Into>, + batch_size: usize, + fetch_prev_txouts: bool, + ) -> Result, Error> { + let mut request: ScanRequest = request.into(); + let start_time = request.start_time(); + + let tip_and_latest_blocks = match request.chain_tip() { + Some(chain_tip) => Some(fetch_tip_and_latest_blocks(&self.inner, chain_tip)?), + None => None, + }; + + let mut tx_update = TxUpdate::::default(); + let mut last_active_indices = BTreeMap::::default(); + let mut pending_anchors = Vec::new(); + + // Discovery: scan keychain spks with stop_gap. + // Treat stop_gap = 0 as 1 to match esplora semantics (see fetch_txs_with_keychain_spks). + let stop_gap = request.stop_gap().max(1); + for keychain in request.keychains() { + let spks = request + .iter_discovery_spks(keychain.clone()) + .map(|(spk_i, spk)| (spk_i, SpkWithExpectedTxids::from(spk))); + if let Some(last_active_index) = self.populate_with_spks( + start_time, + &mut tx_update, + spks, + stop_gap, + batch_size, + &mut pending_anchors, + )? { + last_active_indices.insert(keychain, last_active_index); + } + } + + // Explicit sync: spks, txids, outpoints + self.populate_with_spks( + start_time, + &mut tx_update, + request + .iter_spks_with_expected_txids() + .enumerate() + .map(|(i, spk)| (i as u32, spk)), + usize::MAX, + batch_size, + &mut pending_anchors, + )?; + self.populate_with_txids( + start_time, + &mut tx_update, + request.iter_txids(), + &mut pending_anchors, + )?; + self.populate_with_outpoints( + start_time, + &mut tx_update, + request.iter_outpoints(), + &mut pending_anchors, + )?; + + // Fetch previous TxOuts for fee calculation if flag is enabled. + if fetch_prev_txouts { + self.fetch_prev_txout(&mut tx_update)?; + } + + if !pending_anchors.is_empty() { + let anchors = self.batch_fetch_anchors(&pending_anchors)?; + for (txid, anchor) in anchors { + tx_update.anchors.insert((anchor, txid)); + } + } + + let chain_update = match tip_and_latest_blocks { + Some((chain_tip, latest_blocks)) => Some(chain_update( + chain_tip, + &latest_blocks, + tx_update.anchors.iter().cloned(), + )?), + None => None, + }; + + Ok(ScanResponse { + tx_update, + chain_update, + last_active_indices, + }) + } + /// Populate the `tx_update` with transactions/anchors associated with the given `spks`. /// /// Transactions that contains an output with requested spk, or spends form an output with diff --git a/crates/electrum/src/lib.rs b/crates/electrum/src/lib.rs index 9c1d9f452c..34849f3685 100644 --- a/crates/electrum/src/lib.rs +++ b/crates/electrum/src/lib.rs @@ -1,19 +1,20 @@ //! This crate is used for returning updates from Electrum servers. //! -//! Updates are returned as either a [`SyncResponse`] (if [`BdkElectrumClient::sync()`] is called), -//! or a [`FullScanResponse`] (if [`BdkElectrumClient::full_scan()`] is called). +//! Updates are returned as a [`ScanResponse`] when [`BdkElectrumClient::scan()`] is called. The +//! older [`BdkElectrumClient::sync()`] and [`BdkElectrumClient::full_scan()`] methods remain +//! available, returning [`SyncResponse`] and [`FullScanResponse`] respectively. //! -//! In most cases [`BdkElectrumClient::sync()`] is used to sync the transaction histories of scripts -//! that the application cares about, for example the scripts for all the receive addresses of a -//! Wallet's keychain that it has shown a user. +//! In most cases [`BdkElectrumClient::scan()`] should be used to combine keychain discovery with +//! syncing the transaction histories of scripts that the application cares about, for example the +//! scripts for all the receive addresses of a Wallet's keychain that it has shown a user. //! -//! [`BdkElectrumClient::full_scan`] is meant to be used when importing or restoring a keychain -//! where the range of possibly used scripts is not known. In this case it is necessary to scan all -//! keychain scripts until a number (the "stop gap") of unused scripts is discovered. +//! [`BdkElectrumClient::scan()`] is intended to replace the +//! [`BdkElectrumClient::full_scan()`] and [`BdkElectrumClient::sync()`] APIs. //! //! Refer to [`example_electrum`] for a complete example. //! //! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/examples/example_electrum +//! [`ScanResponse`]: bdk_core::spk_client::ScanResponse //! [`SyncResponse`]: bdk_core::spk_client::SyncResponse //! [`FullScanResponse`]: bdk_core::spk_client::FullScanResponse #![cfg_attr(coverage_nightly, feature(coverage_attribute))] diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index 318708a194..07ca935344 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -1,7 +1,9 @@ use bdk_chain::{ bitcoin::{hashes::Hash, Address, Amount, ScriptBuf, WScriptHash}, + indexer::keychain_txout::{KeychainTxOutIndex, ScanRequestBuilderExt}, local_chain::LocalChain, - spk_client::{FullScanRequest, SyncRequest, SyncResponse}, + miniscript::{Descriptor, DescriptorPublicKey}, + spk_client::{FullScanRequest, ScanRequest, SyncRequest, SyncResponse}, spk_txout::SpkTxOutIndex, Balance, CanonicalizationParams, ConfirmationBlockTime, IndexedTxGraph, Indexer, Merge, TxGraph, @@ -14,6 +16,7 @@ use bdk_electrum::BdkElectrumClient; use bdk_testenv::{ anyhow, corepc_node::{Input, Output}, + utils::DESCRIPTORS, TestEnv, }; use core::time::Duration; @@ -34,6 +37,12 @@ pub fn get_test_spk() -> ScriptBuf { ScriptBuf::new_p2tr(&secp, pk, None) } +fn parse_descriptor(descriptor: &str) -> Descriptor { + Descriptor::::parse_descriptor(&Secp256k1::signing_only(), descriptor) + .expect("descriptor must parse") + .0 +} + fn get_balance( recv_chain: &LocalChain, recv_graph: &IndexedTxGraph>, @@ -756,6 +765,812 @@ fn test_sync_with_coinbase() -> anyhow::Result<()> { Ok(()) } +#[test] +fn test_scan_detect_receive_tx_cancel() -> anyhow::Result<()> { + const SEND_TX_FEE: Amount = Amount::from_sat(1000); + const UNDO_SEND_TX_FEE: Amount = Amount::from_sat(2000); + + let env = TestEnv::new()?; + let rpc_client = env.rpc_client(); + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let mut graph = IndexedTxGraph::::new(SpkTxOutIndex::<()>::default()); + let (chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + + let receiver_spk = get_test_spk(); + let receiver_addr = Address::from_script(&receiver_spk, bdk_chain::bitcoin::Network::Regtest)?; + graph.index.insert_spk((), receiver_spk); + + env.mine_blocks(101, None)?; + + let selected_utxo = rpc_client + .list_unspent()? + .0 + .into_iter() + .find(|utxo| utxo.amount == Amount::from_int_btc(50).to_btc()) + .expect("Must find a block reward UTXO") + .into_model()?; + + let sender_spk = selected_utxo.script_pubkey.clone(); + let sender_addr = Address::from_script(&sender_spk, bdk_chain::bitcoin::Network::Regtest) + .expect("Failed to derive address from UTXO"); + + let inputs = [Input { + txid: selected_utxo.txid, + vout: selected_utxo.vout as u64, + sequence: None, + }]; + + let send_tx_outputs = [Output::new( + receiver_addr, + selected_utxo.amount.to_unsigned()? - SEND_TX_FEE, + )]; + let send_tx = rpc_client + .create_raw_transaction(&inputs, &send_tx_outputs)? + .into_model()? + .0; + let send_tx = rpc_client + .sign_raw_transaction_with_wallet(&send_tx)? + .into_model()? + .tx; + + let undo_send_outputs = [Output::new( + sender_addr, + selected_utxo.amount.to_unsigned()? - UNDO_SEND_TX_FEE, + )]; + let undo_send_tx = rpc_client + .create_raw_transaction(&inputs, &undo_send_outputs)? + .into_model()? + .0; + let undo_send_tx = rpc_client + .sign_raw_transaction_with_wallet(&undo_send_tx)? + .into_model()? + .tx; + + let send_txid = env.rpc_client().send_raw_transaction(&send_tx)?.txid()?; + env.wait_until_electrum_sees_txid(send_txid, Duration::from_secs(6))?; + let scan_request = ScanRequest::<()>::builder() + .chain_tip(chain.tip()) + .spks_with_indexes(graph.index.all_spks().clone()) + .expected_spk_txids( + graph + .canonical_view(&chain, chain.tip().block_id(), Default::default()) + .list_expected_spk_txids(&graph.index, ..), + ); + let scan_response = client.scan(scan_request, BATCH_SIZE, true)?; + assert!( + scan_response + .tx_update + .txs + .iter() + .any(|tx| tx.compute_txid() == send_txid), + "scan response must include the send_tx" + ); + let changeset = graph.apply_update(scan_response.tx_update.clone()); + assert!( + changeset.tx_graph.txs.contains(&send_tx), + "tx graph must deem send_tx relevant and include it" + ); + + let undo_send_txid = env + .rpc_client() + .send_raw_transaction(&undo_send_tx)? + .txid()?; + env.wait_until_electrum_sees_txid(undo_send_txid, Duration::from_secs(6))?; + let scan_request = ScanRequest::<()>::builder() + .chain_tip(chain.tip()) + .spks_with_indexes(graph.index.all_spks().clone()) + .expected_spk_txids( + graph + .canonical_view(&chain, chain.tip().block_id(), Default::default()) + .list_expected_spk_txids(&graph.index, ..), + ); + let scan_response = client.scan(scan_request, BATCH_SIZE, true)?; + assert!( + scan_response + .tx_update + .evicted_ats + .iter() + .any(|(txid, _)| *txid == send_txid), + "scan response must track send_tx as missing from mempool" + ); + let changeset = graph.apply_update(scan_response.tx_update.clone()); + assert!( + changeset.tx_graph.last_evicted.contains_key(&send_txid), + "tx graph must track send_tx as missing" + ); + + Ok(()) +} + +#[test] +fn test_scan_chained_mempool_tx_sync() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let rpc_client = env.rpc_client(); + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + + let tracked_addr = rpc_client.new_address()?; + + env.mine_blocks(100, None)?; + + let txid1 = env.send(&tracked_addr, Amount::from_btc(1.0)?)?; + + let raw_tx = rpc_client.get_raw_transaction(txid1)?.transaction()?; + let (vout, utxo) = raw_tx + .output + .iter() + .enumerate() + .find(|(_, utxo)| utxo.script_pubkey == tracked_addr.script_pubkey()) + .expect("must find the newly created UTXO"); + + let tx_that_spends_unconfirmed = rpc_client + .create_raw_transaction( + &[Input { + txid: raw_tx.compute_txid(), + vout: vout as u64, + sequence: None, + }], + &[Output::new( + tracked_addr.clone(), + utxo.value - Amount::from_sat(1000), + )], + )? + .transaction()?; + + let signed_tx = rpc_client + .sign_raw_transaction_with_wallet(&tx_that_spends_unconfirmed)? + .into_model()? + .tx; + + let txid2 = rpc_client.send_raw_transaction(&signed_tx)?.txid()?; + + env.wait_until_electrum_sees_txid(signed_tx.compute_txid(), Duration::from_secs(6))?; + + let spk = tracked_addr.clone().script_pubkey(); + let script = spk.as_script(); + let spk_history = electrum_client.script_get_history(script)?; + assert!( + spk_history.into_iter().any(|tx_res| tx_res.height < 0), + "must find tx with negative height" + ); + + let client = BdkElectrumClient::new(electrum_client); + let req = ScanRequest::<()>::builder() + .spks(core::iter::once(tracked_addr.script_pubkey())) + .build(); + let req_time = req.start_time(); + let response = client.scan(req, 1, false)?; + assert_eq!( + response.tx_update.seen_ats, + [(txid1, req_time), (txid2, req_time)].into(), + "both txids must have `seen_at` time match the request's `start_time`", + ); + + Ok(()) +} + +#[test] +fn test_scan_update_tx_graph_without_keychain() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let receive_address0 = + Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked(); + let receive_address1 = + Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked(); + + let misc_spks = [ + receive_address0.script_pubkey(), + receive_address1.script_pubkey(), + ]; + + let _block_hashes = env.mine_blocks(101, None)?; + let txid1 = env + .bitcoind + .client + .send_to_address(&receive_address1, Amount::from_sat(10000))? + .txid()?; + let txid2 = env + .bitcoind + .client + .send_to_address(&receive_address0, Amount::from_sat(20000))? + .txid()?; + env.mine_blocks(1, None)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let cp_tip = env.make_checkpoint_tip(); + + let scan_update = { + let request = ScanRequest::<()>::builder() + .chain_tip(cp_tip.clone()) + .spks(misc_spks); + client.scan(request, 1, true)? + }; + + assert!( + { + let update_cps = scan_update + .chain_update + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + let superset_cps = cp_tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + superset_cps.is_superset(&update_cps) + }, + "update should not alter original checkpoint tip since we already started with all checkpoints", + ); + + let tx_update = scan_update.tx_update; + let updated_graph = { + let mut graph = TxGraph::::default(); + let _ = graph.apply_update(tx_update.clone()); + graph + }; + for tx in &tx_update.txs { + let fee = updated_graph.calculate_fee(tx).expect("Fee must exist"); + + let tx_fee = env + .bitcoind + .client + .get_transaction(tx.compute_txid())? + .into_model() + .expect("Tx must exist") + .fee + .expect("Fee must exist") + .abs() + .to_unsigned() + .expect("valid `Amount`"); + + assert_eq!(fee, tx_fee); + } + + assert_eq!( + tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect::>(), + [txid1, txid2].into(), + "update must include all expected transactions", + ); + Ok(()) +} + +#[test] +fn test_scan_update_tx_graph_stop_gap() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + let _block_hashes = env.mine_blocks(101, None)?; + + let addresses = [ + "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4", + "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30", + "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g", + "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww", + "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu", + "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh", + "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2", + "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8", + "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef", + "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk", + ]; + let addresses: Vec<_> = addresses + .into_iter() + .map(|s| Address::from_str(s).unwrap().assume_checked()) + .collect(); + let spks: Vec<_> = addresses + .iter() + .enumerate() + .map(|(i, addr)| (i as u32, addr.script_pubkey())) + .collect(); + + let txid_4th_addr = env + .bitcoind + .client + .send_to_address(&addresses[3], Amount::from_sat(10000))? + .txid()?; + env.mine_blocks(1, None)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let cp_tip = env.make_checkpoint_tip(); + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(3) + .discover_keychain(0, spks.clone()); + client.scan(request, 1, false)? + }; + assert!(scan_update.tx_update.txs.is_empty()); + assert!(scan_update.last_active_indices.is_empty()); + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(4) + .discover_keychain(0, spks.clone()); + client.scan(request, 1, false)? + }; + assert_eq!( + scan_update.tx_update.txs.first().unwrap().compute_txid(), + txid_4th_addr + ); + assert_eq!(scan_update.last_active_indices[&0], 3); + + let txid_last_addr = env + .bitcoind + .client + .send_to_address(&addresses[addresses.len() - 1], Amount::from_sat(10000))? + .txid()?; + env.mine_blocks(1, None)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(6) + .discover_keychain(0, spks.clone()); + client.scan(request, 1, false)? + }; + let txs: HashSet<_> = scan_update + .tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect(); + assert_eq!(txs, [txid_4th_addr, txid_last_addr].into()); + assert_eq!(scan_update.last_active_indices[&0], 9); + + Ok(()) +} + +#[test] +fn test_scan_with_coinbase() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros()); + let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?; + + let (recv_chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + + env.mine_blocks(101, Some(addr_to_track))?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + assert!(client + .scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + ) + .is_ok()); + + Ok(()) +} + +#[test] +fn test_scan_tracks_confirmation_state_across_reorgs() -> anyhow::Result<()> { + const SEND_AMOUNT: Amount = Amount::from_sat(10_000); + + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let addr_to_mine = env.bitcoind.client.new_address()?; + let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros()); + let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?; + + let (mut recv_chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + let mut recv_graph = IndexedTxGraph::::new({ + let mut recv_index = SpkTxOutIndex::default(); + recv_index.insert_spk((), spk_to_track.clone()); + recv_index + }); + + env.mine_blocks(101, Some(addr_to_mine))?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let txid = env.send(&addr_to_track, SEND_AMOUNT)?; + env.wait_until_electrum_sees_txid(txid, Duration::from_secs(6))?; + + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + trusted_pending: SEND_AMOUNT, + ..Balance::default() + }, + "balance must be correct", + ); + + env.mine_blocks(1, None)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + confirmed: SEND_AMOUNT, + ..Balance::default() + }, + "balance must be correct", + ); + + env.reorg_empty_blocks(1)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + trusted_pending: SEND_AMOUNT, + ..Balance::default() + }, + ); + + env.mine_blocks(1, None)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + confirmed: SEND_AMOUNT, + ..Balance::default() + }, + "balance must be correct", + ); + + for tx in recv_graph.graph().full_txs() { + let fee = recv_graph + .graph() + .calculate_fee(&tx.tx) + .expect("fee must exist"); + + let tx_fee = env + .bitcoind + .client + .get_transaction(tx.txid)? + .into_model() + .expect("Tx must exist") + .fee + .expect("Fee must exist") + .abs() + .to_unsigned() + .expect("valid `Amount`"); + + assert_eq!(fee, tx_fee); + } + + Ok(()) +} + +#[test] +fn test_scan_tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> { + const REORG_COUNT: usize = 8; + const SEND_AMOUNT: Amount = Amount::from_sat(10_000); + + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let addr_to_mine = env.bitcoind.client.new_address()?; + let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros()); + let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?; + + let (mut recv_chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + let mut recv_graph = IndexedTxGraph::::new({ + let mut recv_index = SpkTxOutIndex::default(); + recv_index.insert_spk((), spk_to_track.clone()); + recv_index + }); + + env.mine_blocks(101, Some(addr_to_mine))?; + + let mut txids = vec![]; + let mut hashes = vec![]; + for _ in 0..REORG_COUNT { + txids.push(env.send(&addr_to_track, SEND_AMOUNT)?); + hashes.extend(env.mine_blocks(1, None)?); + } + + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + let initial_anchors = update.tx_update.anchors.clone(); + assert_eq!(initial_anchors.len(), REORG_COUNT); + for i in 0..REORG_COUNT { + let (anchor, txid) = initial_anchors.iter().nth(i).unwrap(); + assert_eq!(anchor.block_id.hash, hashes[i]); + assert_eq!(*txid, txids[i]); + } + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + confirmed: SEND_AMOUNT * REORG_COUNT as u64, + ..Balance::default() + }, + "initial balance must be correct", + ); + + for depth in 1..=REORG_COUNT { + env.reorg_empty_blocks(depth)?; + + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + assert!(initial_anchors.is_superset(&update.tx_update.anchors)); + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + trusted_pending: SEND_AMOUNT * depth as u64, + confirmed: SEND_AMOUNT * (REORG_COUNT - depth) as u64, + ..Balance::default() + }, + "reorg_count: {depth}", + ); + } + + Ok(()) +} + +#[test] +fn test_scan_check_fee_calculation() -> anyhow::Result<()> { + const SEND_AMOUNT: Amount = Amount::from_sat(10_000); + const FEE_AMOUNT: Amount = Amount::from_sat(1650); + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros()); + let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?; + + let (mut recv_chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + let mut recv_graph = IndexedTxGraph::::new({ + let mut recv_index = SpkTxOutIndex::default(); + recv_index.insert_spk((), spk_to_track.clone()); + recv_index + }); + + env.mine_blocks(101, None)?; + + let new_addr = env.rpc_client().new_address()?; + let prev_amt = SEND_AMOUNT + FEE_AMOUNT; + env.send(&new_addr, prev_amt)?; + let _prev_block_hash = env + .mine_blocks(1, None)? + .into_iter() + .next() + .expect("should've successfully mined a block"); + + let txid = env.send(&addr_to_track, SEND_AMOUNT)?; + + let _block_hash = env + .mine_blocks(1, None)? + .into_iter() + .next() + .expect("should've successfully mined a block"); + + let tx = env.rpc_client().get_transaction(txid)?.into_model()?.tx; + assert_eq!(tx.input.len(), 1); + assert_eq!(tx.output.len(), 1); + let outpoint = tx.input[0].previous_output; + let prev_txid = outpoint.txid; + + let prev_tx = env + .rpc_client() + .get_transaction(prev_txid)? + .into_model()? + .tx; + let txout = prev_tx + .output + .iter() + .find(|txout| txout.value == prev_amt) + .expect("should've successfully found the existing `TxOut`"); + + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + let update = client.scan( + ScanRequest::<()>::builder() + .chain_tip(recv_chain.tip()) + .spks([spk_to_track.clone()]), + BATCH_SIZE, + true, + )?; + if let Some(chain_update) = update.chain_update.clone() { + let _ = recv_chain + .apply_update(chain_update) + .map_err(|err| anyhow::anyhow!("LocalChain update error: {err:?}"))?; + } + let _ = recv_graph.apply_update(update.tx_update.clone()); + + let graph_txout = recv_graph + .graph() + .all_txouts() + .find(|(_op, txout)| txout.value == prev_amt) + .unwrap(); + assert_eq!(graph_txout, (outpoint, txout)); + + assert_eq!( + get_balance(&recv_chain, &recv_graph)?, + Balance { + confirmed: SEND_AMOUNT, + ..Balance::default() + }, + ); + + for tx in recv_graph.graph().full_txs() { + let fee = recv_graph + .graph() + .calculate_fee(&tx.tx) + .expect("fee must exist"); + + assert_eq!(fee, FEE_AMOUNT); + + let tx_fee = env + .bitcoind + .client + .get_transaction(tx.txid) + .expect("Tx must exist") + .fee + .map(|fee| Amount::from_float_in(fee.abs(), Denomination::BTC)) + .expect("Fee must exist") + .expect("Amount parsing should succeed") + .to_sat(); + + assert_eq!(fee, Amount::from_sat(tx_fee)); + } + Ok(()) +} + +#[test] +fn test_scan_revealed_scripts_beyond_stop_gap() -> anyhow::Result<()> { + const KEYCHAIN: u32 = 0; + const REVEALED_INDEX: u32 = 9; + const STOP_GAP: usize = 3; + + let env = TestEnv::new()?; + let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?; + let client = BdkElectrumClient::new(electrum_client); + + let mut indexer = KeychainTxOutIndex::new(0, true); + let _ = indexer + .insert_descriptor(KEYCHAIN, parse_descriptor(DESCRIPTORS[3])) + .expect("descriptor must insert"); + let _ = indexer + .reveal_to_target(KEYCHAIN, REVEALED_INDEX) + .expect("keychain must exist"); + assert_eq!(indexer.last_revealed_index(KEYCHAIN), Some(REVEALED_INDEX)); + + let (_, revealed_spk) = indexer + .revealed_keychain_spks(KEYCHAIN) + .last() + .expect("revealed spk must exist"); + let revealed_addr = Address::from_script(&revealed_spk, bdk_chain::bitcoin::Network::Regtest)?; + + env.mine_blocks(101, None)?; + let txid = env + .bitcoind + .client + .send_to_address(&revealed_addr, Amount::from_sat(10000))? + .txid()?; + env.mine_blocks(1, None)?; + env.wait_until_electrum_sees_block(Duration::from_secs(6))?; + + let cp_tip = env.make_checkpoint_tip(); + let scan_update = { + let request = ScanRequest::builder() + .chain_tip(cp_tip) + .stop_gap(STOP_GAP) + .revealed_spks_from_indexer(&indexer, KEYCHAIN..=KEYCHAIN) + .discover_from_indexer(&indexer); + client.scan(request, BATCH_SIZE, false)? + }; + + let txids: HashSet<_> = scan_update + .tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect(); + assert!( + txids.contains(&txid), + "scan must sync transactions on revealed scripts even when discovery stop gap would stop earlier", + ); + assert!( + scan_update.last_active_indices.is_empty(), + "the transaction must be found by explicit sync, not discovery", + ); + + Ok(()) +} + #[test] fn test_check_fee_calculation() -> anyhow::Result<()> { const SEND_AMOUNT: Amount = Amount::from_sat(10_000); diff --git a/crates/esplora/README.md b/crates/esplora/README.md index 4a6acb377d..9d3491f301 100644 --- a/crates/esplora/README.md +++ b/crates/esplora/README.md @@ -3,8 +3,9 @@ BDK Esplora extends [`esplora-client`] (with extension traits: [`EsploraExt`] and [`EsploraAsyncExt`]) to update [`bdk_chain`] structures from an Esplora server. -The extension traits are primarily intended to satisfy [`SyncRequest`]s with [`sync`] and -[`FullScanRequest`]s with [`full_scan`]. +The extension traits are primarily intended to satisfy [`ScanRequest`]s with [`scan`]. This +unifies the work previously split across [`sync`] for explicit scripts/txids/outpoints and +[`full_scan`] for keychain discovery. [`scan`] is intended to replace [`sync`] and [`full_scan`]. ## Usage @@ -47,7 +48,9 @@ For full examples, refer to [`example_wallet_esplora_blocking`](https://github.c [`bdk_chain`]: https://docs.rs/bdk-chain/ [`EsploraExt`]: crate::EsploraExt [`EsploraAsyncExt`]: crate::EsploraAsyncExt +[`ScanRequest`]: bdk_core::spk_client::ScanRequest [`SyncRequest`]: bdk_core::spk_client::SyncRequest [`FullScanRequest`]: bdk_core::spk_client::FullScanRequest +[`scan`]: crate::EsploraExt::scan [`sync`]: crate::EsploraExt::sync [`full_scan`]: crate::EsploraExt::full_scan diff --git a/crates/esplora/src/async_ext.rs b/crates/esplora/src/async_ext.rs index dc4d321869..8857cbb8c2 100644 --- a/crates/esplora/src/async_ext.rs +++ b/crates/esplora/src/async_ext.rs @@ -1,7 +1,8 @@ use async_trait::async_trait; use bdk_core::collections::{BTreeMap, BTreeSet, HashSet}; use bdk_core::spk_client::{ - FullScanRequest, FullScanResponse, SpkWithExpectedTxids, SyncRequest, SyncResponse, + FullScanRequest, FullScanResponse, ScanRequest, ScanResponse, SpkWithExpectedTxids, + SyncRequest, SyncResponse, }; use bdk_core::{ bitcoin::{BlockHash, OutPoint, Txid}, @@ -49,6 +50,20 @@ pub trait EsploraAsyncExt { request: R, parallel_requests: usize, ) -> Result; + + /// Scan keychain scripts, explicit scripts, txids, and/or outpoints against Esplora. + /// + /// `request` provides the data required to perform a script-pubkey-based scan (see + /// [`ScanRequest`]). Discovery for each keychain (`K`) stops after a gap of + /// `request.stop_gap()` script pubkeys with no associated transactions. + /// `parallel_requests` specifies the maximum number of HTTP requests to make in parallel. + /// + /// Refer to [crate-level docs](crate) for more. + async fn scan> + Send>( + &self, + request: R, + parallel_requests: usize, + ) -> Result, Error>; } #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] @@ -171,6 +186,92 @@ where tx_update, }) } + + async fn scan> + Send>( + &self, + request: R, + parallel_requests: usize, + ) -> Result, Error> { + let mut request: ScanRequest = request.into(); + let start_time = request.start_time(); + + let chain_tip = request.chain_tip(); + let latest_blocks = if chain_tip.is_some() { + Some(fetch_latest_blocks(self).await?) + } else { + None + }; + + let mut tx_update = TxUpdate::::default(); + let mut inserted_txs = HashSet::::new(); + let mut last_active_indices = BTreeMap::::new(); + + // Discovery: scan keychain spks with stop_gap + let stop_gap = request.stop_gap(); + for keychain in request.keychains() { + let keychain_spks = request + .iter_discovery_spks(keychain.clone()) + .map(|(spk_i, spk)| (spk_i, spk.into())); + let (update, last_active_index) = fetch_txs_with_keychain_spks( + self, + start_time, + &mut inserted_txs, + keychain_spks, + stop_gap, + parallel_requests, + ) + .await?; + tx_update.extend(update); + if let Some(last_active_index) = last_active_index { + last_active_indices.insert(keychain, last_active_index); + } + } + + // Explicit sync: spks, txids, outpoints + tx_update.extend( + fetch_txs_with_spks( + self, + start_time, + &mut inserted_txs, + request.iter_spks_with_expected_txids(), + parallel_requests, + ) + .await?, + ); + tx_update.extend( + fetch_txs_with_txids( + self, + start_time, + &mut inserted_txs, + request.iter_txids(), + parallel_requests, + ) + .await?, + ); + tx_update.extend( + fetch_txs_with_outpoints( + self, + start_time, + &mut inserted_txs, + request.iter_outpoints(), + parallel_requests, + ) + .await?, + ); + + let chain_update = match (chain_tip, latest_blocks) { + (Some(chain_tip), Some(latest_blocks)) => { + Some(chain_update(self, &latest_blocks, &chain_tip, &tx_update.anchors).await?) + } + _ => None, + }; + + Ok(ScanResponse { + chain_update, + tx_update, + last_active_indices, + }) + } } /// Fetch latest blocks from Esplora in an atomic call. diff --git a/crates/esplora/src/blocking_ext.rs b/crates/esplora/src/blocking_ext.rs index 3a25544762..04cc6da6dc 100644 --- a/crates/esplora/src/blocking_ext.rs +++ b/crates/esplora/src/blocking_ext.rs @@ -1,6 +1,7 @@ use bdk_core::collections::{BTreeMap, BTreeSet, HashSet}; use bdk_core::spk_client::{ - FullScanRequest, FullScanResponse, SpkWithExpectedTxids, SyncRequest, SyncResponse, + FullScanRequest, FullScanResponse, ScanRequest, ScanResponse, SpkWithExpectedTxids, + SyncRequest, SyncResponse, }; use bdk_core::{ bitcoin::{BlockHash, OutPoint, Txid}, @@ -46,6 +47,20 @@ pub trait EsploraExt { request: R, parallel_requests: usize, ) -> Result; + + /// Scan keychain scripts, explicit scripts, txids, and/or outpoints against Esplora. + /// + /// `request` provides the data required to perform a script-pubkey-based scan (see + /// [`ScanRequest`]). Discovery for each keychain (`K`) stops after a gap of + /// `request.stop_gap()` script pubkeys with no associated transactions. + /// `parallel_requests` specifies the maximum number of HTTP requests to make in parallel. + /// + /// Refer to [crate-level docs](crate) for more. + fn scan>>( + &self, + request: R, + parallel_requests: usize, + ) -> Result, Error>; } impl EsploraExt for esplora_client::BlockingClient { @@ -157,6 +172,85 @@ impl EsploraExt for esplora_client::BlockingClient { tx_update, }) } + + fn scan>>( + &self, + request: R, + parallel_requests: usize, + ) -> Result, Error> { + let mut request: ScanRequest = request.into(); + let start_time = request.start_time(); + + let chain_tip = request.chain_tip(); + let latest_blocks = if chain_tip.is_some() { + Some(fetch_latest_blocks(self)?) + } else { + None + }; + + let mut tx_update = TxUpdate::default(); + let mut inserted_txs = HashSet::::new(); + let mut last_active_indices = BTreeMap::::new(); + + // Discovery: scan keychain spks with stop_gap + let stop_gap = request.stop_gap(); + for keychain in request.keychains() { + let keychain_spks = request + .iter_discovery_spks(keychain.clone()) + .map(|(spk_i, spk)| (spk_i, spk.into())); + let (update, last_active_index) = fetch_txs_with_keychain_spks( + self, + start_time, + &mut inserted_txs, + keychain_spks, + stop_gap, + parallel_requests, + )?; + tx_update.extend(update); + if let Some(last_active_index) = last_active_index { + last_active_indices.insert(keychain, last_active_index); + } + } + + // Explicit sync: spks, txids, outpoints + tx_update.extend(fetch_txs_with_spks( + self, + start_time, + &mut inserted_txs, + request.iter_spks_with_expected_txids(), + parallel_requests, + )?); + tx_update.extend(fetch_txs_with_txids( + self, + start_time, + &mut inserted_txs, + request.iter_txids(), + parallel_requests, + )?); + tx_update.extend(fetch_txs_with_outpoints( + self, + start_time, + &mut inserted_txs, + request.iter_outpoints(), + parallel_requests, + )?); + + let chain_update = match (chain_tip, latest_blocks) { + (Some(chain_tip), Some(latest_blocks)) => Some(chain_update( + self, + &latest_blocks, + &chain_tip, + &tx_update.anchors, + )?), + _ => None, + }; + + Ok(ScanResponse { + chain_update, + tx_update, + last_active_indices, + }) + } } /// Fetch latest blocks from Esplora in an atomic call. diff --git a/crates/esplora/src/lib.rs b/crates/esplora/src/lib.rs index 60b4f1eb3f..2a3b62c9df 100644 --- a/crates/esplora/src/lib.rs +++ b/crates/esplora/src/lib.rs @@ -1,10 +1,12 @@ #![doc = include_str!("../README.md")] //! # Stop Gap //! -//! [`EsploraExt::full_scan`] takes in a `stop_gap` input which is defined as the maximum number of -//! consecutive unused script pubkeys to scan transactions for before stopping. +//! [`EsploraExt::scan`] uses the stop gap configured on [`bdk_core::spk_client::ScanRequest`] for +//! keychain discovery, while [`EsploraExt::full_scan`] takes `stop_gap` directly. The stop gap is +//! defined as the maximum number of consecutive unused script pubkeys to scan transactions for +//! before stopping. //! -//! For example, with a `stop_gap` of 3, `full_scan` will keep scanning until it encounters 3 +//! For example, with a `stop_gap` of 3, discovery will keep scanning until it encounters 3 //! consecutive script pubkeys with no associated transactions. //! //! This follows the same approach as other Bitcoin-related software, diff --git a/crates/esplora/tests/async_ext.rs b/crates/esplora/tests/async_ext.rs index 209e5b7882..bd1aa52ecb 100644 --- a/crates/esplora/tests/async_ext.rs +++ b/crates/esplora/tests/async_ext.rs @@ -1,11 +1,12 @@ use bdk_chain::bitcoin::{Address, Amount}; +use bdk_chain::indexer::keychain_txout::{KeychainTxOutIndex, ScanRequestBuilderExt}; use bdk_chain::local_chain::LocalChain; -use bdk_chain::spk_client::{FullScanRequest, SyncRequest}; +use bdk_chain::spk_client::{FullScanRequest, ScanRequest, SyncRequest}; use bdk_chain::spk_txout::SpkTxOutIndex; use bdk_chain::{ConfirmationBlockTime, IndexedTxGraph, TxGraph}; use bdk_esplora::EsploraAsyncExt; use bdk_testenv::corepc_node::{Input, Output}; -use bdk_testenv::{anyhow, TestEnv}; +use bdk_testenv::{anyhow, utils::DESCRIPTORS, TestEnv}; use esplora_client::{self, Builder}; use std::collections::{BTreeSet, HashSet}; use std::str::FromStr; @@ -369,3 +370,376 @@ pub async fn test_async_update_tx_graph_stop_gap() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_scan_detect_receive_tx_cancel() -> anyhow::Result<()> { + const SEND_TX_FEE: Amount = Amount::from_sat(1000); + const UNDO_SEND_TX_FEE: Amount = Amount::from_sat(2000); + + let env = TestEnv::new()?; + let rpc_client = env.rpc_client(); + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_async()?; + + let mut graph = IndexedTxGraph::::new(SpkTxOutIndex::<()>::default()); + let (chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + + let receiver_spk = common::get_test_spk(); + let receiver_addr = Address::from_script(&receiver_spk, bdk_chain::bitcoin::Network::Regtest)?; + graph.index.insert_spk((), receiver_spk); + + env.mine_blocks(101, None)?; + + let selected_utxo = rpc_client + .list_unspent()? + .0 + .into_iter() + .find(|utxo| utxo.amount == Amount::from_int_btc(50).to_btc()) + .expect("Must find a block reward UTXO") + .into_model()?; + + let sender_spk = selected_utxo.script_pubkey.clone(); + let sender_addr = Address::from_script(&sender_spk, bdk_chain::bitcoin::Network::Regtest) + .expect("Failed to derive address from UTXO"); + + let inputs = [Input { + txid: selected_utxo.txid, + vout: selected_utxo.vout as u64, + sequence: None, + }]; + + let address = receiver_addr; + let value = selected_utxo.amount.to_unsigned()? - SEND_TX_FEE; + let send_tx_outputs = Output::new(address, value); + + let send_tx = rpc_client + .create_raw_transaction(&inputs, &[send_tx_outputs])? + .into_model()? + .0; + let send_tx = rpc_client + .sign_raw_transaction_with_wallet(&send_tx)? + .into_model()? + .tx; + + let undo_send_outputs = [Output::new( + sender_addr, + selected_utxo.amount.to_unsigned()? - UNDO_SEND_TX_FEE, + )]; + let undo_send_tx = rpc_client + .create_raw_transaction(&inputs, &undo_send_outputs)? + .into_model()? + .0; + let undo_send_tx = rpc_client + .sign_raw_transaction_with_wallet(&undo_send_tx)? + .into_model()? + .tx; + + let send_txid = env + .rpc_client() + .send_raw_transaction(&send_tx)? + .into_model()? + .0; + env.wait_until_electrum_sees_txid(send_txid, Duration::from_secs(6))?; + let scan_request = ScanRequest::<()>::builder() + .chain_tip(chain.tip()) + .spks_with_indexes(graph.index.all_spks().clone()) + .expected_spk_txids( + graph + .canonical_view(&chain, chain.tip().block_id(), Default::default()) + .list_expected_spk_txids(&graph.index, ..), + ); + let scan_response = client.scan(scan_request, 1).await?; + assert!( + scan_response + .tx_update + .txs + .iter() + .any(|tx| tx.compute_txid() == send_txid), + "scan response must include the send_tx" + ); + let changeset = graph.apply_update(scan_response.tx_update.clone()); + assert!( + changeset.tx_graph.txs.contains(&send_tx), + "tx graph must deem send_tx relevant and include it" + ); + + let undo_send_txid = env + .rpc_client() + .send_raw_transaction(&undo_send_tx)? + .txid()?; + env.wait_until_electrum_sees_txid(undo_send_txid, Duration::from_secs(6))?; + let scan_request = ScanRequest::<()>::builder() + .chain_tip(chain.tip()) + .spks_with_indexes(graph.index.all_spks().clone()) + .expected_spk_txids( + graph + .canonical_view(&chain, chain.tip().block_id(), Default::default()) + .list_expected_spk_txids(&graph.index, ..), + ); + let scan_response = client.scan(scan_request, 1).await?; + assert!( + scan_response + .tx_update + .evicted_ats + .iter() + .any(|(txid, _)| *txid == send_txid), + "scan response must track send_tx as missing from mempool" + ); + let changeset = graph.apply_update(scan_response.tx_update.clone()); + assert!( + changeset.tx_graph.last_evicted.contains_key(&send_txid), + "tx graph must track send_tx as missing" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_scan_update_tx_graph_without_keychain() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_async()?; + + let receive_address0 = + Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked(); + let receive_address1 = + Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked(); + + let misc_spks = [ + receive_address0.script_pubkey(), + receive_address1.script_pubkey(), + ]; + + let _block_hashes = env.mine_blocks(101, None)?; + let txid1 = env + .bitcoind + .client + .send_to_address(&receive_address1, Amount::from_sat(10000))? + .txid()?; + let txid2 = env + .bitcoind + .client + .send_to_address(&receive_address0, Amount::from_sat(20000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().await.unwrap() < 102 { + sleep(Duration::from_millis(10)) + } + + let cp_tip = env.make_checkpoint_tip(); + + let scan_update = { + let request = ScanRequest::<()>::builder() + .chain_tip(cp_tip.clone()) + .spks(misc_spks); + client.scan(request, 1).await? + }; + + assert!( + { + let update_cps = scan_update + .chain_update + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + let superset_cps = cp_tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + superset_cps.is_superset(&update_cps) + }, + "update should not alter original checkpoint tip since we already started with all checkpoints", + ); + + let tx_update = scan_update.tx_update; + let updated_graph = { + let mut graph = TxGraph::::default(); + let _ = graph.apply_update(tx_update.clone()); + graph + }; + for tx in &tx_update.txs { + let fee = updated_graph.calculate_fee(tx).expect("Fee must exist"); + + let tx_fee = env + .bitcoind + .client + .get_transaction(tx.compute_txid()) + .expect("Tx must exist") + .into_model()? + .fee + .expect("Fee must exist") + .abs() + .to_unsigned() + .expect("valid `Amount`"); + + assert_eq!(fee, tx_fee); + } + + assert_eq!( + tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect::>(), + [txid1, txid2].into(), + "update must include all expected transactions" + ); + Ok(()) +} + +#[tokio::test] +async fn test_scan_update_tx_graph_stop_gap() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_async()?; + let _block_hashes = env.mine_blocks(101, None)?; + + let addresses = [ + "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4", + "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30", + "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g", + "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww", + "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu", + "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh", + "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2", + "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8", + "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef", + "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk", + ]; + let addresses: Vec<_> = addresses + .into_iter() + .map(|s| Address::from_str(s).unwrap().assume_checked()) + .collect(); + let spks: Vec<_> = addresses + .iter() + .enumerate() + .map(|(i, addr)| (i as u32, addr.script_pubkey())) + .collect(); + + let txid_4th_addr = env + .bitcoind + .client + .send_to_address(&addresses[3], Amount::from_sat(10000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().await.unwrap() < 103 { + sleep(Duration::from_millis(10)) + } + + let cp_tip = env.make_checkpoint_tip(); + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(3) + .discover_keychain(0, spks.clone()); + client.scan(request, 1).await? + }; + assert!(scan_update.tx_update.txs.is_empty()); + assert!(scan_update.last_active_indices.is_empty()); + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(4) + .discover_keychain(0, spks.clone()); + client.scan(request, 1).await? + }; + assert_eq!( + scan_update.tx_update.txs.first().unwrap().compute_txid(), + txid_4th_addr + ); + assert_eq!(scan_update.last_active_indices[&0], 3); + + let txid_last_addr = env + .bitcoind + .client + .send_to_address(&addresses[addresses.len() - 1], Amount::from_sat(10000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().await.unwrap() < 104 { + sleep(Duration::from_millis(10)) + } + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(6) + .discover_keychain(0, spks.clone()); + client.scan(request, 1).await? + }; + let txs: HashSet<_> = scan_update + .tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect(); + assert_eq!(txs, [txid_4th_addr, txid_last_addr].into()); + assert_eq!(scan_update.last_active_indices[&0], 9); + + Ok(()) +} + +#[tokio::test] +async fn test_scan_revealed_scripts_beyond_stop_gap() -> anyhow::Result<()> { + const KEYCHAIN: u32 = 0; + const REVEALED_INDEX: u32 = 9; + const STOP_GAP: usize = 3; + + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_async()?; + + let mut indexer = KeychainTxOutIndex::new(0, true); + let _ = indexer + .insert_descriptor(KEYCHAIN, common::parse_descriptor(DESCRIPTORS[3])) + .expect("descriptor must insert"); + let _ = indexer + .reveal_to_target(KEYCHAIN, REVEALED_INDEX) + .expect("keychain must exist"); + assert_eq!(indexer.last_revealed_index(KEYCHAIN), Some(REVEALED_INDEX)); + + let (_, revealed_spk) = indexer + .revealed_keychain_spks(KEYCHAIN) + .last() + .expect("revealed spk must exist"); + let revealed_addr = Address::from_script(&revealed_spk, bdk_chain::bitcoin::Network::Regtest)?; + + let _block_hashes = env.mine_blocks(101, None)?; + let txid = env + .bitcoind + .client + .send_to_address(&revealed_addr, Amount::from_sat(10000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().await.unwrap() < 103 { + sleep(Duration::from_millis(10)) + } + + let cp_tip = env.make_checkpoint_tip(); + let scan_update = { + let request = ScanRequest::builder() + .chain_tip(cp_tip) + .stop_gap(STOP_GAP) + .revealed_spks_from_indexer(&indexer, KEYCHAIN..=KEYCHAIN) + .discover_from_indexer(&indexer); + client.scan(request, 1).await? + }; + + let txids: HashSet<_> = scan_update + .tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect(); + assert!( + txids.contains(&txid), + "scan must sync transactions on revealed scripts even when discovery stop gap would stop earlier", + ); + assert!( + scan_update.last_active_indices.is_empty(), + "the transaction must be found by explicit sync, not discovery", + ); + + Ok(()) +} diff --git a/crates/esplora/tests/blocking_ext.rs b/crates/esplora/tests/blocking_ext.rs index 76ed28fbb5..931e04686c 100644 --- a/crates/esplora/tests/blocking_ext.rs +++ b/crates/esplora/tests/blocking_ext.rs @@ -1,11 +1,12 @@ use bdk_chain::bitcoin::{Address, Amount}; +use bdk_chain::indexer::keychain_txout::{KeychainTxOutIndex, ScanRequestBuilderExt}; use bdk_chain::local_chain::LocalChain; -use bdk_chain::spk_client::{FullScanRequest, SyncRequest}; +use bdk_chain::spk_client::{FullScanRequest, ScanRequest, SyncRequest}; use bdk_chain::spk_txout::SpkTxOutIndex; use bdk_chain::{ConfirmationBlockTime, IndexedTxGraph, TxGraph}; use bdk_esplora::EsploraExt; use bdk_testenv::corepc_node::{Input, Output}; -use bdk_testenv::{anyhow, TestEnv}; +use bdk_testenv::{anyhow, utils::DESCRIPTORS, TestEnv}; use esplora_client::{self, Builder}; use std::collections::{BTreeSet, HashSet}; use std::str::FromStr; @@ -368,3 +369,373 @@ pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn test_scan_detect_receive_tx_cancel() -> anyhow::Result<()> { + const SEND_TX_FEE: Amount = Amount::from_sat(1000); + const UNDO_SEND_TX_FEE: Amount = Amount::from_sat(2000); + + let env = TestEnv::new()?; + let rpc_client = env.rpc_client(); + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking(); + + let mut graph = IndexedTxGraph::::new(SpkTxOutIndex::<()>::default()); + let (chain, _) = LocalChain::from_genesis(env.genesis_hash()?); + + let receiver_spk = common::get_test_spk(); + let receiver_addr = Address::from_script(&receiver_spk, bdk_chain::bitcoin::Network::Regtest)?; + graph.index.insert_spk((), receiver_spk); + + env.mine_blocks(101, None)?; + + let selected_utxo = rpc_client + .list_unspent()? + .0 + .into_iter() + .find(|utxo| utxo.amount == Amount::from_int_btc(50).to_btc()) + .expect("Must find a block reward UTXO") + .into_model()?; + + let sender_spk = selected_utxo.script_pubkey.clone(); + let sender_addr = Address::from_script(&sender_spk, bdk_chain::bitcoin::Network::Regtest) + .expect("Failed to derive address from UTXO"); + + let inputs = [Input { + txid: selected_utxo.txid, + vout: selected_utxo.vout as u64, + sequence: None, + }]; + + let send_tx_outputs = [Output::new( + receiver_addr, + selected_utxo.amount.to_unsigned()? - SEND_TX_FEE, + )]; + let send_tx = rpc_client + .create_raw_transaction(&inputs, &send_tx_outputs)? + .into_model()? + .0; + let send_tx = rpc_client + .sign_raw_transaction_with_wallet(&send_tx)? + .into_model()? + .tx; + + let undo_send_outputs = [Output::new( + sender_addr, + selected_utxo.amount.to_unsigned()? - UNDO_SEND_TX_FEE, + )]; + let undo_send_tx = rpc_client + .create_raw_transaction(&inputs, &undo_send_outputs)? + .into_model()? + .0; + let undo_send_tx = rpc_client + .sign_raw_transaction_with_wallet(&undo_send_tx)? + .into_model()? + .tx; + + let send_txid = env.rpc_client().send_raw_transaction(&send_tx)?.txid()?; + env.wait_until_electrum_sees_txid(send_txid, Duration::from_secs(6))?; + let scan_request = ScanRequest::<()>::builder() + .chain_tip(chain.tip()) + .spks_with_indexes(graph.index.all_spks().clone()) + .expected_spk_txids( + graph + .canonical_view(&chain, chain.tip().block_id(), Default::default()) + .list_expected_spk_txids(&graph.index, ..), + ); + let scan_response = client.scan(scan_request, 1)?; + assert!( + scan_response + .tx_update + .txs + .iter() + .any(|tx| tx.compute_txid() == send_txid), + "scan response must include the send_tx" + ); + let changeset = graph.apply_update(scan_response.tx_update.clone()); + assert!( + changeset.tx_graph.txs.contains(&send_tx), + "tx graph must deem send_tx relevant and include it" + ); + + let undo_send_txid = env + .rpc_client() + .send_raw_transaction(&undo_send_tx)? + .txid()?; + env.wait_until_electrum_sees_txid(undo_send_txid, Duration::from_secs(6))?; + let scan_request = ScanRequest::<()>::builder() + .chain_tip(chain.tip()) + .spks_with_indexes(graph.index.all_spks().clone()) + .expected_spk_txids( + graph + .canonical_view(&chain, chain.tip().block_id(), Default::default()) + .list_expected_spk_txids(&graph.index, ..), + ); + let scan_response = client.scan(scan_request, 1)?; + assert!( + scan_response + .tx_update + .evicted_ats + .iter() + .any(|(txid, _)| *txid == send_txid), + "scan response must track send_tx as missing from mempool" + ); + let changeset = graph.apply_update(scan_response.tx_update.clone()); + assert!( + changeset.tx_graph.last_evicted.contains_key(&send_txid), + "tx graph must track send_tx as missing" + ); + + Ok(()) +} + +#[test] +fn test_scan_update_tx_graph_without_keychain() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking(); + + let receive_address0 = + Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked(); + let receive_address1 = + Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked(); + + let misc_spks = [ + receive_address0.script_pubkey(), + receive_address1.script_pubkey(), + ]; + + let _block_hashes = env.mine_blocks(101, None)?; + let txid1 = env + .bitcoind + .client + .send_to_address(&receive_address1, Amount::from_sat(10000))? + .txid()?; + let txid2 = env + .bitcoind + .client + .send_to_address(&receive_address0, Amount::from_sat(20000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().unwrap() < 102 { + sleep(Duration::from_millis(10)) + } + + let cp_tip = env.make_checkpoint_tip(); + + let scan_update = { + let request = ScanRequest::<()>::builder() + .chain_tip(cp_tip.clone()) + .spks(misc_spks); + client.scan(request, 1)? + }; + + assert!( + { + let update_cps = scan_update + .chain_update + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + let superset_cps = cp_tip + .iter() + .map(|cp| cp.block_id()) + .collect::>(); + superset_cps.is_superset(&update_cps) + }, + "update should not alter original checkpoint tip since we already started with all checkpoints", + ); + + let tx_update = scan_update.tx_update; + let updated_graph = { + let mut graph = TxGraph::::default(); + let _ = graph.apply_update(tx_update.clone()); + graph + }; + for tx in &tx_update.txs { + let fee = updated_graph.calculate_fee(tx).expect("Fee must exist"); + + let tx_fee = env + .bitcoind + .client + .get_transaction(tx.compute_txid()) + .expect("Tx must exist") + .into_model() + .expect("Tx must exist") + .fee + .expect("Fee must exist") + .abs() + .to_unsigned() + .expect("valid `Amount`"); + + assert_eq!(fee, tx_fee); + } + + assert_eq!( + tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect::>(), + [txid1, txid2].into(), + "update must include all expected transactions" + ); + Ok(()) +} + +#[test] +fn test_scan_update_tx_graph_stop_gap() -> anyhow::Result<()> { + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking(); + let _block_hashes = env.mine_blocks(101, None)?; + + let addresses = [ + "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4", + "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30", + "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g", + "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww", + "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu", + "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh", + "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2", + "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8", + "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef", + "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk", + ]; + let addresses: Vec<_> = addresses + .into_iter() + .map(|s| Address::from_str(s).unwrap().assume_checked()) + .collect(); + let spks: Vec<_> = addresses + .iter() + .enumerate() + .map(|(i, addr)| (i as u32, addr.script_pubkey())) + .collect(); + + let txid_4th_addr = env + .bitcoind + .client + .send_to_address(&addresses[3], Amount::from_sat(10000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().unwrap() < 103 { + sleep(Duration::from_millis(10)) + } + + let cp_tip = env.make_checkpoint_tip(); + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(3) + .discover_keychain(0, spks.clone()); + client.scan(request, 1)? + }; + assert!(scan_update.tx_update.txs.is_empty()); + assert!(scan_update.last_active_indices.is_empty()); + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(4) + .discover_keychain(0, spks.clone()); + client.scan(request, 1)? + }; + assert_eq!( + scan_update.tx_update.txs.first().unwrap().compute_txid(), + txid_4th_addr + ); + assert_eq!(scan_update.last_active_indices[&0], 3); + + let txid_last_addr = env + .bitcoind + .client + .send_to_address(&addresses[addresses.len() - 1], Amount::from_sat(10000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().unwrap() < 104 { + sleep(Duration::from_millis(10)) + } + + let scan_update = { + let request = ScanRequest::::builder() + .chain_tip(cp_tip.clone()) + .stop_gap(6) + .discover_keychain(0, spks.clone()); + client.scan(request, 1)? + }; + let txs: HashSet<_> = scan_update + .tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect(); + assert_eq!(txs, [txid_4th_addr, txid_last_addr].into()); + assert_eq!(scan_update.last_active_indices[&0], 9); + + Ok(()) +} + +#[test] +fn test_scan_revealed_scripts_beyond_stop_gap() -> anyhow::Result<()> { + const KEYCHAIN: u32 = 0; + const REVEALED_INDEX: u32 = 9; + const STOP_GAP: usize = 3; + + let env = TestEnv::new()?; + let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap()); + let client = Builder::new(base_url.as_str()).build_blocking(); + + let mut indexer = KeychainTxOutIndex::new(0, true); + let _ = indexer + .insert_descriptor(KEYCHAIN, common::parse_descriptor(DESCRIPTORS[3])) + .expect("descriptor must insert"); + let _ = indexer + .reveal_to_target(KEYCHAIN, REVEALED_INDEX) + .expect("keychain must exist"); + assert_eq!(indexer.last_revealed_index(KEYCHAIN), Some(REVEALED_INDEX)); + + let (_, revealed_spk) = indexer + .revealed_keychain_spks(KEYCHAIN) + .last() + .expect("revealed spk must exist"); + let revealed_addr = Address::from_script(&revealed_spk, bdk_chain::bitcoin::Network::Regtest)?; + + let _block_hashes = env.mine_blocks(101, None)?; + let txid = env + .bitcoind + .client + .send_to_address(&revealed_addr, Amount::from_sat(10000))? + .txid()?; + let _block_hashes = env.mine_blocks(1, None)?; + while client.get_height().unwrap() < 103 { + sleep(Duration::from_millis(10)) + } + + let cp_tip = env.make_checkpoint_tip(); + let scan_update = { + let request = ScanRequest::builder() + .chain_tip(cp_tip) + .stop_gap(STOP_GAP) + .revealed_spks_from_indexer(&indexer, KEYCHAIN..=KEYCHAIN) + .discover_from_indexer(&indexer); + client.scan(request, 1)? + }; + + let txids: HashSet<_> = scan_update + .tx_update + .txs + .iter() + .map(|tx| tx.compute_txid()) + .collect(); + assert!( + txids.contains(&txid), + "scan must sync transactions on revealed scripts even when discovery stop gap would stop earlier", + ); + assert!( + scan_update.last_active_indices.is_empty(), + "the transaction must be found by explicit sync, not discovery", + ); + + Ok(()) +} diff --git a/crates/esplora/tests/common/mod.rs b/crates/esplora/tests/common/mod.rs index c629c5029c..321d0cc054 100644 --- a/crates/esplora/tests/common/mod.rs +++ b/crates/esplora/tests/common/mod.rs @@ -1,3 +1,5 @@ +use bdk_chain::bitcoin::secp256k1::Secp256k1 as DescriptorSecp256k1; +use bdk_chain::miniscript::{Descriptor, DescriptorPublicKey}; use bdk_core::bitcoin::key::{Secp256k1, UntweakedPublicKey}; use bdk_core::bitcoin::ScriptBuf; @@ -12,3 +14,12 @@ pub fn get_test_spk() -> ScriptBuf { let pk = UntweakedPublicKey::from_slice(PK_BYTES).expect("Must be valid PK"); ScriptBuf::new_p2tr(&secp, pk, None) } + +pub fn parse_descriptor(descriptor: &str) -> Descriptor { + Descriptor::::parse_descriptor( + &DescriptorSecp256k1::signing_only(), + descriptor, + ) + .expect("descriptor must parse") + .0 +}