From c82135c0a62f6c254fc13c145509698e759fb772 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 6 Oct 2025 16:54:39 -0700 Subject: [PATCH 01/10] misc(node): remove unused notification_message_loop --- node/src/client/mod.rs | 7 ++-- node/src/lib.rs | 71 --------------------------------------- node/src/prover/mod.rs | 3 +- node/src/validator/mod.rs | 3 +- 4 files changed, 5 insertions(+), 79 deletions(-) diff --git a/node/src/client/mod.rs b/node/src/client/mod.rs index 3d472fc69b..57f5382670 100644 --- a/node/src/client/mod.rs +++ b/node/src/client/mod.rs @@ -219,6 +219,8 @@ impl> Client { // Initialize the routing. node.initialize_routing().await; + // Pass the node to the signal handler. + let _ = signal_node.set(node.clone()); // Initialize the sync module. node.initialize_sync(); // Initialize solution verification. @@ -227,10 +229,7 @@ impl> Client { node.initialize_deploy_verification(); // Initialize execution verification. node.initialize_execute_verification(); - // Initialize the notification message loop. - node.handles.lock().push(crate::start_notification_message_loop()); - // Pass the node to the signal handler. - let _ = signal_node.set(node.clone()); + // Return the node. Ok(node) } diff --git a/node/src/lib.rs b/node/src/lib.rs index f5740f293c..30ae2b0c5a 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -66,74 +66,3 @@ pub fn log_clean_error(storage_mode: &StorageMode) { } } } - -/// Starts the notification message loop. -pub fn start_notification_message_loop() -> tokio::task::JoinHandle<()> { - // let mut interval = tokio::time::interval(std::time::Duration::from_secs(180)); - tokio::spawn(async move { - // loop { - // interval.tick().await; - // // TODO (howardwu): Swap this with the official message for announcements. - // // info!("{}", notification_message()); - // } - }) -} - -/// Returns the notification message as a string. -pub fn notification_message() -> String { - use colored::Colorize; - - let mut output = String::new(); - output += &r#" - - ================================================================================================== - - 🚧 Welcome to Aleo - Calibration Period 🚧 - - ================================================================================================== - - During the calibration period, the network will be running in limited capacity. - - This calibration period is to ensure validators are stable and ready for mainnet launch. - During this period, the objective is to assess, adjust, and align validators' performance, - stability, and interoperability under varying network conditions. - - Please expect several network resets. With each network reset, software updates will - be performed to address potential bottlenecks, vulnerabilities, and/or inefficiencies, which - will ensure optimal performance for the ecosystem of validators, provers, and developers. - - ================================================================================================== - - Duration: - - Start Date: September 27, 2023 - - End Date: October 18, 2023 (subject to change) - - Participation: - - Node operators are NOT REQUIRED to participate during this calibration period. - - Network Resets: - - IMPORTANT: EXPECT MULTIPLE NETWORK RESETS. - - If participating, BE PREPARED TO RESET YOUR NODE AT ANY TIME. - - When a reset occurs, RUN THE FOLLOWING TO RESET YOUR NODE: - - git checkout mainnet && git pull - - cargo install --locked --path . - - snarkos clean - - snarkos start --nodisplay --client - - Communication: - - Stay ONLINE and MONITOR our Discord and Twitter for community updates. - - Purpose: - - This period is STRICTLY FOR NETWORK CALIBRATION. - - This period is NOT INTENDED for general-purpose usage by developers and provers. - - Incentives: - - There are NO INCENTIVES during this calibration period. - - ================================================================================================== -"# - .white() - .bold(); - - output -} diff --git a/node/src/prover/mod.rs b/node/src/prover/mod.rs index 70e279e4ba..31f534d735 100644 --- a/node/src/prover/mod.rs +++ b/node/src/prover/mod.rs @@ -154,10 +154,9 @@ impl> Prover { node.initialize_routing().await; // Initialize the puzzle. node.initialize_puzzle().await; - // Initialize the notification message loop. - node.handles.lock().push(crate::start_notification_message_loop()); // Pass the node to the signal handler. let _ = signal_node.set(node.clone()); + // Return the node. Ok(node) } diff --git a/node/src/validator/mod.rs b/node/src/validator/mod.rs index c87c7ad436..49e8a53196 100644 --- a/node/src/validator/mod.rs +++ b/node/src/validator/mod.rs @@ -187,10 +187,9 @@ impl> Validator { // Initialize the routing. node.initialize_routing().await; - // Initialize the notification message loop. - node.handles.lock().push(crate::start_notification_message_loop()); // Pass the node to the signal handler. let _ = signal_node.set(node.clone()); + // Return the node. Ok(node) } From dd2bdd50ff57dac040bfef013930ef40199a0022 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 6 Oct 2025 16:59:03 -0700 Subject: [PATCH 02/10] ci: separate test compilation and test execution steps --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f728ef6aa9..1d94c98f5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,7 +137,11 @@ commands: cache_key: v3.3.1-rust-1.88.0-<< parameters.cache_key >>-cache - run: no_output_timeout: 30m - command: cd << parameters.workspace_member >> && RUST_MIN_STACK=67108864 cargo test << parameters.flags >> + command: cd << parameters.workspace_member >> && RUST_MIN_STACK=67108864 cargo test << parameters.flags >> --no-run + - run: + name: "Run Tests" + no_output_timeout: 30m + command: cd << parameters.workspace_member >> && RUST_MIN_STACK=67108864 RUST_BACKTRACE=1 RUST_LOG=snarkos=trace cargo test << parameters.flags >> - clear_environment: cache_key: v3.3.1-rust-1.88.0-<< parameters.cache_key >>-cache From 4b09bf4080356cd7922e049b3f309875e684420e Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 6 Oct 2025 17:25:54 -0700 Subject: [PATCH 03/10] feat(display): show logs in terminal after the UI has stopped --- display/src/lib.rs | 14 ++++++++++++-- display/src/pages/logs.rs | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/display/src/lib.rs b/display/src/lib.rs index 1a03058594..146100861f 100644 --- a/display/src/lib.rs +++ b/display/src/lib.rs @@ -41,6 +41,7 @@ use ratatui::{ }; use std::{ io, + io::Write, thread, time::{Duration, Instant}, }; @@ -91,11 +92,20 @@ impl Display { execute!(terminal.backend_mut(), LeaveAlternateScreen, DisableMouseCapture)?; terminal.show_cursor()?; - // Exit. + // Print any error that may have occurred. if let Err(err) = res { - println!("{err:?}") + eprintln!("{err:?}"); } + // Write any remaining log output to stdout while the node is shutting down. + let mut log_receiver = display.logs.into_log_receiver(); + tokio::spawn(async move { + let mut stdout = io::stdout(); + while let Some(log) = log_receiver.recv().await { + let _ = write!(stdout, "{}", String::from_utf8(log).unwrap_or_default()); + } + }); + Ok(()) } } diff --git a/display/src/pages/logs.rs b/display/src/pages/logs.rs index d9f4bd289c..aba6054499 100644 --- a/display/src/pages/logs.rs +++ b/display/src/pages/logs.rs @@ -72,4 +72,8 @@ impl Logs { .block(Block::default().borders(Borders::ALL).style(header_style()).title("Logs")); f.render_widget(combined_logs, chunks[0]); } + + pub fn into_log_receiver(self) -> mpsc::Receiver> { + self.log_receiver + } } From 6fae7669f1366adcbdd5aa75efdea5c0618c8ad2 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 6 Oct 2025 17:28:09 -0700 Subject: [PATCH 04/10] feat: shut down gracefully --- Cargo.lock | 67 ++++++---- Cargo.toml | 7 +- cli/Cargo.toml | 3 + cli/src/commands/developer/scan.rs | 4 +- cli/src/commands/start.rs | 92 +++++++------- display/Cargo.toml | 3 + display/src/lib.rs | 15 ++- node/Cargo.toml | 6 +- node/bft/Cargo.toml | 3 + node/bft/examples/simple_node.rs | 8 +- node/bft/ledger-service/Cargo.toml | 3 + node/bft/ledger-service/src/ledger.rs | 22 ++-- node/bft/ledger-service/src/translucent.rs | 18 +-- node/bft/src/sync/mod.rs | 7 +- node/bft/tests/common/primary.rs | 5 +- node/bft/tests/common/utils.rs | 6 +- node/cdn/Cargo.toml | 3 + node/cdn/src/blocks.rs | 45 ++++--- node/src/bootstrap_client/mod.rs | 18 +++ node/src/client/mod.rs | 41 +++--- node/src/node.rs | 42 +++++-- node/src/prover/mod.rs | 29 +++-- node/src/traits.rs | 92 +++----------- node/src/validator/mod.rs | 25 ++-- node/tests/common/node.rs | 9 +- utilities/Cargo.toml | 25 ++++ utilities/src/lib.rs | 19 +++ utilities/src/signals.rs | 140 +++++++++++++++++++++ 28 files changed, 475 insertions(+), 282 deletions(-) create mode 100644 utilities/Cargo.toml create mode 100644 utilities/src/lib.rs create mode 100644 utilities/src/signals.rs diff --git a/Cargo.lock b/Cargo.lock index dc7c03c567..052421c554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,9 +599,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.48" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f" dependencies = [ "clap_builder", "clap_derive", @@ -609,9 +609,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.48" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730" dependencies = [ "anstream", "anstyle", @@ -621,9 +621,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -633,9 +633,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "colorchoice" @@ -868,7 +868,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "socket2 0.6.0", + "socket2 0.6.1", "windows-sys 0.59.0", ] @@ -1447,9 +1447,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -1825,7 +1825,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.1", "system-configuration", "tokio", "tower-service", @@ -2922,7 +2922,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.6.0", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -2959,7 +2959,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.0", + "socket2 0.6.1", "tracing", "windows-sys 0.60.2", ] @@ -3155,9 +3155,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a52d8d02cacdb176ef4678de6c052efb4b3da14b78e4db683a4252762be5433" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -3167,9 +3167,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722166aa0d7438abbaa4d5cc2c649dac844e8c56d82fb3d33e9c34b5cd268fc6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -3178,15 +3178,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" -version = "0.12.23" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64 0.22.1", "bytes", @@ -3817,6 +3817,7 @@ dependencies = [ "snarkos-node-cdn", "snarkos-node-metrics", "snarkos-node-rest", + "snarkos-utilities", "snarkvm", "sys-info", "tempfile", @@ -3837,6 +3838,7 @@ dependencies = [ "crossterm 0.29.0", "ratatui", "snarkos-node", + "snarkos-utilities", "snarkvm", "tokio", ] @@ -3857,7 +3859,6 @@ dependencies = [ "locktick", "lru 0.16.1", "num_cpus", - "once_cell", "parking_lot", "paste", "pea2pea", @@ -3873,6 +3874,7 @@ dependencies = [ "snarkos-node-router", "snarkos-node-sync", "snarkos-node-tcp", + "snarkos-utilities", "snarkvm", "time", "tokio", @@ -3921,6 +3923,7 @@ dependencies = [ "snarkos-node-router", "snarkos-node-sync", "snarkos-node-tcp", + "snarkos-utilities", "snarkvm", "test-strategy 0.4.3", "time", @@ -3962,6 +3965,7 @@ dependencies = [ "rand 0.8.5", "rayon", "snarkos-node-metrics", + "snarkos-utilities", "snarkvm", "tokio", "tracing", @@ -3996,6 +4000,7 @@ dependencies = [ "serde", "serde_json", "snarkos-node-metrics", + "snarkos-utilities", "snarkvm", "tokio", "tokio-test", @@ -4182,6 +4187,14 @@ dependencies = [ "tracing", ] +[[package]] +name = "snarkos-utilities" +version = "4.2.1" +dependencies = [ + "tokio", + "tracing", +] + [[package]] name = "snarkvm" version = "4.2.1" @@ -5093,12 +5106,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5517,7 +5530,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.6.0", + "socket2 0.6.1", "tokio-macros", "windows-sys 0.59.0", ] diff --git a/Cargo.toml b/Cargo.toml index 1f968c53b9..7fbe38a618 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,8 @@ members = [ "node/sync", "node/sync/communication-service", "node/sync/locators", - "node/tcp" + "node/tcp", + "utilities", ] [workspace.dependencies.aleo-std] @@ -239,6 +240,10 @@ version = "=4.2.2" path = "node/tcp" version = "=4.2.2" +[workspace.dependencies.snarkos-utilities] +path = "utilities" +version = "=4.2.1" + [[bin]] name = "snarkos" path = "snarkos/main.rs" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d6e781284b..bd3788b39b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -121,6 +121,9 @@ optional = true [dependencies.snarkos-node-rest] workspace = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.snarkvm] workspace = true features = [ "parameters", "circuit", "package" ] diff --git a/cli/src/commands/developer/scan.rs b/cli/src/commands/developer/scan.rs index 3284e82baf..41e5add81a 100644 --- a/cli/src/commands/developer/scan.rs +++ b/cli/src/commands/developer/scan.rs @@ -17,6 +17,8 @@ use super::DEFAULT_ENDPOINT; use crate::helpers::{args::prepare_endpoint, dev::get_development_key}; use snarkos_node_cdn::CDN_BASE_URL; +use snarkos_utilities::SimpleStoppable; + use snarkvm::{ console::network::Network, prelude::{Ciphertext, Field, FromBytes, Plaintext, PrivateKey, Record, ViewKey, block::Block}, @@ -285,7 +287,7 @@ impl Scan { let rt = tokio::runtime::Runtime::new()?; // Create a placeholder shutdown flag. - let _shutdown = Default::default(); + let _shutdown = SimpleStoppable::new(); // Copy endpoint for background task. let endpoint = endpoint.clone(); diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index ccba698f99..e4d8de8575 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -23,6 +23,8 @@ use snarkos_node::{ rest::DEFAULT_REST_PORT, router::{DEFAULT_NODE_PORT, bootstrap_peers, messages::NodeType}, }; +use snarkos_utilities::SignalHandler; + use snarkvm::{ console::{ account::{Address, PrivateKey}, @@ -54,7 +56,10 @@ use std::{ path::PathBuf, sync::{Arc, atomic::AtomicBool}, }; -use tokio::runtime::{self, Runtime}; +use tokio::{ + runtime::{self, Runtime}, + sync::mpsc, +}; use tracing::warn; use ureq::http; @@ -258,7 +263,7 @@ pub struct Start { } impl Start { - /// Starts the snarkOS node. + /// Starts the snarkOS node and blocks until it terminates. pub fn parse(self) -> Result { // Prepare the shutdown flag. let shutdown: Arc = Default::default(); @@ -276,45 +281,30 @@ impl Start { // Initialize the runtime. Self::runtime().block_on(async move { // Error messages. - let node_parse_error = || "Failed to parse node arguments"; - let display_start_error = || "Failed to initialize the display"; + let node_parse_error = || "Failed to start node"; + let signal_handler = SignalHandler::new(); // Clone the configurations. - let mut cli = self.clone(); - // Parse the network. - match cli.network { - MainnetV0::ID => { - // Parse the node from the configurations. - let node = cli.parse_node::(shutdown.clone()).await.with_context(node_parse_error)?; - // If the display is enabled, render the display. - if !cli.nodisplay { - // Initialize the display. - Display::start(node, log_receiver).with_context(display_start_error)?; - } - } - TestnetV0::ID => { - // Parse the node from the configurations. - let node = cli.parse_node::(shutdown.clone()).await.with_context(node_parse_error)?; - // If the display is enabled, render the display. - if !cli.nodisplay { - // Initialize the display. - Display::start(node, log_receiver).with_context(display_start_error)?; - } - } - CanaryV0::ID => { - // Parse the node from the configurations. - let node = cli.parse_node::(shutdown.clone()).await.with_context(node_parse_error)?; - // If the display is enabled, render the display. - if !cli.nodisplay { - // Initialize the display. - Display::start(node, log_receiver).with_context(display_start_error)?; - } - } + let mut self_ = self.clone(); + + // Parse the node arguments, start it, and block until shutdown. + match self_.network { + MainnetV0::ID => self_ + .parse_node::(log_receiver, signal_handler.clone()) + .await + .with_context(node_parse_error)?, + + TestnetV0::ID => self_ + .parse_node::(log_receiver, signal_handler.clone()) + .await + .with_context(node_parse_error)?, + CanaryV0::ID => self_ + .parse_node::(log_receiver, signal_handler.clone()) + .await + .with_context(node_parse_error)?, _ => panic!("Invalid network ID specified"), }; - // Note: Do not move this. The pending await must be here otherwise - // other snarkOS commands will not exit. - std::future::pending::<()>().await; + Ok(String::new()) }) } @@ -575,9 +565,9 @@ impl Start { } } - /// Returns the node type corresponding to the given configurations. + /// Start the node and blocks until it terminates. #[rustfmt::skip] - async fn parse_node(&mut self, shutdown: Arc) -> Result> { + async fn parse_node(&mut self, log_receiver: mpsc::Receiver>, signal_handler: Arc) -> Result<()> { if !self.nobanner { // Print the welcome banner. println!("{}", crate::helpers::welcome_message()); @@ -716,22 +706,28 @@ impl Start { } }; - // TODO(kaimast): start the display earlier and show sync progress. if !self.nodisplay && !self.nocdn { println!("🪧 The terminal UI will not start until the node has finished syncing from the CDN. If this step takes too long, consider restarting with `--nodisplay`."); } // Initialize the node. - match node_type { - NodeType::Validator => Node::new_validator(node_ip, self.bft, rest_ip, self.rest_rps, account, &trusted_peers, &trusted_validators, genesis, cdn, storage_mode, self.allow_external_peers, dev_txs, self.dev, shutdown.clone()).await, - NodeType::Prover => Node::new_prover(node_ip, account, &trusted_peers, genesis, storage_mode, self.dev, shutdown.clone()).await, - NodeType::Client => Node::new_client(node_ip, rest_ip, self.rest_rps, account, &trusted_peers, genesis, cdn, storage_mode, self.rotate_external_peers, self.dev, shutdown).await, + let node = match node_type { + NodeType::Validator => Node::new_validator(node_ip, self.bft, rest_ip, self.rest_rps, account, &trusted_peers, &trusted_validators, genesis, cdn, storage_mode, self.allow_external_peers, dev_txs, self.dev, signal_handler.clone()).await, + NodeType::Prover => Node::new_prover(node_ip, account, &trusted_peers, genesis, storage_mode, self.dev, signal_handler.clone()).await, + NodeType::Client => Node::new_client(node_ip, rest_ip, self.rest_rps, account, &trusted_peers, genesis, cdn, storage_mode, self.rotate_external_peers, self.dev, signal_handler.clone()).await, NodeType::BootstrapClient => Node::new_bootstrap_client(node_ip, account, *genesis.header(), self.dev).await, + }?; + + if !self.nodisplay { + Display::start(node.clone(), log_receiver, signal_handler.clone()).with_context(|| "Failed to start the display")?; } + + node.wait_for_signals(&signal_handler).await; + Ok(()) } - /// Returns a runtime for the node. + /// Starts a rayon thread pool and tokio runtime for the node, and returns the tokio `Runtime`. fn runtime() -> Runtime { // Retrieve the number of cores. let num_cores = num_cpus::get(); @@ -742,14 +738,16 @@ impl Start { let (num_tokio_worker_threads, max_tokio_blocking_threads, num_rayon_cores_global) = (2 * num_cores, 512, num_cores); - // Initialize the parallelization parameters. + // Set up the rayon thread pool. + // A custom panic handler is not needed here, as rayon propagates the panic to the calling thread by default (except for `rayon::spawn` which we do not use). rayon::ThreadPoolBuilder::new() .stack_size(8 * 1024 * 1024) .num_threads(num_rayon_cores_global) .build_global() .unwrap(); - // Initialize the runtime configuration. + // Set up the tokio Runtime. + // TODO(kaimast): set up a panic handler here for each worker thread once [`tokio::runtime::Builder::unhandled_panic`](https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.unhandled_panic) is stabilized. runtime::Builder::new_multi_thread() .enable_all() .thread_stack_size(8 * 1024 * 1024) diff --git a/display/Cargo.toml b/display/Cargo.toml index 7d94718ff9..873c19dd83 100644 --- a/display/Cargo.toml +++ b/display/Cargo.toml @@ -28,6 +28,9 @@ version = "0.29" [dependencies.snarkos-node] workspace = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.snarkvm] workspace = true diff --git a/display/src/lib.rs b/display/src/lib.rs index 146100861f..1dda3193b9 100644 --- a/display/src/lib.rs +++ b/display/src/lib.rs @@ -22,6 +22,8 @@ mod tabs; use tabs::Tabs; use snarkos_node::Node; +use snarkos_utilities::Stoppable; + use snarkvm::prelude::Network; use anyhow::Result; @@ -42,6 +44,7 @@ use ratatui::{ use std::{ io, io::Write, + sync::Arc, thread, time::{Duration, Instant}, }; @@ -68,7 +71,7 @@ fn content_style() -> Style { impl Display { /// Initializes a new display. - pub fn start(node: Node, log_receiver: Receiver>) -> Result<()> { + pub fn start(node: Node, log_receiver: Receiver>, stoppable: Arc) -> Result<()> { // Initialize the display. enable_raw_mode()?; let mut stdout = io::stdout(); @@ -85,7 +88,7 @@ impl Display { }; // Render the display. - let res = display.render(&mut terminal); + let res = display.render(&mut terminal, stoppable); // Terminate the display. disable_raw_mode()?; @@ -112,7 +115,7 @@ impl Display { impl Display { /// Renders the display. - fn render(&mut self, terminal: &mut Terminal) -> io::Result<()> { + fn render(&mut self, terminal: &mut Terminal, stoppable: Arc) -> io::Result<()> { let mut last_tick = Instant::now(); loop { terminal.draw(|f| self.draw(f))?; @@ -124,11 +127,7 @@ impl Display { if let Event::Key(key) = event::read()? { match key.code { KeyCode::Esc => { - // // TODO (howardwu): @ljedrz to implement a wrapping scope for Display within Node/Server. - // #[allow(unused_must_use)] - // { - // self.node.shut_down(); - // } + stoppable.stop(); return Ok(()); } KeyCode::Left => self.tabs.previous(), diff --git a/node/Cargo.toml b/node/Cargo.toml index d9cf8407a4..8156299761 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -81,9 +81,6 @@ workspace = true [dependencies.num_cpus] workspace = true -[dependencies.once_cell] -workspace = true - [dependencies.parking_lot] workspace = true @@ -125,6 +122,9 @@ workspace = true [dependencies.snarkos-node-tcp] workspace = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.snarkvm] workspace = true diff --git a/node/bft/Cargo.toml b/node/bft/Cargo.toml index 8eeb1434d2..ce119f4ebf 100644 --- a/node/bft/Cargo.toml +++ b/node/bft/Cargo.toml @@ -120,6 +120,9 @@ workspace = true [dependencies.snarkos-node-tcp] workspace = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.snarkvm] workspace = true features = [ "utilities" ] diff --git a/node/bft/examples/simple_node.rs b/node/bft/examples/simple_node.rs index 6001617668..b71e712d61 100644 --- a/node/bft/examples/simple_node.rs +++ b/node/bft/examples/simple_node.rs @@ -19,7 +19,6 @@ extern crate tracing; #[cfg(feature = "metrics")] extern crate snarkos_node_metrics as metrics; -use aleo_std::StorageMode; use snarkos_account::Account; use snarkos_node_bft::{ BFT, @@ -30,6 +29,9 @@ use snarkos_node_bft::{ use snarkos_node_bft_ledger_service::TranslucentLedgerService; use snarkos_node_bft_storage_service::BFTMemoryService; use snarkos_node_sync::BlockSync; +use snarkos_utilities::SimpleStoppable; + +use aleo_std::StorageMode; use snarkvm::{ console::{account::PrivateKey, algorithms::BHP256, types::Address}, ledger::{ @@ -64,7 +66,7 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, str::FromStr, - sync::{Arc, Mutex, OnceLock, atomic::AtomicBool}, + sync::{Arc, Mutex, OnceLock}, }; use tokio::{net::TcpListener, sync::oneshot}; use tracing_subscriber::{ @@ -221,7 +223,7 @@ fn create_ledger( } let mut rng = TestRng::default(); let gen_ledger = genesis_ledger(*gen_key, committee.clone(), balances.clone(), node_id, &mut rng); - Arc::new(TranslucentLedgerService::new(gen_ledger, Arc::new(AtomicBool::new(false)))) + Arc::new(TranslucentLedgerService::new(gen_ledger, SimpleStoppable::new())) } pub type CurrentLedger = Ledger>; diff --git a/node/bft/ledger-service/Cargo.toml b/node/bft/ledger-service/Cargo.toml index d468168c54..4c84ed3d8a 100644 --- a/node/bft/ledger-service/Cargo.toml +++ b/node/bft/ledger-service/Cargo.toml @@ -48,6 +48,9 @@ optional = true workspace = true optional = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.parking_lot] workspace = true optional = true diff --git a/node/bft/ledger-service/src/ledger.rs b/node/bft/ledger-service/src/ledger.rs index 503d911c0f..2189ac319d 100644 --- a/node/bft/ledger-service/src/ledger.rs +++ b/node/bft/ledger-service/src/ledger.rs @@ -14,6 +14,9 @@ // limitations under the License. use crate::{LedgerService, fmt_id, spawn_blocking}; + +use snarkos_utilities::Stoppable; + use snarkvm::{ ledger::{ Ledger, @@ -49,16 +52,7 @@ use parking_lot::RwLock; #[cfg(not(feature = "serial"))] use rayon::prelude::*; -use std::{ - collections::BTreeMap, - fmt, - io::Read, - ops::Range, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, -}; +use std::{collections::BTreeMap, fmt, io::Read, ops::Range, sync::Arc}; /// The capacity of the cache holding the highest blocks. const BLOCK_CACHE_SIZE: usize = 10; @@ -69,14 +63,14 @@ pub struct CoreLedgerService> { ledger: Ledger, block_cache: Arc>>>, latest_leader: Arc)>>>, - shutdown: Arc, + stoppable: Arc, } impl> CoreLedgerService { /// Initializes a new core ledger service. - pub fn new(ledger: Ledger, shutdown: Arc) -> Self { + pub fn new(ledger: Ledger, stoppable: Arc) -> Self { let block_cache = Arc::new(RwLock::new(BTreeMap::new())); - Self { ledger, block_cache, latest_leader: Default::default(), shutdown } + Self { ledger, block_cache, latest_leader: Default::default(), stoppable } } } @@ -374,7 +368,7 @@ impl> LedgerService for CoreLedgerService< #[cfg(feature = "ledger-write")] fn advance_to_next_block(&self, block: &Block) -> Result<()> { // If the Ctrl-C handler registered the signal, then skip advancing to the next block. - if self.shutdown.load(Ordering::Acquire) { + if self.stoppable.is_stopped() { bail!("Skipping advancing to block {} - The node is shutting down", block.height()); } // Advance to the next block. diff --git a/node/bft/ledger-service/src/translucent.rs b/node/bft/ledger-service/src/translucent.rs index 7c3a5fcd26..b4ffb988df 100644 --- a/node/bft/ledger-service/src/translucent.rs +++ b/node/bft/ledger-service/src/translucent.rs @@ -14,8 +14,9 @@ // limitations under the License. use crate::{CoreLedgerService, LedgerService}; -use async_trait::async_trait; -use indexmap::IndexMap; + +use snarkos_utilities::Stoppable; + use snarkvm::{ ledger::{ Ledger, @@ -27,11 +28,10 @@ use snarkvm::{ }, prelude::{Address, ConsensusVersion, Field, Network, Result, narwhal::BatchCertificate}, }; -use std::{ - fmt, - ops::Range, - sync::{Arc, atomic::AtomicBool}, -}; + +use async_trait::async_trait; +use indexmap::IndexMap; +use std::{fmt, ops::Range, sync::Arc}; pub struct TranslucentLedgerService> { inner: CoreLedgerService, @@ -46,8 +46,8 @@ impl> fmt::Debug for TranslucentLedgerService impl> TranslucentLedgerService { /// Initializes a new ledger service wrapper. - pub fn new(ledger: Ledger, shutdown: Arc) -> Self { - Self { inner: CoreLedgerService::new(ledger, shutdown) } + pub fn new(ledger: Ledger, stoppable: Arc) -> Self { + Self { inner: CoreLedgerService::new(ledger, stoppable) } } } diff --git a/node/bft/src/sync/mod.rs b/node/bft/src/sync/mod.rs index eff35bf4db..b7f2dd1ae9 100644 --- a/node/bft/src/sync/mod.rs +++ b/node/bft/src/sync/mod.rs @@ -995,6 +995,7 @@ mod tests { use snarkos_account::Account; use snarkos_node_sync::BlockSync; + use snarkos_utilities::SimpleStoppable; use snarkvm::{ console::{ account::{Address, PrivateKey}, @@ -1046,7 +1047,7 @@ mod tests { // Initialize the ledger with the genesis block. let ledger = CurrentLedger::load(genesis.clone(), StorageMode::new_test(None)).unwrap(); // Initialize the ledger. - let core_ledger = Arc::new(CoreLedgerService::new(ledger.clone(), Default::default())); + let core_ledger = Arc::new(CoreLedgerService::new(ledger.clone(), SimpleStoppable::new())); // Sample 5 rounds of batch certificates starting at the genesis round from a static set of 4 authors. let (round_to_certificates_map, committee) = { @@ -1220,7 +1221,7 @@ mod tests { let storage_mode = StorageMode::new_test(None); let syncing_ledger = Arc::new(CoreLedgerService::new( CurrentLedger::load(genesis, storage_mode.clone()).unwrap(), - Default::default(), + SimpleStoppable::new(), )); // Initialize the gateway. let gateway = @@ -1273,7 +1274,7 @@ mod tests { // Initialize the ledger with the genesis block. let ledger = CurrentLedger::load(genesis.clone(), StorageMode::new_test(None)).unwrap(); // Initialize the ledger. - let core_ledger = Arc::new(CoreLedgerService::new(ledger.clone(), Default::default())); + let core_ledger = Arc::new(CoreLedgerService::new(ledger.clone(), SimpleStoppable::new())); // Sample rounds of batch certificates starting at the genesis round from a static set of 4 authors. let (round_to_certificates_map, committee) = { // Initialize the committee. diff --git a/node/bft/tests/common/primary.rs b/node/bft/tests/common/primary.rs index 1e29f26d99..d9e4f5e0d6 100644 --- a/node/bft/tests/common/primary.rs +++ b/node/bft/tests/common/primary.rs @@ -18,6 +18,7 @@ use crate::common::{ TranslucentLedgerService, utils::{fire_unconfirmed_solutions, fire_unconfirmed_transactions, initialize_logger}, }; + use snarkos_account::Account; use snarkos_node_bft::{ BFT, @@ -29,6 +30,8 @@ use snarkos_node_bft::{ use snarkos_node_bft_storage_service::BFTMemoryService; use snarkos_node_router::PeerPoolHandling; use snarkos_node_sync::BlockSync; +use snarkos_utilities::SimpleStoppable; + use snarkvm::{ console::{ account::{Address, PrivateKey}, @@ -160,7 +163,7 @@ impl TestNetwork { for (id, account) in accounts.into_iter().enumerate() { let gen_ledger = genesis_ledger(gen_key, committee.clone(), balances.clone(), bonded_balances.clone(), &mut rng); - let ledger = Arc::new(TranslucentLedgerService::new(gen_ledger, Default::default())); + let ledger = Arc::new(TranslucentLedgerService::new(gen_ledger, SimpleStoppable::new())); let storage = Storage::new( ledger.clone(), Arc::new(BFTMemoryService::new()), diff --git a/node/bft/tests/common/utils.rs b/node/bft/tests/common/utils.rs index a269a41787..e61e9d5b75 100644 --- a/node/bft/tests/common/utils.rs +++ b/node/bft/tests/common/utils.rs @@ -22,7 +22,9 @@ use snarkos_node_bft::{ helpers::{PrimarySender, Storage}, }; -use snarkos_node_bft_storage_service::BFTMemoryService; +use snarkos_node_bft::storage_service::BFTMemoryService; +use snarkos_utilities::SimpleStoppable; + use snarkvm::{ console::account::Address, ledger::{ @@ -202,7 +204,7 @@ pub fn sample_ledger( let gen_ledger = primary::genesis_ledger(gen_key, committee.clone(), balances.clone(), bonded_balances.clone(), rng); - Arc::new(TranslucentLedgerService::new(gen_ledger, Default::default())) + Arc::new(TranslucentLedgerService::new(gen_ledger, SimpleStoppable::new())) } /// Samples a new storage with the given ledger. diff --git a/node/cdn/Cargo.toml b/node/cdn/Cargo.toml index e626730108..4771bc25c7 100644 --- a/node/cdn/Cargo.toml +++ b/node/cdn/Cargo.toml @@ -45,6 +45,9 @@ workspace = true optional = true features = [ "metrics" ] +[dependencies.snarkos-utilities] +workspace = true + [dependencies.rayon] workspace = true optional = true diff --git a/node/cdn/src/blocks.rs b/node/cdn/src/blocks.rs index 4b489d195e..ab9a13db10 100644 --- a/node/cdn/src/blocks.rs +++ b/node/cdn/src/blocks.rs @@ -17,6 +17,8 @@ // https://github.com/rust-lang/rust-clippy/issues/6446 #![allow(clippy::await_holding_lock)] +use snarkos_utilities::Stoppable; + use snarkvm::prelude::{ Deserialize, DeserializeOwned, @@ -87,11 +89,11 @@ impl CdnBlockSync { pub fn new>( base_url: http::Uri, ledger: Ledger, - shutdown: Arc, + stoppable: Arc, ) -> Self { let task = { let base_url = base_url.clone(); - tokio::spawn(async move { Self::worker(base_url, ledger, shutdown).await }) + tokio::spawn(async move { Self::worker(base_url, ledger, stoppable).await }) }; debug!("Started sync from CDN at {base_url}"); @@ -119,13 +121,13 @@ impl CdnBlockSync { async fn worker>( base_url: http::Uri, ledger: Ledger, - shutdown: Arc, + stoppable: Arc, ) -> SyncResult { // Fetch the node height. let start_height = ledger.latest_height() + 1; // Load the blocks from the CDN into the ledger. let ledger_clone = ledger.clone(); - let result = load_blocks(&base_url, start_height, None, shutdown, move |block: Block| { + let result = load_blocks(&base_url, start_height, None, stoppable, move |block: Block| { ledger_clone.advance_to_next_block(&block) }) .await; @@ -172,7 +174,7 @@ pub async fn load_blocks( base_url: &http::Uri, start_height: u32, end_height: Option, - shutdown: Arc, + stoppable: Arc, process: impl FnMut(Block) -> Result<()> + Clone + Send + Sync + 'static, ) -> Result { // Create a Client to maintain a connection pool throughout the sync. @@ -225,16 +227,19 @@ pub async fn load_blocks( // Spawn a background task responsible for concurrent downloads. let pending_blocks_clone = pending_blocks.clone(); let base_url = base_url.to_owned(); - let shutdown_clone = shutdown.clone(); - tokio::spawn(async move { - download_block_bundles(client, &base_url, cdn_start, cdn_end, pending_blocks_clone, shutdown_clone).await; - }); + + { + let stoppable = stoppable.clone(); + tokio::spawn(async move { + download_block_bundles(client, &base_url, cdn_start, cdn_end, pending_blocks_clone, stoppable).await; + }); + } // A loop for inserting the pending blocks into the ledger. let mut current_height = start_height.saturating_sub(1); while current_height < end_height - 1 { // If we are instructed to shut down, abort. - if shutdown.load(Ordering::Acquire) { + if stoppable.is_stopped() { info!("Stopping block sync at {} - shutting down", current_height); // We can shut down cleanly from here, as the node hasn't been started yet. std::process::exit(0); @@ -269,12 +274,12 @@ pub async fn load_blocks( // Attempt to advance the ledger using the CDN block bundle. let mut process_clone = process.clone(); - let shutdown_clone = shutdown.clone(); + let stoppable_clone = stoppable.clone(); current_height = tokio::task::spawn_blocking(move || { threadpool.install(|| { for block in next_blocks.into_iter().filter(|b| (start_height..end_height).contains(&b.height())) { // If we are instructed to shut down, abort. - if shutdown_clone.load(Ordering::Relaxed) { + if stoppable_clone.is_stopped() { info!("Stopping block sync at {} - the node is shutting down", current_height); // We can shut down cleanly from here, as the node hasn't been started yet. std::process::exit(0); @@ -314,7 +319,7 @@ async fn download_block_bundles( cdn_start: u32, cdn_end: u32, pending_blocks: Arc>>>, - shutdown: Arc, + stoppable: Arc, ) { // Keep track of the number of concurrent requests. let active_requests: Arc = Default::default(); @@ -322,7 +327,7 @@ async fn download_block_bundles( let mut start = cdn_start; while start < cdn_end - 1 { // If we are instructed to shut down, stop downloading. - if shutdown.load(Ordering::Acquire) { + if stoppable.is_stopped() { break; } @@ -356,7 +361,7 @@ async fn download_block_bundles( let base_url_clone = base_url.clone(); let pending_blocks_clone = pending_blocks.clone(); let active_requests_clone = active_requests.clone(); - let shutdown_clone = shutdown.clone(); + let stoppable_clone = stoppable.clone(); tokio::spawn(async move { // Increment the number of active requests. active_requests_clone.fetch_add(1, Ordering::Relaxed); @@ -392,7 +397,7 @@ async fn download_block_bundles( attempts += 1; if attempts > MAXIMUM_REQUEST_ATTEMPTS { warn!("Maximum number of requests to {blocks_url} reached - shutting down..."); - shutdown_clone.store(true, Ordering::Relaxed); + stoppable_clone.stop(); break; } tokio::time::sleep(Duration::from_secs(attempts as u64 * 10)).await; @@ -553,8 +558,10 @@ fn log_progress( #[cfg(test)] mod tests { - use super::{BLOCKS_PER_FILE, CDN_BASE_URL, cdn_height, log_progress}; - use crate::load_blocks; + use super::{BLOCKS_PER_FILE, CDN_BASE_URL, cdn_height, load_blocks, log_progress}; + + use snarkos_utilities::SimpleStoppable; + use snarkvm::prelude::{MainnetV0, block::Block}; use http::Uri; @@ -576,7 +583,7 @@ mod tests { let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { let completed_height = - load_blocks(&testnet_cdn_url, start, end, Default::default(), process).await.unwrap(); + load_blocks(&testnet_cdn_url, start, end, SimpleStoppable::new(), process).await.unwrap(); assert_eq!(blocks.read().len(), expected); if expected > 0 { assert_eq!(blocks.read().last().unwrap().height(), completed_height); diff --git a/node/src/bootstrap_client/mod.rs b/node/src/bootstrap_client/mod.rs index 2f2b6cbd44..dd5d6d011f 100644 --- a/node/src/bootstrap_client/mod.rs +++ b/node/src/bootstrap_client/mod.rs @@ -23,6 +23,7 @@ use crate::{ }; use snarkos_account::Account; use snarkos_node_tcp::{P2P, protocols::*}; +use snarkos_utilities::SignalHandler; use snarkvm::{ ledger::committee::Committee, prelude::{Address, Field, Header, Network, PrivateKey, ViewKey}, @@ -205,4 +206,21 @@ impl BootstrapClient { let _ = shutdown_tx.send(()); } } + + /// Blocks until a shutdown signal was received or manual shutdown was triggered. + pub async fn wait_for_signals(&self, handler: &SignalHandler) { + handler.wait_for_signals().await; + + warn!("=========================================================================================="); + warn!("⚠️ Attention - Starting the graceful shutdown procedure (ETA: 30 seconds)..."); + warn!("⚠️ Attention - To avoid DATA CORRUPTION, do NOT interrupt snarkOS (or press Ctrl+C again)"); + warn!("⚠️ Attention - Please wait until the shutdown gracefully completes (ETA: 30 seconds)"); + warn!("=========================================================================================="); + + // If the node is already initialized, then shut it down. + self.shut_down().await; + + // A best-effort attempt to let any ongoing activity conclude. + tokio::time::sleep(Duration::from_secs(3)).await; + } } diff --git a/node/src/client/mod.rs b/node/src/client/mod.rs index 57f5382670..e1851c5031 100644 --- a/node/src/client/mod.rs +++ b/node/src/client/mod.rs @@ -15,11 +15,13 @@ mod router; -use crate::traits::NodeInterface; +use crate::{ + bft::{events::DataBlocks, helpers::fmt_id, ledger_service::CoreLedgerService}, + cdn::CdnBlockSync, + traits::NodeInterface, +}; use snarkos_account::Account; -use snarkos_node_bft::{events::DataBlocks, helpers::fmt_id, ledger_service::CoreLedgerService}; -use snarkos_node_cdn::CdnBlockSync; use snarkos_node_rest::Rest; use snarkos_node_router::{ Heartbeat, @@ -34,6 +36,8 @@ use snarkos_node_tcp::{ P2P, protocols::{Disconnect, Handshake, OnConnect, Reading}, }; +use snarkos_utilities::{SignalHandler, Stoppable}; + use snarkvm::{ console::network::Network, ledger::{ @@ -60,7 +64,6 @@ use std::{ sync::{ Arc, atomic::{ - AtomicBool, AtomicUsize, Ordering::{Acquire, Relaxed}, }, @@ -121,10 +124,10 @@ pub struct Client> { num_verifying_executions: Arc, /// The spawned handles. handles: Arc>>>, - /// The shutdown signal. - shutdown: Arc, /// Keeps track of sending pings. ping: Arc>, + /// The signal handling logic. + signal_handler: Arc, } impl> Client { @@ -140,16 +143,13 @@ impl> Client { storage_mode: StorageMode, rotate_external_peers: bool, dev: Option, - shutdown: Arc, + signal_handler: Arc, ) -> Result { - // Initialize the signal handler. - let signal_node = Self::handle_signals(shutdown.clone()); - // Initialize the ledger. let ledger = Ledger::::load(genesis.clone(), storage_mode.clone())?; // Initialize the ledger service. - let ledger_service = Arc::new(CoreLedgerService::::new(ledger.clone(), shutdown.clone())); + let ledger_service = Arc::new(CoreLedgerService::::new(ledger.clone(), signal_handler.clone())); // Determine if the client should allow external peers. let allow_external_peers = true; @@ -191,13 +191,13 @@ impl> Client { num_verifying_deploys: Default::default(), num_verifying_executions: Default::default(), handles: Default::default(), - shutdown: shutdown.clone(), + signal_handler: signal_handler.clone(), }; // Perform sync with CDN (if enabled). let cdn_sync = cdn.map(|base_url| { trace!("CDN sync is enabled"); - Arc::new(CdnBlockSync::new(base_url, ledger.clone(), shutdown)) + Arc::new(CdnBlockSync::new(base_url, ledger.clone(), signal_handler)) }); // Initialize the REST server. @@ -219,8 +219,6 @@ impl> Client { // Initialize the routing. node.initialize_routing().await; - // Pass the node to the signal handler. - let _ = signal_node.set(node.clone()); // Initialize the sync module. node.initialize_sync(); // Initialize solution verification. @@ -261,11 +259,9 @@ impl> Client { // Start the block request generation loop (outgoing). let self_ = self.clone(); self.spawn(async move { - while !self_.shutdown.load(std::sync::atomic::Ordering::Acquire) { + while !self_.signal_handler.is_stopped() { // Perform the sync routine. self_.try_issuing_block_requests().await; - - // Rate limiting happens in [`Self::send_block_requests`] and no additional sleeps are needed here } info!("Stopped block request generation"); @@ -274,7 +270,7 @@ impl> Client { // Start the block response processing loop (incoming). let self_ = self.clone(); self.spawn(async move { - while !self_.shutdown.load(std::sync::atomic::Ordering::Acquire) { + while !self_.signal_handler.is_stopped() { // Wait until there is something to do or until the timeout. let _ = timeout(Self::MAX_SYNC_INTERVAL, self_.sync.wait_for_block_responses()).await; @@ -378,7 +374,7 @@ impl> Client { self.spawn(async move { loop { // If the Ctrl-C handler registered the signal, stop the node. - if node.shutdown.load(Acquire) { + if node.signal_handler.is_stopped() { info!("Shutting down solution verification"); break; } @@ -452,7 +448,7 @@ impl> Client { self.spawn(async move { loop { // If the Ctrl-C handler registered the signal, stop the node. - if node.shutdown.load(Acquire) { + if node.signal_handler.is_stopped() { info!("Shutting down deployment verification"); break; } @@ -520,7 +516,7 @@ impl> Client { self.spawn(async move { loop { // If the Ctrl-C handler registered the signal, stop the node. - if node.shutdown.load(Acquire) { + if node.signal_handler.is_stopped() { info!("Shutting down execution verification"); break; } @@ -598,7 +594,6 @@ impl> NodeInterface for Client { // Shut down the node. trace!("Shutting down the node..."); - self.shutdown.store(true, std::sync::atomic::Ordering::Release); // Abort the tasks. trace!("Shutting down the client..."); diff --git a/node/src/node.rs b/node/src/node.rs index c855bb95b9..1fa5d3cf5b 100644 --- a/node/src/node.rs +++ b/node/src/node.rs @@ -13,9 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{BootstrapClient, Client, Prover, Validator, traits::NodeInterface}; +use crate::{ + BootstrapClient, + Client, + Prover, + Validator, + router::{Outbound, Peer, PeerPoolHandling, messages::NodeType}, + traits::NodeInterface, +}; + use snarkos_account::Account; -use snarkos_node_router::{Outbound, Peer, PeerPoolHandling, messages::NodeType}; +use snarkos_utilities::SignalHandler; + use snarkvm::prelude::{ Address, Header, @@ -29,15 +38,12 @@ use snarkvm::prelude::{ use aleo_std::StorageMode; use anyhow::Result; + #[cfg(feature = "locktick")] use locktick::parking_lot::RwLock; #[cfg(not(feature = "locktick"))] use parking_lot::RwLock; -use std::{ - collections::HashMap, - net::SocketAddr, - sync::{Arc, atomic::AtomicBool}, -}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; #[derive(Clone)] pub enum Node { @@ -67,7 +73,7 @@ impl Node { allow_external_peers: bool, dev_txs: bool, dev: Option, - shutdown: Arc, + signal_handler: Arc, ) -> Result { Ok(Self::Validator(Arc::new( Validator::new( @@ -84,7 +90,7 @@ impl Node { allow_external_peers, dev_txs, dev, - shutdown, + signal_handler, ) .await?, ))) @@ -98,10 +104,10 @@ impl Node { genesis: Block, storage_mode: StorageMode, dev: Option, - shutdown: Arc, + signal_handler: Arc, ) -> Result { Ok(Self::Prover(Arc::new( - Prover::new(node_ip, account, trusted_peers, genesis, storage_mode, dev, shutdown).await?, + Prover::new(node_ip, account, trusted_peers, genesis, storage_mode, dev, signal_handler).await?, ))) } @@ -117,7 +123,7 @@ impl Node { storage_mode: StorageMode, rotate_external_peers: bool, dev: Option, - shutdown: Arc, + signal_handler: Arc, ) -> Result { Ok(Self::Client(Arc::new( Client::new( @@ -131,7 +137,7 @@ impl Node { storage_mode, rotate_external_peers, dev, - shutdown, + signal_handler, ) .await?, ))) @@ -258,4 +264,14 @@ impl Node { Self::BootstrapClient(node) => node.shut_down().await, } } + + /// Waits until the node receives a signal. + pub async fn wait_for_signals(&self, signal_handler: &SignalHandler) { + match self { + Self::Validator(node) => node.wait_for_signals(signal_handler).await, + Self::Prover(node) => node.wait_for_signals(signal_handler).await, + Self::Client(node) => node.wait_for_signals(signal_handler).await, + Self::BootstrapClient(node) => node.wait_for_signals(signal_handler).await, + } + } } diff --git a/node/src/prover/mod.rs b/node/src/prover/mod.rs index 31f534d735..5c44e81d1a 100644 --- a/node/src/prover/mod.rs +++ b/node/src/prover/mod.rs @@ -15,9 +15,13 @@ mod router; -use crate::traits::NodeInterface; +use crate::{ + bft::ledger_service::ProverLedgerService, + sync::{BlockSync, Ping}, + traits::NodeInterface, +}; + use snarkos_account::Account; -use snarkos_node_bft::ledger_service::ProverLedgerService; use snarkos_node_router::{ Heartbeat, Inbound, @@ -27,11 +31,12 @@ use snarkos_node_router::{ Routing, messages::{Message, NodeType, UnconfirmedSolution}, }; -use snarkos_node_sync::{BlockSync, Ping}; use snarkos_node_tcp::{ P2P, protocols::{Disconnect, Handshake, OnConnect, Reading}, }; +use snarkos_utilities::{SignalHandler, Stoppable}; + use snarkvm::{ ledger::narwhal::Data, prelude::{ @@ -57,7 +62,7 @@ use std::{ net::SocketAddr, sync::{ Arc, - atomic::{AtomicBool, AtomicU8, Ordering}, + atomic::{AtomicU8, Ordering}, }, }; use tokio::task::JoinHandle; @@ -83,10 +88,10 @@ pub struct Prover> { max_puzzle_instances: u8, /// The spawned handles. handles: Arc>>>, - /// The shutdown signal. - shutdown: Arc, /// Keeps track of sending pings. ping: Arc>, + /// The signal handling logic. + signal_handler: Arc, /// PhantomData. _phantom: PhantomData, } @@ -100,11 +105,8 @@ impl> Prover { genesis: Block, storage_mode: StorageMode, dev: Option, - shutdown: Arc, + signal_handler: Arc, ) -> Result { - // Initialize the signal handler. - let signal_node = Self::handle_signals(shutdown.clone()); - // Initialize the ledger service. let ledger_service = Arc::new(ProverLedgerService::new()); // Determine if the prover should allow external peers. @@ -147,15 +149,13 @@ impl> Prover { max_puzzle_instances: u8::try_from(max_puzzle_instances)?, handles: Default::default(), ping, - shutdown, + signal_handler, _phantom: Default::default(), }; // Initialize the routing. node.initialize_routing().await; // Initialize the puzzle. node.initialize_puzzle().await; - // Pass the node to the signal handler. - let _ = signal_node.set(node.clone()); // Return the node. Ok(node) @@ -174,7 +174,6 @@ impl> NodeInterface for Prover { // Shut down the puzzle. debug!("Shutting down the puzzle..."); - self.shutdown.store(true, Ordering::Release); // Abort the tasks. debug!("Shutting down the prover..."); @@ -245,7 +244,7 @@ impl> Prover { } // If the Ctrl-C handler registered the signal, stop the prover. - if self.shutdown.load(Ordering::Acquire) { + if self.signal_handler.is_stopped() { debug!("Shutting down the puzzle..."); break; } diff --git a/node/src/traits.rs b/node/src/traits.rs index b84786707d..d227025f6a 100644 --- a/node/src/traits.rs +++ b/node/src/traits.rs @@ -13,19 +13,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use snarkos_node_router::{PeerPoolHandling, Routing, messages::NodeType}; +use crate::router::{PeerPoolHandling, Routing, messages::NodeType}; + +use snarkos_utilities::SignalHandler; + use snarkvm::prelude::{Address, Network, PrivateKey, ViewKey}; -use once_cell::sync::OnceCell; -use std::{ - future::Future, - io, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Duration, -}; +use std::time::Duration; #[async_trait] pub trait NodeInterface: Routing { @@ -54,67 +48,21 @@ pub trait NodeInterface: Routing { self.router().is_dev() } - /// Handles OS signals for the node to intercept and perform a clean shutdown. - /// The optional `shutdown_flag` flag can be used to cleanly terminate the syncing process. - fn handle_signals(shutdown_flag: Arc) -> Arc> { - // In order for the signal handler to be started as early as possible, a reference to the node needs - // to be passed to it at a later time. - let node: Arc> = Default::default(); - - #[cfg(target_family = "unix")] - fn signal_listener() -> impl Future> { - use tokio::signal::unix::{SignalKind, signal}; - - // Handle SIGINT, SIGTERM, SIGQUIT, and SIGHUP. - let mut s_int = signal(SignalKind::interrupt()).unwrap(); - let mut s_term = signal(SignalKind::terminate()).unwrap(); - let mut s_quit = signal(SignalKind::quit()).unwrap(); - let mut s_hup = signal(SignalKind::hangup()).unwrap(); - - // Return when any of the signals above is received. - async move { - tokio::select!( - _ = s_int.recv() => (), - _ = s_term.recv() => (), - _ = s_quit.recv() => (), - _ = s_hup.recv() => (), - ); - Ok(()) - } - } - #[cfg(not(target_family = "unix"))] - fn signal_listener() -> impl Future> { - tokio::signal::ctrl_c() - } - - let node_clone = node.clone(); - tokio::task::spawn(async move { - match signal_listener().await { - Ok(()) => { - warn!("=========================================================================================="); - warn!("⚠️ Attention - Starting the graceful shutdown procedure (ETA: 30 seconds)..."); - warn!("⚠️ Attention - To avoid DATA CORRUPTION, do NOT interrupt snarkOS (or press Ctrl+C again)"); - warn!("⚠️ Attention - Please wait until the shutdown gracefully completes (ETA: 30 seconds)"); - warn!("=========================================================================================="); - - match node_clone.get() { - // If the node is already initialized, then shut it down. - Some(node) => node.shut_down().await, - // Otherwise, if the node is not yet initialized, then set the shutdown flag directly. - None => shutdown_flag.store(true, Ordering::Relaxed), - } - - // A best-effort attempt to let any ongoing activity conclude. - tokio::time::sleep(Duration::from_secs(3)).await; - - // Terminate the process. - std::process::exit(0); - } - Err(error) => error!("tokio::signal::ctrl_c encountered an error: {}", error), - } - }); - - node + /// Blocks until a shutdown signal was received or manual shutdown was triggered. + async fn wait_for_signals(&self, handler: &SignalHandler) { + handler.wait_for_signals().await; + + warn!("=========================================================================================="); + warn!("⚠️ Attention - Starting the graceful shutdown procedure (ETA: 30 seconds)..."); + warn!("⚠️ Attention - To avoid DATA CORRUPTION, do NOT interrupt snarkOS (or press Ctrl+C again)"); + warn!("⚠️ Attention - Please wait until the shutdown gracefully completes (ETA: 30 seconds)"); + warn!("=========================================================================================="); + + // If the node is already initialized, then shut it down. + self.shut_down().await; + + // A best-effort attempt to let any ongoing activity conclude. + tokio::time::sleep(Duration::from_secs(3)).await; } /// Shuts down the node. diff --git a/node/src/validator/mod.rs b/node/src/validator/mod.rs index 49e8a53196..5a85a48391 100644 --- a/node/src/validator/mod.rs +++ b/node/src/validator/mod.rs @@ -36,6 +36,8 @@ use snarkos_node_tcp::{ P2P, protocols::{Disconnect, Handshake, OnConnect, Reading}, }; +use snarkos_utilities::SignalHandler; + use snarkvm::prelude::{ Ledger, Network, @@ -51,11 +53,7 @@ use core::future::Future; use locktick::parking_lot::Mutex; #[cfg(not(feature = "locktick"))] use parking_lot::Mutex; -use std::{ - net::SocketAddr, - sync::{Arc, atomic::AtomicBool}, - time::Duration, -}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use tokio::task::JoinHandle; /// A validator is a full node, capable of validating blocks. @@ -73,8 +71,6 @@ pub struct Validator> { sync: Arc>, /// The spawned handles. handles: Arc>>>, - /// The shutdown signal. - shutdown: Arc, /// Keeps track of sending pings. ping: Arc>, } @@ -95,16 +91,13 @@ impl> Validator { allow_external_peers: bool, dev_txs: bool, dev: Option, - shutdown: Arc, + signal_handler: Arc, ) -> Result { - // Initialize the signal handler. - let signal_node = Self::handle_signals(shutdown.clone()); - // Initialize the ledger. let ledger = Ledger::load(genesis, storage_mode.clone())?; // Initialize the ledger service. - let ledger_service = Arc::new(CoreLedgerService::new(ledger.clone(), shutdown.clone())); + let ledger_service = Arc::new(CoreLedgerService::new(ledger.clone(), signal_handler.clone())); // Determine if the validator should rotate external peers. let rotate_external_peers = false; @@ -151,11 +144,10 @@ impl> Validator { sync: sync.clone(), ping, handles: Default::default(), - shutdown: shutdown.clone(), }; // Perform sync with CDN (if enabled). - let cdn_sync = cdn.map(|base_url| Arc::new(CdnBlockSync::new(base_url, ledger.clone(), shutdown))); + let cdn_sync = cdn.map(|base_url| Arc::new(CdnBlockSync::new(base_url, ledger.clone(), signal_handler))); // Initialize the transaction pool. node.initialize_transaction_pool(dev, dev_txs)?; @@ -187,8 +179,6 @@ impl> Validator { // Initialize the routing. node.initialize_routing().await; - // Pass the node to the signal handler. - let _ = signal_node.set(node.clone()); // Return the node. Ok(node) @@ -461,7 +451,6 @@ impl> NodeInterface for Validator { // Shut down the node. trace!("Shutting down the node..."); - self.shutdown.store(true, std::sync::atomic::Ordering::Release); // Abort the tasks. trace!("Shutting down the validator..."); @@ -531,7 +520,7 @@ mod tests { false, dev_txs, None, - Default::default(), + SignalHandler::new(), ) .await .unwrap(); diff --git a/node/tests/common/node.rs b/node/tests/common/node.rs index e161a703e7..67f31dc4b6 100644 --- a/node/tests/common/node.rs +++ b/node/tests/common/node.rs @@ -14,8 +14,11 @@ // limitations under the License. use crate::common::test_peer::sample_genesis_block; + use snarkos_account::Account; use snarkos_node::{Client, Prover, Validator}; +use snarkos_utilities::SignalHandler; + use snarkvm::prelude::{MainnetV0 as CurrentNetwork, store::helpers::memory::ConsensusMemory}; use aleo_std::StorageMode; @@ -33,7 +36,7 @@ pub async fn client() -> Client> StorageMode::new_test(None), false, // No extra peer rotation. None, - Default::default(), + SignalHandler::new(), ) .await .expect("couldn't create client instance") @@ -47,7 +50,7 @@ pub async fn prover() -> Prover> sample_genesis_block(), StorageMode::new_test(None), None, - Default::default(), + SignalHandler::new(), ) .await .expect("couldn't create prover instance") @@ -68,7 +71,7 @@ pub async fn validator() -> Validator" ] +description = "Utilities for a decentralized operating system" +homepage = "https://aleo.org" +repository = "https://github.com/ProvableHQ/snarkOS" +keywords = [ + "aleo", + "cryptography", + "blockchain", + "decentralized", + "zero-knowledge" +] +include = ["../LICENSE.md"] +categories = [ "cryptography", "cryptography::cryptocurrencies", "os" ] +license = "Apache-2.0" +edition = "2024" + +[dependencies.tokio] +workspace = true +features = [ "macros", "signal" ] + +[dependencies.tracing] +workspace = true diff --git a/utilities/src/lib.rs b/utilities/src/lib.rs new file mode 100644 index 0000000000..de7548d27a --- /dev/null +++ b/utilities/src/lib.rs @@ -0,0 +1,19 @@ +// Copyright (c) 2019-2025 Provable Inc. +// This file is part of the snarkOS library. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Utilities for signal and shutdown handling. +pub mod signals; + +pub use signals::*; diff --git a/utilities/src/signals.rs b/utilities/src/signals.rs new file mode 100644 index 0000000000..99f1006a8c --- /dev/null +++ b/utilities/src/signals.rs @@ -0,0 +1,140 @@ +// Copyright (c) 2019-2025 Provable Inc. +// This file is part of the snarkOS library. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use tokio::sync::Notify; + +use tracing::{debug, error}; + +/// Generic trait that can be queried for whether current process should be stopped. +/// This is implemented by `SignalHandler` and `SimpleStoppable`. +pub trait Stoppable: Send + Sync { + /// Initiates shutdown of the node. + fn stop(&self); + + /// Returns `true` if the node is (in the process of being) stopped. + fn is_stopped(&self) -> bool; +} + +/// Wrapper around `AtomicBool` that implements the `Stoppable` trait. +/// +/// This is useful when no signal or complex shutdown handling is necessary (e.g., in a test environment). +pub struct SimpleStoppable { + state: AtomicBool, +} + +impl SimpleStoppable { + pub fn new() -> Arc { + Arc::new(Self { state: AtomicBool::new(false) }) + } +} + +impl Stoppable for SimpleStoppable { + fn stop(&self) { + self.state.store(true, Ordering::SeqCst); + } + + fn is_stopped(&self) -> bool { + self.state.load(Ordering::SeqCst) + } +} + +/// Helper for signal handling that implements the `Stoppable` trait. +/// +/// This struct will set itself to "stopped" as soon as the process receives Ctrl+C. +/// It can also be manually stopped (e.g., when the node encounters a fatal error). +pub struct SignalHandler { + stopped: AtomicBool, + notify: Notify, +} + +impl SignalHandler { + /// Spawns a background tasks that listens for Ctrl+C and returns `Self`. + pub fn new() -> Arc { + let obj = Arc::new(Self { stopped: AtomicBool::new(false), notify: Default::default() }); + + { + let obj = obj.clone(); + tokio::spawn(async move { + obj.handle_signals().await; + }); + } + + obj + } + + /// Logic for the background task that waits for a signal. + async fn handle_signals(&self) { + #[cfg(target_family = "unix")] + let signal_listener = async move { + use tokio::signal::unix::{SignalKind, signal}; + + // Handle SIGINT, SIGTERM, SIGQUIT, and SIGHUP. + let mut s_int = signal(SignalKind::interrupt())?; + let mut s_term = signal(SignalKind::terminate())?; + let mut s_quit = signal(SignalKind::quit())?; + let mut s_hup = signal(SignalKind::hangup())?; + + tokio::select!( + _ = s_int.recv() => debug!("Received SIGINT"), + _ = s_term.recv() => debug!("Received SIGTERM"), + _ = s_quit.recv() => debug!("Received SIGQUIT"), + _ = s_hup.recv() => debug!("Received SIGHUP"), + ); + + std::io::Result::<()>::Ok(()) + }; + + #[cfg(not(target_family = "unix"))] + let signal_listener = async move { + tokio::signal::ctrl_c().await?; + debug!("Received signal"); + + std::io::Result::<()>::Ok(()) + }; + + // Block until the signal. + match signal_listener.await { + Ok(()) => {} + Err(error) => { + error!("tokio::signal encountered an error: {error}"); + } + } + + self.stop(); + } + + /// Blocks until the signal handler was invoked or the stopped flag was set some other way. + /// Note: This can only be called once, and must not be called concurrently. + pub async fn wait_for_signals(&self) { + while !self.is_stopped() { + self.notify.notified().await + } + } +} + +impl Stoppable for SignalHandler { + fn stop(&self) { + self.stopped.store(true, Ordering::SeqCst); + self.notify.notify_one(); + } + + fn is_stopped(&self) -> bool { + self.stopped.load(Ordering::SeqCst) + } +} From 6b4fda4cb1791d8c826830a0440c719718fed6a4 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 6 Oct 2025 17:33:06 -0700 Subject: [PATCH 05/10] misc(node): remove obsolete sleep during shutdown --- node/src/traits.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/node/src/traits.rs b/node/src/traits.rs index d227025f6a..8851e46f03 100644 --- a/node/src/traits.rs +++ b/node/src/traits.rs @@ -19,8 +19,6 @@ use snarkos_utilities::SignalHandler; use snarkvm::prelude::{Address, Network, PrivateKey, ViewKey}; -use std::time::Duration; - #[async_trait] pub trait NodeInterface: Routing { /// Returns the node type. @@ -60,9 +58,6 @@ pub trait NodeInterface: Routing { // If the node is already initialized, then shut it down. self.shut_down().await; - - // A best-effort attempt to let any ongoing activity conclude. - tokio::time::sleep(Duration::from_secs(3)).await; } /// Shuts down the node. From 95fb94c003256416786fe0a26a682d71ea705eb0 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Fri, 12 Sep 2025 17:29:22 -0700 Subject: [PATCH 06/10] misc(bft): use snarkvm::utilities::task everywhere --- Cargo.lock | 122 ++++++++++++++-------------- Cargo.toml | 3 +- node/bft/Cargo.toml | 2 +- node/bft/src/primary.rs | 82 ++++++++++--------- node/bft/src/sync/mod.rs | 13 +-- node/bft/src/worker.rs | 39 ++++++--- node/bft/tests/components/worker.rs | 17 ++-- 7 files changed, 150 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 052421c554..d6211583f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4198,7 +4198,7 @@ dependencies = [ [[package]] name = "snarkvm" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "dotenvy", @@ -4221,7 +4221,7 @@ dependencies = [ [[package]] name = "snarkvm-algorithms" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4249,7 +4249,7 @@ dependencies = [ [[package]] name = "snarkvm-algorithms-cuda" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "blst", "cc", @@ -4260,7 +4260,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-account", "snarkvm-circuit-algorithms", @@ -4274,7 +4274,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-account" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-network", "snarkvm-circuit-types", @@ -4284,7 +4284,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-algorithms" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-types", "snarkvm-console-algorithms", @@ -4294,7 +4294,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-collections" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-types", @@ -4304,7 +4304,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-environment" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "indexmap 2.11.4", "itertools 0.14.0", @@ -4322,12 +4322,12 @@ dependencies = [ [[package]] name = "snarkvm-circuit-environment-witness" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" [[package]] name = "snarkvm-circuit-network" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-collections", @@ -4338,7 +4338,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-program" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-account", "snarkvm-circuit-algorithms", @@ -4352,7 +4352,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-address", @@ -4367,7 +4367,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-address" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4380,7 +4380,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-boolean" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-console-types-boolean", @@ -4389,7 +4389,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-field" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4399,7 +4399,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-group" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4411,7 +4411,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-integers" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4423,7 +4423,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-scalar" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4434,7 +4434,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-string" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4446,7 +4446,7 @@ dependencies = [ [[package]] name = "snarkvm-console" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-account", "snarkvm-console-algorithms", @@ -4459,7 +4459,7 @@ dependencies = [ [[package]] name = "snarkvm-console-account" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "bs58", "snarkvm-console-network", @@ -4470,7 +4470,7 @@ dependencies = [ [[package]] name = "snarkvm-console-algorithms" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "blake2s_simd", "smallvec", @@ -4483,7 +4483,7 @@ dependencies = [ [[package]] name = "snarkvm-console-collections" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "rayon", @@ -4494,7 +4494,7 @@ dependencies = [ [[package]] name = "snarkvm-console-network" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "enum-iterator", @@ -4514,7 +4514,7 @@ dependencies = [ [[package]] name = "snarkvm-console-network-environment" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "bech32", @@ -4532,7 +4532,7 @@ dependencies = [ [[package]] name = "snarkvm-console-program" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "enum-iterator", "enum_index", @@ -4552,7 +4552,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-address", @@ -4567,7 +4567,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-address" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4578,7 +4578,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-boolean" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", ] @@ -4586,7 +4586,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-field" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4596,7 +4596,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-group" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4607,7 +4607,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-integers" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4618,7 +4618,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-scalar" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4629,7 +4629,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-string" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4640,7 +4640,7 @@ dependencies = [ [[package]] name = "snarkvm-curves" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "rand 0.8.5", "rayon", @@ -4654,7 +4654,7 @@ dependencies = [ [[package]] name = "snarkvm-fields" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4671,7 +4671,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4699,7 +4699,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-authority" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "rand 0.8.5", @@ -4711,7 +4711,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-block" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "indexmap 2.11.4", @@ -4733,7 +4733,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-committee" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "indexmap 2.11.4", @@ -4752,7 +4752,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-ledger-narwhal-batch-certificate", "snarkvm-ledger-narwhal-batch-header", @@ -4765,7 +4765,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-batch-certificate" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "indexmap 2.11.4", "rayon", @@ -4778,7 +4778,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-batch-header" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "indexmap 2.11.4", "rayon", @@ -4791,7 +4791,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-data" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "bytes", "serde_json", @@ -4802,7 +4802,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-subdag" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "indexmap 2.11.4", "rayon", @@ -4817,7 +4817,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-transmission" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "bytes", "serde_json", @@ -4830,7 +4830,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-transmission-id" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "snarkvm-console", "snarkvm-ledger-puzzle", @@ -4839,7 +4839,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-puzzle" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4859,7 +4859,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-puzzle-epoch" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4882,7 +4882,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-query" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "anyhow", "async-trait", @@ -4899,7 +4899,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-store" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std-storage", "anyhow", @@ -4927,7 +4927,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-test-helpers" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4945,7 +4945,7 @@ dependencies = [ [[package]] name = "snarkvm-metrics" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "metrics", ] @@ -4953,7 +4953,7 @@ dependencies = [ [[package]] name = "snarkvm-parameters" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -4976,7 +4976,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", @@ -5009,7 +5009,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-process" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "colored 3.0.0", @@ -5034,7 +5034,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-program" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "indexmap 2.11.4", "paste", @@ -5052,7 +5052,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-snark" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "bincode", "serde_json", @@ -5065,11 +5065,12 @@ dependencies = [ [[package]] name = "snarkvm-utilities" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "aleo-std", "anyhow", "bincode", + "colored 3.0.0", "num-bigint", "num_cpus", "rand 0.8.5", @@ -5080,6 +5081,7 @@ dependencies = [ "smol_str", "snarkvm-utilities-derives", "thiserror 2.0.17", + "tokio", "tracing", "zeroize", ] @@ -5087,7 +5089,7 @@ dependencies = [ [[package]] name = "snarkvm-utilities-derives" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?rev=58b326d#58b326dfa5808aeb08e354cd58da8f0c50dd38c7" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" dependencies = [ "proc-macro2", "quote 1.0.41", diff --git a/Cargo.toml b/Cargo.toml index 7fbe38a618..6177b34321 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,8 @@ default-features = false [workspace.dependencies.snarkvm] #path = "../snarkVM" git = "https://github.com/ProvableHQ/snarkVM.git" -rev = "58b326d" +branch = "feat/track-error" +#rev = "619464c8edf75636809d0bfb8e8404de78a62c97" #version = "=4.2.1" default-features = false diff --git a/node/bft/Cargo.toml b/node/bft/Cargo.toml index ce119f4ebf..850f3ae228 100644 --- a/node/bft/Cargo.toml +++ b/node/bft/Cargo.toml @@ -125,7 +125,7 @@ workspace = true [dependencies.snarkvm] workspace = true -features = [ "utilities" ] +features = [ "utilities", "async" ] [dependencies.time] workspace = true diff --git a/node/bft/src/primary.rs b/node/bft/src/primary.rs index 542b6b18e5..08fb5d9f52 100644 --- a/node/bft/src/primary.rs +++ b/node/bft/src/primary.rs @@ -57,6 +57,7 @@ use snarkvm::{ puzzle::{Solution, SolutionID}, }, prelude::{ConsensusVersion, committee::Committee}, + utilities::task::{self, JoinHandle}, }; use aleo_std::StorageMode; @@ -81,7 +82,7 @@ use std::{ }; #[cfg(not(feature = "locktick"))] use tokio::sync::Mutex as TMutex; -use tokio::{sync::OnceCell, task::JoinHandle}; +use tokio::sync::OnceCell; /// A helper type for an optional proposed batch. pub type ProposedBatch = RwLock>>; @@ -407,7 +408,7 @@ impl Primary { // Resend the batch proposal to the validator for signing. Some(peer_ip) => { let (gateway, event_, round) = (self.gateway.clone(), event.clone(), proposal.round()); - tokio::spawn(async move { + task::spawn(async move { debug!("Resending batch proposal for round {round} to peer '{peer_ip}'"); // Resend the batch proposal to the peer. if gateway.send(peer_ip, event_).await.is_none() { @@ -573,14 +574,13 @@ impl Primary { } // Deserialize the transaction. If the transaction exceeds the maximum size, then return an error. - let transaction = spawn_blocking!({ - match transaction { - Data::Object(transaction) => Ok(transaction), - Data::Buffer(bytes) => { - Ok(Transaction::::read_le(&mut bytes.take(N::MAX_TRANSACTION_SIZE as u64))?) - } + let transaction = task::spawn_blocking(|| match transaction { + Data::Object(transaction) => Ok(transaction), + Data::Buffer(bytes) => { + Transaction::::read_le(&mut bytes.take(N::MAX_TRANSACTION_SIZE as u64)) } - })?; + }) + .await?; // TODO (raychu86): Record Commitment - Remove this logic after the next migration height is reached. // ConsensusVersion V8 Migration logic - @@ -675,15 +675,18 @@ impl Primary { // Prepare the previous batch certificate IDs. let previous_certificate_ids = previous_certificates.into_iter().map(|c| c.id()).collect(); // Sign the batch header and construct the proposal. - let (batch_header, proposal) = spawn_blocking!(BatchHeader::new( - &private_key, - round, - current_timestamp, - committee_id, - transmission_ids, - previous_certificate_ids, - &mut rand::thread_rng() - )) + let (batch_header, proposal) = task::spawn_blocking(move || { + BatchHeader::new( + &private_key, + round, + current_timestamp, + committee_id, + transmission_ids, + previous_certificate_ids, + &mut rand::thread_rng(), + ) + }) + .await .and_then(|batch_header| { Proposal::new(committee_lookback, batch_header.clone(), transmissions.clone()) .map(|proposal| (batch_header, proposal)) @@ -716,7 +719,7 @@ impl Primary { let BatchPropose { round: batch_round, batch_header } = batch_propose; // Deserialize the batch header. - let batch_header = spawn_blocking!(batch_header.deserialize_blocking())?; + let batch_header = task::spawn_blocking(|| batch_header.deserialize_blocking()).await?; // Ensure the round matches in the batch header. if batch_round != batch_header.round() { // Proceed to disconnect the validator. @@ -782,7 +785,7 @@ impl Primary { // Instead, rebroadcast the cached signature to the peer. if signed_round == batch_header.round() && signed_batch_id == batch_header.batch_id() { let gateway = self.gateway.clone(); - tokio::spawn(async move { + task::spawn(async move { debug!("Resending a signature for a batch in round {batch_round} from '{peer_ip}'"); let event = Event::BatchSignature(BatchSignature::new(batch_header.batch_id(), signature)); // Resend the batch signature to the peer. @@ -846,8 +849,10 @@ impl Primary { // Ensure the batch header from the peer is valid. let (storage, header) = (self.storage.clone(), batch_header.clone()); - let missing_transmissions = - spawn_blocking!(storage.check_batch_header(&header, missing_transmissions, Default::default()))?; + let missing_transmissions = task::spawn_blocking(move || { + storage.check_batch_header(&header, missing_transmissions, Default::default()) + }) + .await?; // Inserts the missing transmissions into the workers. self.insert_missing_transmissions_into_workers(peer_ip, missing_transmissions.into_iter())?; @@ -872,14 +877,13 @@ impl Primary { (transmission_id, transmission) { // Deserialize the transaction. If the transaction exceeds the maximum size, then return an error. - let transaction = spawn_blocking!({ - match transaction { - Data::Object(transaction) => Ok(transaction), - Data::Buffer(bytes) => { - Ok(Transaction::::read_le(&mut bytes.take(N::MAX_TRANSACTION_SIZE as u64))?) - } + let transaction = task::spawn_blocking(|| match transaction { + Data::Object(transaction) => Ok(transaction), + Data::Buffer(bytes) => { + Transaction::::read_le(&mut bytes.take(N::MAX_TRANSACTION_SIZE as u64)) } - })?; + }) + .await?; // TODO (raychu86): Record Commitment - Remove this logic after the next migration height is reached. // ConsensusVersion V8 Migration logic - @@ -937,7 +941,7 @@ impl Primary { let batch_id = batch_header.batch_id(); // Sign the batch ID. let account = self.gateway.account().clone(); - let signature = spawn_blocking!(account.sign(&[batch_id], &mut rand::thread_rng()))?; + let signature = task::spawn_blocking(move || account.sign(&[batch_id], &mut rand::thread_rng())).await?; // Ensure the proposal has not already been signed. // @@ -965,7 +969,7 @@ impl Primary { // Broadcast the signature back to the validator. let self_ = self.clone(); - tokio::spawn(async move { + task::spawn(async move { let event = Event::BatchSignature(BatchSignature::new(batch_id, signature)); // Send the batch signature to the peer. if self_.gateway.send(peer_ip, event).await.is_some() { @@ -1010,7 +1014,7 @@ impl Primary { } let self_ = self.clone(); - let Some(proposal) = spawn_blocking!({ + let Some(proposal) = task::spawn_blocking(move || { // Acquire the write lock. let mut proposed_batch = self_.proposed_batch.write(); // Add the signature to the batch, and determine if the batch is ready to be certified. @@ -1058,7 +1062,7 @@ impl Primary { Some(proposal) => Ok(Some(proposal)), None => Ok(None), } - })? + }).await? else { return Ok(()); }; @@ -1204,7 +1208,7 @@ impl Primary { // Retrieve the block locators. let self__ = self_.clone(); - let block_locators = match spawn_blocking!(self__.sync.get_block_locators()) { + let block_locators = match task::spawn_blocking(move || self__.sync.get_block_locators()).await { Ok(block_locators) => block_locators, Err(e) => { warn!("Failed to retrieve block locators - {e}"); @@ -1626,7 +1630,8 @@ impl Primary { let transmissions = transmissions.into_iter().collect::>(); // Store the certified batch. let (storage, certificate_) = (self.storage.clone(), certificate.clone()); - spawn_blocking!(storage.insert_certificate(certificate_, transmissions, Default::default()))?; + task::spawn_blocking(move || storage.insert_certificate(certificate_, transmissions, Default::default())) + .await?; debug!("Stored a batch certificate for round {}", certificate.round()); // If a BFT sender was provided, send the certificate to the BFT. if let Some(bft_sender) = self.bft_sender.get() { @@ -1713,7 +1718,10 @@ impl Primary { if !self.storage.contains_certificate(certificate.id()) { // Store the batch certificate. let (storage, certificate_) = (self.storage.clone(), certificate.clone()); - spawn_blocking!(storage.insert_certificate(certificate_, missing_transmissions, Default::default()))?; + task::spawn_blocking(move || { + storage.insert_certificate(certificate_, missing_transmissions, Default::default()) + }) + .await?; debug!("Stored a batch certificate for round {batch_round} from '{peer_ip}'"); // If a BFT sender was provided, send the round and certificate to the BFT. if let Some(bft_sender) = self.bft_sender.get() { @@ -1926,7 +1934,7 @@ impl Primary { impl Primary { /// Spawns a task with the given future; it should only be used for long-running tasks. fn spawn + Send + 'static>(&self, future: T) { - self.handles.lock().push(tokio::spawn(future)); + self.handles.lock().push(task::spawn(future)); } /// Shuts down the primary. diff --git a/node/bft/src/sync/mod.rs b/node/bft/src/sync/mod.rs index b7f2dd1ae9..fecf0cdc7e 100644 --- a/node/bft/src/sync/mod.rs +++ b/node/bft/src/sync/mod.rs @@ -28,7 +28,11 @@ use snarkos_node_sync::{BLOCK_REQUEST_BATCH_DELAY, BlockSync, Ping, PrepareSyncR use snarkvm::{ console::{network::Network, types::Field}, ledger::{authority::Authority, block::Block, narwhal::BatchCertificate}, - prelude::{cfg_into_iter, cfg_iter}, + utilities::{ + cfg_into_iter, + cfg_iter, + task::{self, JoinHandle}, + }, }; use anyhow::{Result, anyhow, bail}; @@ -48,10 +52,7 @@ use std::{ }; #[cfg(not(feature = "locktick"))] use tokio::sync::Mutex as TMutex; -use tokio::{ - sync::{OnceCell, oneshot}, - task::JoinHandle, -}; +use tokio::sync::{OnceCell, oneshot}; /// Block synchronization logic for validators. /// @@ -972,7 +973,7 @@ impl Sync { impl Sync { /// Spawns a task with the given future; it should only be used for long-running tasks. fn spawn + Send + 'static>(&self, future: T) { - self.handles.lock().push(tokio::spawn(future)); + self.handles.lock().push(task::spawn(future)); } /// Shuts down the primary. diff --git a/node/bft/src/worker.rs b/node/bft/src/worker.rs index a4c2fda37e..eda99b96f0 100644 --- a/node/bft/src/worker.rs +++ b/node/bft/src/worker.rs @@ -24,14 +24,19 @@ use crate::{ }; use snarkos_node_bft_ledger_service::LedgerService; use snarkvm::{ - console::prelude::*, + console::{network::Network, prelude::Read}, ledger::{ block::Transaction, narwhal::{BatchHeader, Data, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, }, + utilities::{ + FromBytes, + task::{self, JoinHandle}, + }, }; +use anyhow::{Context, Result, bail, ensure}; use colored::Colorize; use indexmap::{IndexMap, IndexSet}; #[cfg(feature = "locktick")] @@ -40,7 +45,7 @@ use locktick::parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock}; use rand::seq::IteratorRandom; use std::{future::Future, net::SocketAddr, sync::Arc, time::Duration}; -use tokio::{sync::oneshot, task::JoinHandle, time::timeout}; +use tokio::{sync::oneshot, time::timeout}; /// A worker's main role is maintaining a queue of verified ("ready") transmissions, /// which will eventually be fetched by the primary when the primary generates a new batch. @@ -498,12 +503,11 @@ impl Worker { ); } // Wait for the transmission to be fetched. - match timeout(Duration::from_millis(MAX_FETCH_TIMEOUT_IN_MS), callback_receiver).await { - // If the transmission was fetched, return it. - Ok(result) => Ok((transmission_id, result?)), - // If the transmission was not fetched, return an error. - Err(e) => bail!("Unable to fetch transmission - (timeout) {e}"), - } + let transmission = timeout(Duration::from_millis(MAX_FETCH_TIMEOUT_IN_MS), callback_receiver) + .await + .with_context(|| "Unable to fetch transmission - (timeout)")??; + + Ok((transmission_id, transmission)) } /// Handles the incoming transmission response. @@ -540,7 +544,7 @@ impl Worker { /// Spawns a task with the given future; it should only be used for long-running tasks. fn spawn + Send + 'static>(&self, future: T) { - self.handles.lock().push(tokio::spawn(future)); + self.handles.lock().push(task::spawn(future)); } /// Shuts down the worker. @@ -558,19 +562,25 @@ mod tests { use snarkos_node_bft_ledger_service::LedgerService; use snarkos_node_bft_storage_service::BFTMemoryService; use snarkvm::{ - console::{network::Network, types::Field}, + console::{ + network::{ConsensusVersion, Network}, + types::{Address, Field}, + }, ledger::{ block::Block, committee::Committee, narwhal::{BatchCertificate, Subdag, Transmission, TransmissionID}, test_helpers::sample_execution_transaction_with_fee, }, - prelude::Address, + prelude::{Itertools, Uniform}, + utilities::TestRng, }; + use anyhow::anyhow; use bytes::Bytes; use indexmap::IndexMap; use mockall::mock; + use rand::Rng; use std::{io, ops::Range}; type CurrentNetwork = snarkvm::prelude::MainnetV0; @@ -926,7 +936,7 @@ mod tests { for i in 1..=num_flood_requests { let worker_ = worker.clone(); let peer_ip = peer_ips.pop().unwrap(); - tokio::spawn(async move { + task::spawn(async move { let _ = worker_.send_transmission_request(peer_ip, transmission_id).await; }); tokio::time::sleep(Duration::from_millis(10)).await; @@ -946,7 +956,7 @@ mod tests { // Flood the pending queue with transmission requests again, this time to a single peer for i in 1..=num_flood_requests { let worker_ = worker.clone(); - tokio::spawn(async move { + task::spawn(async move { let _ = worker_.send_transmission_request(first_peer_ip, transmission_id).await; }); tokio::time::sleep(Duration::from_millis(10)).await; @@ -999,12 +1009,15 @@ mod tests { mod prop_tests { use super::*; use crate::Gateway; + use snarkos_node_bft_ledger_service::MockLedgerService; use snarkvm::{ console::account::Address, ledger::committee::{Committee, MIN_VALIDATOR_STAKE}, + prelude::TestRng, }; + use rand::Rng; use test_strategy::proptest; type CurrentNetwork = snarkvm::prelude::MainnetV0; diff --git a/node/bft/tests/components/worker.rs b/node/bft/tests/components/worker.rs index be53686ae0..4408afbf99 100644 --- a/node/bft/tests/components/worker.rs +++ b/node/bft/tests/components/worker.rs @@ -19,10 +19,7 @@ use crate::common::{ utils::{sample_ledger, sample_worker}, }; use snarkos_node_bft::helpers::max_redundant_requests; -use snarkvm::{ - ledger::narwhal::TransmissionID, - prelude::{Network, TestRng}, -}; +use snarkvm::{console::network::Network, ledger::narwhal::TransmissionID, prelude::TestRng, utilities::task}; use std::net::SocketAddr; @@ -57,7 +54,7 @@ async fn test_resend_transmission_request() { // Send a request to fetch the dummy transmission. let worker_ = worker.clone(); - tokio::spawn(async move { worker_.get_or_fetch_transmission(initial_peer_ip, transmission_id).await }); + task::spawn(async move { worker_.get_or_fetch_transmission(initial_peer_ip, transmission_id).await }); tokio::time::sleep(std::time::Duration::from_millis(10)).await; @@ -76,7 +73,7 @@ async fn test_resend_transmission_request() { for i in 1..num_test_requests { let worker_ = worker.clone(); let peer_ip = initial_peer_ip; - tokio::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); + task::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); tokio::time::sleep(std::time::Duration::from_millis(10)).await; @@ -95,7 +92,7 @@ async fn test_resend_transmission_request() { for i in 1..num_test_requests { let peer_ip = peer_ips.pop().unwrap(); let worker_ = worker.clone(); - tokio::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); + task::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); tokio::time::sleep(std::time::Duration::from_millis(10)).await; @@ -141,7 +138,7 @@ async fn test_flood_transmission_requests() { // Send the maximum number of redundant requests to fetch the dummy transmission. for peer_ip in remaining_peer_ips.clone() { let worker_ = worker.clone(); - tokio::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); + task::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); } tokio::time::sleep(std::time::Duration::from_millis(10)).await; @@ -160,7 +157,7 @@ async fn test_flood_transmission_requests() { for i in 1..=6 { let worker_ = worker.clone(); let peer_ip = initial_peer_ip; - tokio::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); + task::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); tokio::time::sleep(std::time::Duration::from_millis(10)).await; @@ -179,7 +176,7 @@ async fn test_flood_transmission_requests() { for i in 1..=6 { let worker_ = worker.clone(); let peer_ip = remaining_peer_ips.pop().unwrap(); - tokio::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); + task::spawn(async move { worker_.get_or_fetch_transmission(peer_ip, transmission_id).await }); tokio::time::sleep(std::time::Duration::from_millis(10)).await; From 00d05706dbb5bf0c9efa9f65d52d25509754fa60 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 15 Sep 2025 16:32:44 -0700 Subject: [PATCH 07/10] feat: use thread-safe panic handling refactor(utils): move signal handling to dedicated snarkos-utilities crate --- cli/src/commands/start.rs | 1 + node/src/validator/mod.rs | 18 ++++++------ snarkos/main.rs | 60 +++++++++++++++++++-------------------- utilities/src/lib.rs | 1 - 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index e4d8de8575..ef1d193c80 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -748,6 +748,7 @@ impl Start { // Set up the tokio Runtime. // TODO(kaimast): set up a panic handler here for each worker thread once [`tokio::runtime::Builder::unhandled_panic`](https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.unhandled_panic) is stabilized. + // As of now, detached tasks may panic and the error may not be handled by the top-level `catch_unwind`. runtime::Builder::new_multi_thread() .enable_all() .thread_stack_size(8 * 1024 * 1024) diff --git a/node/src/validator/mod.rs b/node/src/validator/mod.rs index 5a85a48391..054e774eca 100644 --- a/node/src/validator/mod.rs +++ b/node/src/validator/mod.rs @@ -38,12 +38,15 @@ use snarkos_node_tcp::{ }; use snarkos_utilities::SignalHandler; -use snarkvm::prelude::{ - Ledger, - Network, - block::{Block, Header}, - puzzle::Solution, - store::ConsensusStorage, +use snarkvm::{ + prelude::{ + Ledger, + Network, + block::{Block, Header}, + puzzle::Solution, + store::ConsensusStorage, + }, + utilities::task::{self, JoinHandle}, }; use aleo_std::StorageMode; @@ -54,7 +57,6 @@ use locktick::parking_lot::Mutex; #[cfg(not(feature = "locktick"))] use parking_lot::Mutex; use std::{net::SocketAddr, sync::Arc, time::Duration}; -use tokio::task::JoinHandle; /// A validator is a full node, capable of validating blocks. #[derive(Clone)] @@ -439,7 +441,7 @@ impl> Validator { /// Spawns a task with the given future; it should only be used for long-running tasks. pub fn spawn + Send + 'static>(&self, future: T) { - self.handles.lock().push(tokio::spawn(future)); + self.handles.lock().push(task::spawn(future)); } } diff --git a/snarkos/main.rs b/snarkos/main.rs index f0558f35bc..77d8c63306 100644 --- a/snarkos/main.rs +++ b/snarkos/main.rs @@ -14,14 +14,17 @@ // limitations under the License. use snarkos_cli::{commands::CLI, helpers::Updater}; -use snarkvm::utilities::display_error; +use snarkvm::utilities::{ + display_error, + errors::{catch_unwind, set_panic_hook}, +}; use clap::Parser; #[cfg(feature = "locktick")] use locktick::lock_snapshots; +use std::env; #[cfg(feature = "locktick")] use std::time::Instant; -use std::{backtrace::Backtrace, env, panic::catch_unwind}; use tracing::log::logger; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] @@ -92,36 +95,10 @@ fn main() { } }); - // Set a custom hook here to show "pretty" errors when panicking. - std::panic::set_hook(Box::new(|err| { - print_error!("⚠️ {}\n", err.to_string().replace("panicked at", "snarkOS encountered an unexpected error at")); - - // Always show backtraces. - let backtrace = Backtrace::force_capture().to_string(); - - let mut msg = "Backtrace:\n".to_string(); - msg.push_str(" [...]\n"); - - // Remove all the low level frames. - // This can be done more cleanly once the `backtrace_frames` feature is stabilized. - let lines = backtrace.lines().skip_while(|line| !line.contains("core::panicking")); - - for line in lines { - // Stop printing once we hit the panic handler. - if line.contains("snarkos::main") { - break; - } - - msg.push_str(&format!("{line}\n")); - } - - // Print the entire backtrace as a single log message. - print_error!("{msg}"); - })); - // Run the CLI. // We use `catch_unwind` here to ensure a panic stops execution and not just a single thread. // Note: `catch_unwind` can be nested without problems. + set_panic_hook(); let result = catch_unwind(|| { // Parse the given arguments. let cli = CLI::parse(); @@ -151,7 +128,30 @@ fn main() { exit(1); } - Err(_) => { + Err((msg, backtrace)) => { + print_error!("⚠️ {}\n", msg.replace("panicked at", "snarkOS encountered an unexpected error at")); + + // Always show backtraces. + let mut msg = "Backtrace:\n".to_string(); + msg.push_str(" [...]\n"); + + // Remove all the low level frames. + // This can be done more cleanly once the `backtrace_frames` feature is stabilized. + let backtrace = backtrace.to_string(); + let lines = backtrace.lines().skip_while(|line| !line.contains("core::panicking")); + + for line in lines { + // Stop printing once we hit the panic handler. + if line.contains("snarkos::main") { + break; + } + + msg.push_str(&format!("{line}\n")); + } + + // Print the entire backtrace as a single log message. + print_error!("{msg}"); + // Print some information for the end-user. print_error!( "This is most likely a bug!\n\ Please report it to the snarkOS developers: https://github.com/ProvableHQ/snarkOS/issues/new?template=bug.md" diff --git a/utilities/src/lib.rs b/utilities/src/lib.rs index de7548d27a..b6aae9f9be 100644 --- a/utilities/src/lib.rs +++ b/utilities/src/lib.rs @@ -15,5 +15,4 @@ /// Utilities for signal and shutdown handling. pub mod signals; - pub use signals::*; From 380f7ddc9e1ad218e113121627c064b226eb874c Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Wed, 1 Oct 2025 14:59:23 -0700 Subject: [PATCH 08/10] logs(consensus): print full error on failed advance --- node/consensus/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/node/consensus/src/lib.rs b/node/consensus/src/lib.rs index 80415491bf..44be5c7176 100644 --- a/node/consensus/src/lib.rs +++ b/node/consensus/src/lib.rs @@ -501,8 +501,10 @@ impl Consensus { let result = spawn_blocking! { self_.try_advance_to_next_block(subdag, transmissions_) }; // If the block failed to advance, reinsert the transmissions into the memory pool. - if let Err(e) = &result { - error!("Unable to advance to the next block - {e}"); + if let Err(err) = &result { + // We cannot log it the usual way because the result will be used later. + // TODO don't log this here once channel indirection is gone. + error!("Unable to advance to the next block - {err:?}"); // On failure, reinsert the transmissions into the memory pool. self.reinsert_transmissions(transmissions).await; } From 7a792f8547cada66adbc9acf93b2c55d200481a9 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Wed, 17 Sep 2025 21:20:42 +0000 Subject: [PATCH 09/10] Revert "Merge pull request #3847 from ProvableHQ/fix/revert-pending-blocks" This reverts commit de856fd42d6caa022810742329d7fd7c51e8913c, reversing changes made to 647f6f325420d994104a9bd3c1062984653e7d1f. --- Cargo.lock | 172 +++--- Cargo.toml | 2 +- cli/src/helpers/logger.rs | 2 + node/bft/ledger-service/src/ledger.rs | 12 +- node/bft/ledger-service/src/mock.rs | 12 +- node/bft/ledger-service/src/prover.rs | 12 +- node/bft/ledger-service/src/traits.rs | 10 +- node/bft/ledger-service/src/translucent.rs | 12 +- node/bft/src/sync/mod.rs | 603 ++++++++++----------- node/bft/src/worker.rs | 7 +- 10 files changed, 438 insertions(+), 406 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6211583f1..3c5177f1b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2271,6 +2271,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "locktick" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a4a7c4b9459e549968abf200cb77c6faf0bdedc87df5065f73ca95031795050" +dependencies = [ + "backtrace", + "parking_lot", + "simple_moving_average", +] + [[package]] name = "log" version = "0.4.28" @@ -3760,7 +3771,7 @@ version = "4.2.2" dependencies = [ "built", "clap", - "locktick", + "locktick 0.3.0", "rusty-hook", "snarkos-account", "snarkos-cli", @@ -3800,7 +3811,7 @@ dependencies = [ "colored 3.0.0", "crossterm 0.29.0", "indexmap 2.11.4", - "locktick", + "locktick 0.3.0", "nix", "num_cpus", "parking_lot", @@ -3856,7 +3867,7 @@ dependencies = [ "futures-util", "http 1.3.1", "indexmap 2.11.4", - "locktick", + "locktick 0.3.0", "lru 0.16.1", "num_cpus", "parking_lot", @@ -3901,7 +3912,7 @@ dependencies = [ "futures", "indexmap 2.11.4", "itertools 0.14.0", - "locktick", + "locktick 0.3.0", "lru 0.16.1", "mockall", "open", @@ -3960,7 +3971,7 @@ dependencies = [ "anyhow", "async-trait", "indexmap 2.11.4", - "locktick", + "locktick 0.3.0", "parking_lot", "rand 0.8.5", "rayon", @@ -3978,7 +3989,7 @@ dependencies = [ "aleo-std", "anyhow", "indexmap 2.11.4", - "locktick", + "locktick 0.3.0", "lru 0.16.1", "parking_lot", "snarkvm", @@ -3993,7 +4004,7 @@ dependencies = [ "bincode", "colored 3.0.0", "http 1.3.1", - "locktick", + "locktick 0.3.0", "parking_lot", "rayon", "reqwest", @@ -4016,7 +4027,7 @@ dependencies = [ "colored 3.0.0", "indexmap 2.11.4", "itertools 0.14.0", - "locktick", + "locktick 0.3.0", "lru 0.16.1", "once_cell", "parking_lot", @@ -4036,7 +4047,7 @@ dependencies = [ name = "snarkos-node-metrics" version = "4.2.2" dependencies = [ - "locktick", + "locktick 0.3.0", "metrics-exporter-prometheus", "parking_lot", "rayon", @@ -4056,7 +4067,7 @@ dependencies = [ "http 1.3.1", "indexmap 2.11.4", "jsonwebtoken", - "locktick", + "locktick 0.3.0", "once_cell", "parking_lot", "rand 0.8.5", @@ -4089,7 +4100,7 @@ dependencies = [ "futures", "futures-util", "linked-hash-map", - "locktick", + "locktick 0.3.0", "parking_lot", "peak_alloc", "rand 0.8.5", @@ -4135,7 +4146,7 @@ dependencies = [ "futures", "indexmap 2.11.4", "itertools 0.14.0", - "locktick", + "locktick 0.3.0", "parking_lot", "rand 0.8.5", "serde", @@ -4177,7 +4188,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "locktick", + "locktick 0.3.0", "once_cell", "parking_lot", "snarkos-node-metrics", @@ -4198,7 +4209,7 @@ dependencies = [ [[package]] name = "snarkvm" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "dotenvy", @@ -4221,7 +4232,7 @@ dependencies = [ [[package]] name = "snarkvm-algorithms" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", @@ -4249,7 +4260,7 @@ dependencies = [ [[package]] name = "snarkvm-algorithms-cuda" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "blst", "cc", @@ -4260,7 +4271,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-account", "snarkvm-circuit-algorithms", @@ -4274,7 +4285,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-account" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-network", "snarkvm-circuit-types", @@ -4284,7 +4295,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-algorithms" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-types", "snarkvm-console-algorithms", @@ -4294,7 +4305,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-collections" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-types", @@ -4304,7 +4315,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-environment" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "indexmap 2.11.4", "itertools 0.14.0", @@ -4322,12 +4333,12 @@ dependencies = [ [[package]] name = "snarkvm-circuit-environment-witness" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" [[package]] name = "snarkvm-circuit-network" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-algorithms", "snarkvm-circuit-collections", @@ -4338,7 +4349,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-program" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-account", "snarkvm-circuit-algorithms", @@ -4352,7 +4363,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-address", @@ -4367,7 +4378,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-address" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4380,7 +4391,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-boolean" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-console-types-boolean", @@ -4389,7 +4400,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-field" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4399,7 +4410,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-group" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4411,7 +4422,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-integers" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4423,7 +4434,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-scalar" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4434,7 +4445,7 @@ dependencies = [ [[package]] name = "snarkvm-circuit-types-string" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-circuit-environment", "snarkvm-circuit-types-boolean", @@ -4446,7 +4457,7 @@ dependencies = [ [[package]] name = "snarkvm-console" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-account", "snarkvm-console-algorithms", @@ -4459,7 +4470,7 @@ dependencies = [ [[package]] name = "snarkvm-console-account" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "bs58", "snarkvm-console-network", @@ -4470,7 +4481,7 @@ dependencies = [ [[package]] name = "snarkvm-console-algorithms" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "blake2s_simd", "smallvec", @@ -4483,7 +4494,7 @@ dependencies = [ [[package]] name = "snarkvm-console-collections" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "rayon", @@ -4494,7 +4505,7 @@ dependencies = [ [[package]] name = "snarkvm-console-network" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "enum-iterator", @@ -4514,7 +4525,7 @@ dependencies = [ [[package]] name = "snarkvm-console-network-environment" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "bech32", @@ -4532,7 +4543,7 @@ dependencies = [ [[package]] name = "snarkvm-console-program" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "enum-iterator", "enum_index", @@ -4552,7 +4563,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-address", @@ -4567,7 +4578,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-address" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4578,7 +4589,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-boolean" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", ] @@ -4586,7 +4597,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-field" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4596,7 +4607,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-group" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4607,7 +4618,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-integers" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4618,7 +4629,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-scalar" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4629,7 +4640,7 @@ dependencies = [ [[package]] name = "snarkvm-console-types-string" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console-network-environment", "snarkvm-console-types-boolean", @@ -4640,7 +4651,7 @@ dependencies = [ [[package]] name = "snarkvm-curves" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "rand 0.8.5", "rayon", @@ -4654,7 +4665,7 @@ dependencies = [ [[package]] name = "snarkvm-fields" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", @@ -4671,12 +4682,12 @@ dependencies = [ [[package]] name = "snarkvm-ledger" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", "indexmap 2.11.4", - "locktick", + "locktick 0.4.0", "lru 0.16.1", "parking_lot", "rand 0.8.5", @@ -4692,6 +4703,7 @@ dependencies = [ "snarkvm-ledger-store", "snarkvm-ledger-test-helpers", "snarkvm-synthesizer", + "snarkvm-utilities", "time", "tracing", ] @@ -4699,7 +4711,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-authority" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "rand 0.8.5", @@ -4711,7 +4723,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-block" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "indexmap 2.11.4", @@ -4733,7 +4745,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-committee" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "indexmap 2.11.4", @@ -4752,7 +4764,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-ledger-narwhal-batch-certificate", "snarkvm-ledger-narwhal-batch-header", @@ -4765,7 +4777,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-batch-certificate" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "indexmap 2.11.4", "rayon", @@ -4778,7 +4790,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-batch-header" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "indexmap 2.11.4", "rayon", @@ -4791,7 +4803,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-data" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "bytes", "serde_json", @@ -4802,7 +4814,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-subdag" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "indexmap 2.11.4", "rayon", @@ -4817,7 +4829,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-transmission" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "bytes", "serde_json", @@ -4830,7 +4842,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-narwhal-transmission-id" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "snarkvm-console", "snarkvm-ledger-puzzle", @@ -4839,13 +4851,13 @@ dependencies = [ [[package]] name = "snarkvm-ledger-puzzle" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", "bincode", "indexmap 2.11.4", - "locktick", + "locktick 0.4.0", "lru 0.16.1", "parking_lot", "rand 0.8.5", @@ -4859,13 +4871,13 @@ dependencies = [ [[package]] name = "snarkvm-ledger-puzzle-epoch" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", "colored 3.0.0", "indexmap 2.11.4", - "locktick", + "locktick 0.4.0", "lru 0.16.1", "parking_lot", "rand 0.8.5", @@ -4882,7 +4894,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-query" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "anyhow", "async-trait", @@ -4899,13 +4911,13 @@ dependencies = [ [[package]] name = "snarkvm-ledger-store" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std-storage", "anyhow", "bincode", "indexmap 2.11.4", - "locktick", + "locktick 0.4.0", "parking_lot", "rayon", "rocksdb", @@ -4927,7 +4939,7 @@ dependencies = [ [[package]] name = "snarkvm-ledger-test-helpers" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", @@ -4945,7 +4957,7 @@ dependencies = [ [[package]] name = "snarkvm-metrics" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "metrics", ] @@ -4953,7 +4965,7 @@ dependencies = [ [[package]] name = "snarkvm-parameters" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", @@ -4962,7 +4974,7 @@ dependencies = [ "curl", "hex", "lazy_static", - "locktick", + "locktick 0.4.0", "parking_lot", "paste", "rand 0.8.5", @@ -4976,13 +4988,13 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", "indexmap 2.11.4", "itertools 0.14.0", - "locktick", + "locktick 0.4.0", "lru 0.16.1", "parking_lot", "rand 0.8.5", @@ -5009,12 +5021,12 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-process" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "colored 3.0.0", "indexmap 2.11.4", - "locktick", + "locktick 0.4.0", "parking_lot", "rand 0.8.5", "rand_chacha 0.3.1", @@ -5034,7 +5046,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-program" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "indexmap 2.11.4", "paste", @@ -5052,7 +5064,7 @@ dependencies = [ [[package]] name = "snarkvm-synthesizer-snark" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "bincode", "serde_json", @@ -5065,7 +5077,7 @@ dependencies = [ [[package]] name = "snarkvm-utilities" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "aleo-std", "anyhow", @@ -5089,7 +5101,7 @@ dependencies = [ [[package]] name = "snarkvm-utilities-derives" version = "4.2.1" -source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=feat%2Ftrack-error#ba6c3f0936b0afa966e6deb9d1db642bc5751b81" +source = "git+https://github.com/ProvableHQ/snarkVM.git?branch=fix%2Frestore-pending-blocks#3e94722a34d2608079bd419d397ceda529f668f6" dependencies = [ "proc-macro2", "quote 1.0.41", diff --git a/Cargo.toml b/Cargo.toml index 6177b34321..e31b822614 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,8 +47,8 @@ default-features = false [workspace.dependencies.snarkvm] #path = "../snarkVM" git = "https://github.com/ProvableHQ/snarkVM.git" -branch = "feat/track-error" #rev = "619464c8edf75636809d0bfb8e8404de78a62c97" +branch = "fix/restore-pending-blocks" #version = "=4.2.1" default-features = false diff --git a/cli/src/helpers/logger.rs b/cli/src/helpers/logger.rs index 4c4c652fbe..006cadc42b 100644 --- a/cli/src/helpers/logger.rs +++ b/cli/src/helpers/logger.rs @@ -107,7 +107,9 @@ fn parse_log_filter(filter_str: &str) -> Result { } /// Sets the log filter based on the given verbosity level. +/// Initializes the logger with the specified verbosity level, where 0 is the lowest verbosity and 6 the highest. /// +/// The following shows what messages are enabled at each level. /// ```ignore /// 0 => info /// 1 => info, debug diff --git a/node/bft/ledger-service/src/ledger.rs b/node/bft/ledger-service/src/ledger.rs index 2189ac319d..05f701c898 100644 --- a/node/bft/ledger-service/src/ledger.rs +++ b/node/bft/ledger-service/src/ledger.rs @@ -19,8 +19,10 @@ use snarkos_utilities::Stoppable; use snarkvm::{ ledger::{ + Block, Ledger, - block::{Block, Transaction}, + PendingBlock, + Transaction, committee::Committee, narwhal::{BatchCertificate, Data, Subdag, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, @@ -349,6 +351,14 @@ impl> LedgerService for CoreLedgerService< spawn_blocking!(ledger.check_transaction_basic(&transaction, None, &mut rand::thread_rng())) } + fn check_block_subdag(&self, block: Block, prefix: &[PendingBlock]) -> Result> { + self.ledger.check_block_subdag(block, prefix) + } + + fn check_block_content(&self, block: PendingBlock) -> Result> { + self.ledger.check_block_content(block, &mut rand::thread_rng()) + } + /// Checks the given block is valid next block. fn check_next_block(&self, block: &Block) -> Result<()> { self.ledger.check_next_block(block, &mut rand::thread_rng()) diff --git a/node/bft/ledger-service/src/mock.rs b/node/bft/ledger-service/src/mock.rs index a34e28d1e6..a0c335f2a5 100644 --- a/node/bft/ledger-service/src/mock.rs +++ b/node/bft/ledger-service/src/mock.rs @@ -16,7 +16,9 @@ use crate::{LedgerService, fmt_id}; use snarkvm::{ ledger::{ - block::{Block, Transaction}, + Block, + PendingBlock, + Transaction, committee::Committee, narwhal::{BatchCertificate, Data, Subdag, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, @@ -211,6 +213,14 @@ impl LedgerService for MockLedgerService { Ok(()) } + fn check_block_subdag(&self, _block: Block, _prefix: &[PendingBlock]) -> Result> { + unimplemented!(); + } + + fn check_block_content(&self, _block: PendingBlock) -> Result> { + unimplemented!(); + } + /// Checks the given block is valid next block. fn check_next_block(&self, _block: &Block) -> Result<()> { Ok(()) diff --git a/node/bft/ledger-service/src/prover.rs b/node/bft/ledger-service/src/prover.rs index 02854f3682..b496c58c95 100644 --- a/node/bft/ledger-service/src/prover.rs +++ b/node/bft/ledger-service/src/prover.rs @@ -16,7 +16,9 @@ use crate::LedgerService; use snarkvm::{ ledger::{ - block::{Block, Transaction}, + Block, + PendingBlock, + Transaction, committee::Committee, narwhal::{BatchCertificate, Data, Subdag, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, @@ -166,6 +168,14 @@ impl LedgerService for ProverLedgerService { Ok(()) } + fn check_block_subdag(&self, _block: Block, _prefix: &[PendingBlock]) -> Result> { + bail!("Cannot check block subDAG in prover") + } + + fn check_block_content(&self, _pending_block: PendingBlock) -> Result> { + bail!("Cannot check block content in prover") + } + /// Checks the given block is valid next block. fn check_next_block(&self, _block: &Block) -> Result<()> { Ok(()) diff --git a/node/bft/ledger-service/src/traits.rs b/node/bft/ledger-service/src/traits.rs index 842fb19f6f..99576fc534 100644 --- a/node/bft/ledger-service/src/traits.rs +++ b/node/bft/ledger-service/src/traits.rs @@ -15,7 +15,9 @@ use snarkvm::{ ledger::{ - block::{Block, Transaction}, + Block, + PendingBlock, + Transaction, committee::Committee, narwhal::{BatchCertificate, Data, Subdag, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, @@ -106,6 +108,12 @@ pub trait LedgerService: Debug + Send + Sync { transaction: Transaction, ) -> Result<()>; + /// Checks that the subDAG in a given block is valid, but does not fully verify the block. + fn check_block_subdag(&self, block: Block, prefix: &[PendingBlock]) -> Result>; + + /// Takes a pending block and performs the remaining checks to full verify it. + fn check_block_content(&self, _block: PendingBlock) -> Result>; + /// Checks the given block is valid next block. fn check_next_block(&self, block: &Block) -> Result<()>; diff --git a/node/bft/ledger-service/src/translucent.rs b/node/bft/ledger-service/src/translucent.rs index b4ffb988df..e7f38e2cdd 100644 --- a/node/bft/ledger-service/src/translucent.rs +++ b/node/bft/ledger-service/src/translucent.rs @@ -19,8 +19,10 @@ use snarkos_utilities::Stoppable; use snarkvm::{ ledger::{ + Block, Ledger, - block::{Block, Transaction}, + PendingBlock, + Transaction, committee::Committee, narwhal::{Data, Subdag, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, @@ -177,6 +179,14 @@ impl> LedgerService for TranslucentLedgerS Ok(()) } + fn check_block_subdag(&self, _block: Block, _prefix: &[PendingBlock]) -> Result> { + unimplemented!(); + } + + fn check_block_content(&self, _block: PendingBlock) -> Result> { + unimplemented!(); + } + /// Always succeeds. fn check_next_block(&self, _block: &Block) -> Result<()> { Ok(()) diff --git a/node/bft/src/sync/mod.rs b/node/bft/src/sync/mod.rs index fecf0cdc7e..a168915707 100644 --- a/node/bft/src/sync/mod.rs +++ b/node/bft/src/sync/mod.rs @@ -27,15 +27,16 @@ use snarkos_node_router::PeerPoolHandling; use snarkos_node_sync::{BLOCK_REQUEST_BATCH_DELAY, BlockSync, Ping, PrepareSyncRequest, locators::BlockLocators}; use snarkvm::{ console::{network::Network, types::Field}, - ledger::{authority::Authority, block::Block, narwhal::BatchCertificate}, + ledger::{PendingBlock, authority::Authority, block::Block, narwhal::BatchCertificate}, utilities::{ cfg_into_iter, cfg_iter, + ensure_equals, task::{self, JoinHandle}, }, }; -use anyhow::{Result, anyhow, bail}; +use anyhow::{Context, Result, anyhow, bail, ensure}; use indexmap::IndexMap; #[cfg(feature = "locktick")] use locktick::{parking_lot::Mutex, tokio::Mutex as TMutex}; @@ -44,7 +45,7 @@ use parking_lot::Mutex; #[cfg(not(feature = "serial"))] use rayon::prelude::*; use std::{ - collections::{BTreeMap, HashMap}, + collections::{HashMap, VecDeque}, future::Future, net::SocketAddr, sync::Arc, @@ -88,12 +89,12 @@ pub struct Sync { sync_lock: Arc>, /// The latest block responses. /// - /// This is used in [`Sync::sync_storage_with_block()`] to accumulate blocks whose addition to the ledger is - /// deferred until certain checks pass. + /// This is used in [`Sync::sync_storage_with_block()`] to accumulate blocks + /// whose addition to the ledger is deferred until certain checks pass. /// Blocks need to be processed in order, hence a BTree map. /// /// Whenever a new block is added to this map, BlockSync::set_sync_height needs to be called. - latest_block_responses: Arc>>>, + pending_blocks: Arc>>>, } impl Sync { @@ -119,7 +120,7 @@ impl Sync { handles: Default::default(), response_lock: Default::default(), sync_lock: Default::default(), - latest_block_responses: Default::default(), + pending_blocks: Default::default(), } } @@ -467,13 +468,17 @@ impl Sync { /// If there are queued block responses, this might be higher than the latest block in the ledger. async fn compute_sync_height(&self) -> u32 { let ledger_height = self.ledger.latest_block_height(); - let mut responses = self.latest_block_responses.lock().await; + let mut pending_blocks = self.pending_blocks.lock().await; // Remove any old responses. - responses.retain(|height, _| *height > ledger_height); + while let Some(b) = pending_blocks.front() + && b.height() <= ledger_height + { + pending_blocks.pop_front(); + } // Ensure the returned value is always greater or equal than ledger height. - responses.last_key_value().map(|(height, _)| *height).unwrap_or(0).max(ledger_height) + pending_blocks.back().map(|b| b.height()).unwrap_or(0).max(ledger_height) } /// BFT-version of [`snarkos_node_client::Client::try_advancing_block_synchronization`]. @@ -505,7 +510,7 @@ impl Sync { /// This returns Ok(true) if we successfully advanced the ledger by at least one new block. /// /// A key difference to `BlockSync`'s versions is that it will only add blocks to the ledger once they have been confirmed by the network. - /// If blocks are not confirmed yet, they will be kept in [`Self::latest_block_responses`]. + /// If blocks are not confirmed yet, they will be kept in [`Self::pending_blocks`]. /// It will also pass certificates from synced blocks to the BFT module so that consensus can progress as expected /// (see [`Self::sync_storage_with_block`] for more details). /// @@ -662,216 +667,195 @@ impl Sync { .await? } + /// Helper function for [`Self::sync_storage_with_block`]. + /// It syncs the batch certificates with the BFT, if the block's authority is a sub-DAG. + /// + /// Note that the block authority is always a sub-DAG in production; beacon signatures are only used for testing, + /// and as placeholder (irrelevant) block authority in the genesis block.i + async fn add_block_subdag_to_bft(&self, block: &Block) -> Result<()> { + // Nothing to do if this is a beacon block + let Authority::Quorum(subdag) = block.authority() else { + return Ok(()); + }; + + // Reconstruct the unconfirmed transactions. + let unconfirmed_transactions = cfg_iter!(block.transactions()) + .filter_map(|tx| tx.to_unconfirmed_transaction().map(|unconfirmed| (unconfirmed.id(), unconfirmed)).ok()) + .collect::>(); + + // Iterate over the certificates. + for certificates in subdag.values().cloned() { + cfg_into_iter!(certificates.clone()).for_each(|certificate| { + // Sync the batch certificate with the block. + self.storage.sync_certificate_with_block(block, certificate.clone(), &unconfirmed_transactions); + }); + + // Sync the BFT DAG with the certificates. + for certificate in certificates { + // If a BFT sender was provided, send the certificate to the BFT. + // For validators, BFT spawns a receiver task in `BFT::start_handlers`. + if let Some(bft_sender) = self.bft_sender.get() { + let (callback_tx, callback_rx) = oneshot::channel(); + bft_sender + .tx_sync_bft + .send((certificate, callback_tx)) + .await + .with_context(|| "Failed to sync certificate")?; + callback_rx.await?.with_context(|| "Failed to sync certificate")?; + } + } + } + Ok(()) + } + + /// Helper function for [`Self::sync_storage_with_block`]. + /// + /// It checks that successor of a given block contains enough votes to commit it. + /// This can only return `Ok(true)` if the certificates of the block's successor were added to the storage. + fn is_block_availability_threshold_reached(&self, block: &PendingBlock) -> Result { + // Fetch the leader certificate and the relevant rounds. + let leader_certificate = match block.authority() { + Authority::Quorum(subdag) => subdag.leader_certificate().clone(), + _ => bail!("Received a block with an unexpected authority type."), + }; + let commit_round = leader_certificate.round(); + let certificate_round = + commit_round.checked_add(1).ok_or_else(|| anyhow!("Integer overflow on round number"))?; + + // Get the committee lookback for the round just after the leader. + let certificate_committee_lookback = self.ledger.get_committee_lookback_for_round(certificate_round)?; + // Retrieve all of the certificates for the round just after the leader. + let certificates = self.storage.get_certificates_for_round(certificate_round); + // Construct a set over the authors, at the round just after the leader, + // who included the leader's certificate in their previous certificate IDs. + let authors = certificates + .iter() + .filter_map(|c| match c.previous_certificate_ids().contains(&leader_certificate.id()) { + true => Some(c.author()), + false => None, + }) + .collect(); + + // Check if the leader is ready to be committed. + if certificate_committee_lookback.is_availability_threshold_reached(&authors) { + trace!( + "Block {hash} at height {height} has reached availability threshold", + hash = block.hash(), + height = block.height() + ); + Ok(true) + } else { + Ok(false) + } + } + /// Advances the ledger by the given block and updates the storage accordingly. /// /// This also updates the DAG, and uses the DAG to ensure that the block's leader certificate /// meets the voter availability threshold (i.e. > f voting stake) /// or is reachable via a DAG path from a later leader certificate that does. /// Since performing this check requires DAG certificates from later blocks, - /// the block is stored in `Sync::latest_block_responses`, + /// the block is stored in `Sync::pending_blocks`, /// and its addition to the ledger is deferred until the check passes. - /// Several blocks may be stored in `Sync::latest_block_responses` + /// Several blocks may be stored in `Sync::pending_blocks` /// before they can be all checked and added to the ledger. - async fn sync_storage_with_block(&self, block: Block) -> Result<()> { + /// + /// # Usage + /// This function assumes that blocks are passed in order, i.e., + /// that the given block is a direct successor of the block that was last passed to this function. + async fn sync_storage_with_block(&self, new_block: Block) -> Result<()> { // Acquire the sync lock. let _lock = self.sync_lock.lock().await; // If this block has already been processed, return early. // TODO(kaimast): Should we remove the response here? - if self.ledger.contains_block_height(block.height()) { - debug!("Ledger is already synced with block at height {}. Will not sync.", block.height()); + if self.ledger.contains_block_height(new_block.height()) { + debug!( + "Ledger is already synced with block at height {height}. Will not sync.", + height = new_block.height() + ); return Ok(()); } - // Acquire the latest block responses lock. - let mut latest_block_responses = self.latest_block_responses.lock().await; - - if latest_block_responses.contains_key(&block.height()) { - debug!("An unconfirmed block is queued already for height {}. Will not sync.", block.height()); - return Ok(()); - } + // Append the certificates to the storage. + self.add_block_subdag_to_bft(&new_block).await?; - // If the block authority is a sub-DAG, then sync the batch certificates with the block. - // Note that the block authority is always a sub-DAG in production; - // beacon signatures are only used for testing, - // and as placeholder (irrelevant) block authority in the genesis block. - if let Authority::Quorum(subdag) = block.authority() { - // Reconstruct the unconfirmed transactions. - let unconfirmed_transactions = cfg_iter!(block.transactions()) - .filter_map(|tx| { - tx.to_unconfirmed_transaction().map(|unconfirmed| (unconfirmed.id(), unconfirmed)).ok() - }) - .collect::>(); - - // Iterate over the certificates. - for certificates in subdag.values().cloned() { - cfg_into_iter!(certificates.clone()).for_each(|certificate| { - // Sync the batch certificate with the block. - self.storage.sync_certificate_with_block(&block, certificate.clone(), &unconfirmed_transactions); - }); + // Acquire the pending blocks lock. + let mut pending_blocks = self.pending_blocks.lock().await; - // Sync the BFT DAG with the certificates. - for certificate in certificates { - // If a BFT sender was provided, send the certificate to the BFT. - // For validators, BFT spawns a receiver task in `BFT::start_handlers`. - if let Some(bft_sender) = self.bft_sender.get() { - // Await the callback to continue. - if let Err(err) = bft_sender.send_sync_bft(certificate).await { - bail!("Failed to sync certificate - {err}"); - }; - } - } + if let Some(tail) = pending_blocks.back() { + if tail.height() >= new_block.height() { + debug!( + "A unconfirmed block is queued already for height {height}. \ + Will not sync.", + height = new_block.height() + ); + return Ok(()); } + + ensure_equals!(tail.height() + 1, new_block.height(), "Got an out-of-order block"); } // Fetch the latest block height. let ledger_block_height = self.ledger.latest_block_height(); - // Insert the latest block response. - latest_block_responses.insert(block.height(), block); - // Clear the latest block responses of older blocks. - latest_block_responses.retain(|height, _| *height > ledger_block_height); + // Clear any older pending blocks. + // TODO(kaimast): ensure there are no dangling block requests + while let Some(pending_block) = pending_blocks.front() { + if pending_block.height() > ledger_block_height { + break; + } - // Get a list of contiguous blocks from the latest block responses. - let contiguous_blocks: Vec> = (ledger_block_height.saturating_add(1)..) - .take_while(|&k| latest_block_responses.contains_key(&k)) - .filter_map(|k| latest_block_responses.get(&k).cloned()) - .collect(); + pending_blocks.pop_front(); + } - // Check if each block response, from the contiguous sequence just constructed, - // is ready to be added to the ledger. - // Ensure that the block's leader certificate meets the availability threshold - // based on the certificates in the DAG just after the block's round. - // If the availability threshold is not met, - // process the next block and check if it is linked to the current block, - // in the sense that there is a path in the DAG - // from the next block's leader certificate - // to the current block's leader certificate. - // Note: We do not advance to the most recent block response because we would be unable to - // validate if the leader certificate in the block has been certified properly. - for next_block in contiguous_blocks.into_iter() { - // Retrieve the height of the next block. - let next_block_height = next_block.height(); - - // Fetch the leader certificate and the relevant rounds. - let leader_certificate = match next_block.authority() { - Authority::Quorum(subdag) => subdag.leader_certificate().clone(), - _ => bail!("Received a block with an unexpected authority type."), - }; - let commit_round = leader_certificate.round(); - let certificate_round = - commit_round.checked_add(1).ok_or_else(|| anyhow!("Integer overflow on round number"))?; - - // Get the committee lookback for the round just after the leader. - let certificate_committee_lookback = self.ledger.get_committee_lookback_for_round(certificate_round)?; - // Retrieve all of the certificates for the round just after the leader. - let certificates = self.storage.get_certificates_for_round(certificate_round); - // Construct a set over the authors, at the round just after the leader, - // who included the leader's certificate in their previous certificate IDs. - let authors = certificates - .iter() - .filter_map(|c| match c.previous_certificate_ids().contains(&leader_certificate.id()) { - true => Some(c.author()), - false => None, - }) - .collect(); - - debug!("Validating sync block {next_block_height} at round {commit_round}..."); - // Check if the leader is ready to be committed. - if certificate_committee_lookback.is_availability_threshold_reached(&authors) { - // Initialize the current certificate. - let mut current_certificate = leader_certificate; - // Check if there are any linked blocks that need to be added. - let mut blocks_to_add = vec![next_block]; - - // Check if there are other blocks to process based on `is_linked`. - for height in (self.ledger.latest_block_height().saturating_add(1)..next_block_height).rev() { - // Retrieve the previous block. - let Some(previous_block) = latest_block_responses.get(&height) else { - bail!("Block {height} is missing from the latest block responses."); - }; - // Retrieve the previous block's leader certificate. - let previous_certificate = match previous_block.authority() { - Authority::Quorum(subdag) => subdag.leader_certificate().clone(), - _ => bail!("Received a block with an unexpected authority type."), - }; - // Determine if there is a path between the previous certificate and the current certificate. - if self.is_linked(previous_certificate.clone(), current_certificate.clone())? { - debug!("Previous sync block {height} is linked to the current block {next_block_height}"); - // Add the previous leader certificate to the list of certificates to commit. - blocks_to_add.insert(0, previous_block.clone()); - // Update the current certificate to the previous leader certificate. - current_certificate = previous_certificate; - } - } + // Check the block against the chain of pending blocks and append it on success. + let new_block = self.ledger.check_block_subdag(new_block, pending_blocks.make_contiguous())?; + pending_blocks.push_back(new_block); + + // Now, figure out if and which pending block we can commit. + // To do this effectively and because commits are transitive, + // we iterate in reverse so that we can stop at the first successful check. + // + // Note, that if the storage already contains certificates for the round after new block, + // the availability threshold for the new block could also be reached. + let mut commit_height = None; + for block in pending_blocks.iter().rev() { + if self.is_block_availability_threshold_reached(block)? { + commit_height = Some(block.height()); + break; + } + } - // Add the blocks to the ledger. - for block in blocks_to_add { - // Check that the blocks are sequential and can be added to the ledger. - let block_height = block.height(); - if block_height != self.ledger.latest_block_height().saturating_add(1) { - warn!("Skipping block {block_height} from the latest block responses - not sequential."); - continue; - } - #[cfg(feature = "telemetry")] - let block_authority = block.authority().clone(); - - let self_ = self.clone(); - tokio::task::spawn_blocking(move || { - // Check the next block. - self_.ledger.check_next_block(&block)?; - // Attempt to advance to the next block. - self_.ledger.advance_to_next_block(&block)?; - - // Sync the height with the block. - self_.storage.sync_height_with_block(block.height()); - // Sync the round with the block. - self_.storage.sync_round_with_block(block.round()); - - Ok::<(), anyhow::Error>(()) - }) - .await??; - // Remove the block height from the latest block responses. - latest_block_responses.remove(&block_height); - - // Update the validator telemetry. - #[cfg(feature = "telemetry")] - if let Authority::Quorum(subdag) = block_authority { - self_.gateway.validator_telemetry().insert_subdag(&subdag); - } - } - } else { - debug!( - "Availability threshold was not reached for block {next_block_height} at round {commit_round}. Checking next block..." + if let Some(commit_height) = commit_height { + let start_height = ledger_block_height + 1; + ensure!(commit_height >= start_height, "Invalid commit height"); + let num_blocks = (commit_height - start_height + 1) as usize; + // Create a more detailed log message if we are committing more than one block at a time. + if num_blocks > 1 { + trace!( + "Attempting to commit {chain_length} pending block(s) starting at height {start_height}.", + chain_length = pending_blocks.len(), ); } - // Don't remove the response from BlockSync yet as we might fall back to non-BFT sync. + for pending_block in pending_blocks.drain(0..num_blocks) { + let hash = pending_block.hash(); + let height = pending_block.height(); + let block = self + .ledger + .check_block_content(pending_block) + .with_context(|| format!("Failed to check contents of pending block {hash} at height {height}"))?; + + trace!("Adding pending block {hash} at height {height} to the ledger"); + self.ledger + .advance_to_next_block(&block) + .with_context(|| "Failed to advance ledger by pending block")?; + } } Ok(()) } - - /// Returns `true` if there is a path from the previous certificate to the current certificate. - fn is_linked( - &self, - previous_certificate: BatchCertificate, - current_certificate: BatchCertificate, - ) -> Result { - // Initialize the list containing the traversal. - let mut traversal = vec![current_certificate.clone()]; - // Iterate over the rounds from the current certificate to the previous certificate. - for round in (previous_certificate.round()..current_certificate.round()).rev() { - // Retrieve all of the certificates for this past round. - let certificates = self.storage.get_certificates_for_round(round); - // Filter the certificates to only include those that are in the traversal. - traversal = certificates - .into_iter() - .filter(|p| traversal.iter().any(|c| c.previous_certificate_ids().contains(&p.id()))) - .collect(); - } - Ok(traversal.contains(&previous_certificate)) - } } // Methods to assist with the block sync module. @@ -1019,13 +1003,24 @@ mod tests { type CurrentLedger = Ledger>; type CurrentConsensusStore = ConsensusStore>; + /// Tests that commits work as expected when some anchors are not committed immediately. #[tokio::test] #[tracing_test::traced_test] - async fn test_commit_via_is_linked() -> anyhow::Result<()> { + async fn test_commit_chain() -> anyhow::Result<()> { let rng = &mut TestRng::default(); // Initialize the round parameters. let max_gc_rounds = BatchHeader::::MAX_GC_ROUNDS as u64; - let commit_round = 2; + + // The first round of the first block. + let first_round = 1; + // The total number of blocks we test + let num_blocks = 3; + // The number of certificate rounds needed. + // There is one additional round to provide availability for the inal block. + let num_rounds = first_round + num_blocks * 2 + 1; + // The first round that has at least N-f certificates referencing the anchor from the previous round. + // This is also the last round we use in the test. + let first_committed_round = num_rounds - 1; // Initialize the store. let store = CurrentConsensusStore::open(StorageMode::new_test(None)).unwrap(); @@ -1066,77 +1061,70 @@ mod tests { HashMap::new(); let mut previous_certificates: IndexSet> = IndexSet::with_capacity(4); - for round in 0..=commit_round + 8 { + for round in first_round..=first_committed_round { let mut current_certificates = IndexSet::new(); let previous_certificate_ids: IndexSet<_> = if round == 0 || round == 1 { IndexSet::new() } else { previous_certificates.iter().map(|c| c.id()).collect() }; + let committee_id = committee.id(); + let prev_leader = committee.get_leader(round - 1).unwrap(); + + // For the first two blocks non-leaders will not reference the leader certificate. + // This means, while there is an anchor, it is isn't committed + // until later. + for (i, private_key) in private_keys.iter().enumerate() { + let leader_index = addresses.iter().position(|&address| address == prev_leader).unwrap(); + let is_certificate_round = round % 2 == 1; + let is_leader = i == leader_index; + + let previous_certs = if round < first_committed_round && is_certificate_round && !is_leader { + previous_certificate_ids + .iter() + .cloned() + .enumerate() + .filter(|(idx, _)| *idx != leader_index) + .map(|(_, id)| id) + .collect() + } else { + previous_certificate_ids.clone() + }; - // Create a certificate for the leader. - if round <= 5 { - let leader = committee.get_leader(round).unwrap(); - let leader_index = addresses.iter().position(|&address| address == leader).unwrap(); - let non_leader_index = addresses.iter().position(|&address| address != leader).unwrap(); - for i in [leader_index, non_leader_index].into_iter() { - let batch_header = BatchHeader::new( - &private_keys[i], - round, - now(), - committee_id, - Default::default(), - previous_certificate_ids.clone(), - rng, - ) - .unwrap(); - // Sign the batch header. - let mut signatures = IndexSet::with_capacity(4); - for (j, private_key_2) in private_keys.iter().enumerate() { - if i != j { - signatures.insert(private_key_2.sign(&[batch_header.batch_id()], rng).unwrap()); - } - } - current_certificates.insert(BatchCertificate::from(batch_header, signatures).unwrap()); - } - } + let batch_header = BatchHeader::new( + private_key, + round, + now(), + committee_id, + Default::default(), + previous_certs, + rng, + ) + .unwrap(); - // Create a certificate for each validator. - if round > 5 { - for (i, private_key_1) in private_keys.iter().enumerate() { - let batch_header = BatchHeader::new( - private_key_1, - round, - now(), - committee_id, - Default::default(), - previous_certificate_ids.clone(), - rng, - ) - .unwrap(); - // Sign the batch header. - let mut signatures = IndexSet::with_capacity(4); - for (j, private_key_2) in private_keys.iter().enumerate() { - if i != j { - signatures.insert(private_key_2.sign(&[batch_header.batch_id()], rng).unwrap()); - } + // Sign the batch header. + let mut signatures = IndexSet::with_capacity(4); + for (j, private_key_2) in private_keys.iter().enumerate() { + if i != j { + signatures.insert(private_key_2.sign(&[batch_header.batch_id()], rng).unwrap()); } - current_certificates.insert(BatchCertificate::from(batch_header, signatures).unwrap()); } + current_certificates.insert(BatchCertificate::from(batch_header, signatures).unwrap()); } + // Update the map of certificates. round_to_certificates_map.insert(round, current_certificates.clone()); - previous_certificates = current_certificates.clone(); + previous_certificates = current_certificates; } (round_to_certificates_map, committee) }; // Initialize the storage. let storage = Storage::new(core_ledger.clone(), Arc::new(BFTMemoryService::new()), max_gc_rounds); - // Insert certificates into storage. + // Insert all certificates into storage. let mut certificates: Vec> = Vec::new(); - for i in 1..=commit_round + 8 { + for i in first_round..=first_committed_round { let c = (*round_to_certificates_map.get(&i).unwrap()).clone(); certificates.extend(c); } @@ -1144,102 +1132,81 @@ mod tests { storage.testing_only_insert_certificate_testing_only(certificate.clone()); } - // Create block 1. - let leader_round_1 = commit_round; - let leader_1 = committee.get_leader(leader_round_1).unwrap(); - let leader_certificate = storage.get_certificate_for_round_with_author(commit_round, leader_1).unwrap(); - let block_1 = { + // Create the blocks + let mut previous_leader_cert = None; + let mut blocks = vec![]; + + for block_height in 1..=num_blocks { + let leader_round = block_height * 2; + + let leader = committee.get_leader(leader_round).unwrap(); + let leader_certificate = storage.get_certificate_for_round_with_author(leader_round, leader).unwrap(); + let mut subdag_map: BTreeMap>> = BTreeMap::new(); let mut leader_cert_map = IndexSet::new(); leader_cert_map.insert(leader_certificate.clone()); - let mut previous_cert_map = IndexSet::new(); - for cert in storage.get_certificates_for_round(commit_round - 1) { - previous_cert_map.insert(cert); + + let previous_cert_map = storage.get_certificates_for_round(leader_round - 1); + + subdag_map.insert(leader_round, leader_cert_map.clone()); + subdag_map.insert(leader_round - 1, previous_cert_map.clone()); + + if leader_round > 2 { + let previous_commit_cert_map: IndexSet<_> = storage + .get_certificates_for_round(leader_round - 2) + .into_iter() + .filter(|cert| { + if let Some(previous_leader_cert) = &previous_leader_cert { + cert != previous_leader_cert + } else { + true + } + }) + .collect(); + subdag_map.insert(leader_round - 2, previous_commit_cert_map); } - subdag_map.insert(commit_round, leader_cert_map.clone()); - subdag_map.insert(commit_round - 1, previous_cert_map.clone()); + let subdag = Subdag::from(subdag_map.clone())?; - core_ledger.prepare_advance_to_next_quorum_block(subdag, Default::default())? - }; - // Insert block 1. - core_ledger.advance_to_next_block(&block_1)?; + let block = core_ledger.prepare_advance_to_next_quorum_block(subdag, Default::default())?; - // Create block 2. - let leader_round_2 = commit_round + 2; - let leader_2 = committee.get_leader(leader_round_2).unwrap(); - let leader_certificate_2 = storage.get_certificate_for_round_with_author(leader_round_2, leader_2).unwrap(); - let block_2 = { - let mut subdag_map_2: BTreeMap>> = BTreeMap::new(); - let mut leader_cert_map_2 = IndexSet::new(); - leader_cert_map_2.insert(leader_certificate_2.clone()); - let mut previous_cert_map_2 = IndexSet::new(); - for cert in storage.get_certificates_for_round(leader_round_2 - 1) { - previous_cert_map_2.insert(cert); - } - let mut prev_commit_cert_map_2 = IndexSet::new(); - for cert in storage.get_certificates_for_round(leader_round_2 - 2) { - if cert != leader_certificate { - prev_commit_cert_map_2.insert(cert); - } - } - subdag_map_2.insert(leader_round_2, leader_cert_map_2.clone()); - subdag_map_2.insert(leader_round_2 - 1, previous_cert_map_2.clone()); - subdag_map_2.insert(leader_round_2 - 2, prev_commit_cert_map_2.clone()); - let subdag_2 = Subdag::from(subdag_map_2.clone())?; - core_ledger.prepare_advance_to_next_quorum_block(subdag_2, Default::default())? - }; - // Insert block 2. - core_ledger.advance_to_next_block(&block_2)?; + previous_leader_cert = Some(leader_certificate); - // Create block 3 - let leader_round_3 = commit_round + 4; - let leader_3 = committee.get_leader(leader_round_3).unwrap(); - let leader_certificate_3 = storage.get_certificate_for_round_with_author(leader_round_3, leader_3).unwrap(); - let block_3 = { - let mut subdag_map_3: BTreeMap>> = BTreeMap::new(); - let mut leader_cert_map_3 = IndexSet::new(); - leader_cert_map_3.insert(leader_certificate_3.clone()); - let mut previous_cert_map_3 = IndexSet::new(); - for cert in storage.get_certificates_for_round(leader_round_3 - 1) { - previous_cert_map_3.insert(cert); - } - let mut prev_commit_cert_map_3 = IndexSet::new(); - for cert in storage.get_certificates_for_round(leader_round_3 - 2) { - if cert != leader_certificate_2 { - prev_commit_cert_map_3.insert(cert); - } - } - subdag_map_3.insert(leader_round_3, leader_cert_map_3.clone()); - subdag_map_3.insert(leader_round_3 - 1, previous_cert_map_3.clone()); - subdag_map_3.insert(leader_round_3 - 2, prev_commit_cert_map_3.clone()); - let subdag_3 = Subdag::from(subdag_map_3.clone())?; - core_ledger.prepare_advance_to_next_quorum_block(subdag_3, Default::default())? - }; - // Insert block 3. - core_ledger.advance_to_next_block(&block_3)?; + core_ledger.advance_to_next_block(&block)?; + blocks.push(block); + } + + // ### Test that sync works as expected ### + let storage_mode = StorageMode::Test(None); - // Initialize the syncing ledger. - let storage_mode = StorageMode::new_test(None); + // Create a new ledger to test with, but use the existing storage + // so that the certificates exist. let syncing_ledger = Arc::new(CoreLedgerService::new( CurrentLedger::load(genesis, storage_mode.clone()).unwrap(), SimpleStoppable::new(), )); - // Initialize the gateway. + + // Set up sync and its dependencies. let gateway = Gateway::new(account.clone(), storage.clone(), syncing_ledger.clone(), None, &[], storage_mode, None)?; - // Initialize the block synchronization logic. let block_sync = Arc::new(BlockSync::new(syncing_ledger.clone())); - // Initialize the sync module. let sync = Sync::new(gateway.clone(), storage.clone(), syncing_ledger.clone(), block_sync); - // Try to sync block 1. - sync.sync_storage_with_block(block_1).await?; - assert_eq!(syncing_ledger.latest_block_height(), 1); - // Try to sync block 2. - sync.sync_storage_with_block(block_2).await?; - assert_eq!(syncing_ledger.latest_block_height(), 2); - // Try to sync block 3. - sync.sync_storage_with_block(block_3).await?; + + let mut block_iter = blocks.into_iter(); + + // Insert the blocks into the new sync module + for _ in 0..num_blocks - 1 { + let block = block_iter.next().unwrap(); + sync.sync_storage_with_block(block).await?; + + // Availability threshold is not met, so we should not advance yet. + assert_eq!(syncing_ledger.latest_block_height(), 0); + } + + // Only for the final block, the availability threshold is met, + // because certificates for the subsequent round are already in storage. + sync.sync_storage_with_block(block_iter.next().unwrap()).await?; assert_eq!(syncing_ledger.latest_block_height(), 3); + // Ensure blocks 1 and 2 were added to the ledger. assert!(syncing_ledger.contains_block_height(1)); assert!(syncing_ledger.contains_block_height(2)); diff --git a/node/bft/src/worker.rs b/node/bft/src/worker.rs index eda99b96f0..03017590cc 100644 --- a/node/bft/src/worker.rs +++ b/node/bft/src/worker.rs @@ -26,7 +26,7 @@ use snarkos_node_bft_ledger_service::LedgerService; use snarkvm::{ console::{network::Network, prelude::Read}, ledger::{ - block::Transaction, + Transaction, narwhal::{BatchHeader, Data, Transmission, TransmissionID}, puzzle::{Solution, SolutionID}, }, @@ -567,7 +567,8 @@ mod tests { types::{Address, Field}, }, ledger::{ - block::Block, + Block, + PendingBlock, committee::Committee, narwhal::{BatchCertificate, Subdag, Transmission, TransmissionID}, test_helpers::sample_execution_transaction_with_fee, @@ -636,6 +637,8 @@ mod tests { transaction_id: N::TransactionID, transaction: Transaction, ) -> Result<()>; + fn check_block_subdag(&self, _block: Block, _prefix: &[PendingBlock]) -> Result>; + fn check_block_content(&self, _block: PendingBlock) -> Result>; fn check_next_block(&self, block: &Block) -> Result<()>; fn prepare_advance_to_next_quorum_block( &self, From 1432ad67ca14f3322666d03dbaa13c1801a5f4f3 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Thu, 2 Oct 2025 12:54:01 -0700 Subject: [PATCH 10/10] feat(node/bft): log panics in sync block advancement --- node/bft/src/bft.rs | 2 +- node/bft/src/sync/mod.rs | 70 ++++++++++++++++---------- node/consensus/src/lib.rs | 26 ++++++---- node/src/client/mod.rs | 7 +-- node/sync/src/block_sync.rs | 4 +- node/sync/src/block_sync/sync_state.rs | 2 +- 6 files changed, 67 insertions(+), 44 deletions(-) diff --git a/node/bft/src/bft.rs b/node/bft/src/bft.rs index 2d878d7bfb..9db84100fc 100644 --- a/node/bft/src/bft.rs +++ b/node/bft/src/bft.rs @@ -722,7 +722,7 @@ impl BFT { } info!( - "\n\nCommitting a subdag from round {anchor_round} with {num_transmissions} transmissions: {subdag_metadata:?}\n" + "\n\nCommitting a subDAG with anchor round {anchor_round} and {num_transmissions} transmissions: {subdag_metadata:?} (syncing={IS_SYNCING})\n", ); } diff --git a/node/bft/src/sync/mod.rs b/node/bft/src/sync/mod.rs index a168915707..773a49a4fd 100644 --- a/node/bft/src/sync/mod.rs +++ b/node/bft/src/sync/mod.rs @@ -29,6 +29,7 @@ use snarkvm::{ console::{network::Network, types::Field}, ledger::{PendingBlock, authority::Authority, block::Block, narwhal::BatchCertificate}, utilities::{ + LoggableError, cfg_into_iter, cfg_iter, ensure_equals, @@ -194,7 +195,17 @@ impl Sync { let _ = tokio::time::timeout(Self::MAX_SYNC_INTERVAL, self_.block_sync.wait_for_block_responses()).await; - self_.try_advancing_block_synchronization(&ping).await; + let ping = ping.clone(); + let self_ = self_.clone(); + let hdl = tokio::spawn(async move { + self_.try_advancing_block_synchronization(&ping).await; + }); + + if let Err(err) = hdl.await + && let Ok(panic) = err.try_into_panic() + { + error!("Sync block advancement panicked: {panic:?}"); + } // We perform no additional rate limiting here as // requests are already rate-limited. @@ -237,7 +248,13 @@ impl Sync { let self_ = self.clone(); self.spawn(async move { while let Some((peer_ip, blocks, callback)) = rx_block_sync_insert_block_response.recv().await { - callback.send(self_.insert_block_response(peer_ip, blocks).await).ok(); + let result = self_.insert_block_response(peer_ip, blocks).await; + //TODO remove this once channels are gone + if let Err(err) = &result { + warn!("Failed to insret block response: {err:?}"); + } + + callback.send(result).ok(); } }); @@ -487,7 +504,7 @@ impl Sync { let new_blocks = match self.try_advancing_block_synchronization_inner().await { Ok(new_blocks) => new_blocks, Err(err) => { - error!("Block synchronization failed - {err}"); + err.log_error("Block synchronization failed"); false } }; @@ -497,7 +514,7 @@ impl Sync { { match self.get_block_locators() { Ok(locators) => ping.update_block_locators(locators), - Err(err) => error!("Failed to update block locators: {err}"), + Err(err) => err.log_error("Failed to update block locators"), } } } @@ -561,10 +578,6 @@ impl Sync { // (unconfirmed) blocks that are already queued up. let start_height = self.compute_sync_height().await; - // For sanity, update the sync height before starting. - // (if this is lower or equal to the current sync height, this is a noop) - self.block_sync.set_sync_height(start_height); - // The height is incremented as blocks are added. let mut current_height = start_height; trace!("Try advancing with block responses (at block {current_height})"); @@ -599,10 +612,6 @@ impl Sync { let start_height = ledger_height; let mut current_height = start_height; - // For sanity, update the sync height before starting. - // (if this is lower or equal to the current sync height, this is a noop) - self.block_sync.set_sync_height(start_height); - // Try to advance the ledger *to tip* without updating the BFT. // TODO(kaimast): why to tip and not to tip-GC? loop { @@ -783,6 +792,19 @@ impl Sync { // Acquire the pending blocks lock. let mut pending_blocks = self.pending_blocks.lock().await; + // Fetch the latest block height. + let ledger_block_height = self.ledger.latest_block_height(); + + // First, clear any older pending blocks. + // TODO(kaimast): ensure there are no dangling block requests + while let Some(pending_block) = pending_blocks.front() { + if pending_block.height() > ledger_block_height { + break; + } + + pending_blocks.pop_front(); + } + if let Some(tail) = pending_blocks.back() { if tail.height() >= new_block.height() { debug!( @@ -796,21 +818,12 @@ impl Sync { ensure_equals!(tail.height() + 1, new_block.height(), "Got an out-of-order block"); } - // Fetch the latest block height. - let ledger_block_height = self.ledger.latest_block_height(); - - // Clear any older pending blocks. - // TODO(kaimast): ensure there are no dangling block requests - while let Some(pending_block) = pending_blocks.front() { - if pending_block.height() > ledger_block_height { - break; - } - - pending_blocks.pop_front(); - } - // Check the block against the chain of pending blocks and append it on success. - let new_block = self.ledger.check_block_subdag(new_block, pending_blocks.make_contiguous())?; + // TODO(kaimast): handle the case where the ledger already advance better + let new_block = self + .ledger + .check_block_subdag(new_block, pending_blocks.make_contiguous()) + .with_context(|| "SubDAG check failed")?; pending_blocks.push_back(new_block); // Now, figure out if and which pending block we can commit. @@ -821,7 +834,10 @@ impl Sync { // the availability threshold for the new block could also be reached. let mut commit_height = None; for block in pending_blocks.iter().rev() { - if self.is_block_availability_threshold_reached(block)? { + if self + .is_block_availability_threshold_reached(block) + .with_context(|| "Availability threshold check failed")? + { commit_height = Some(block.height()); break; } diff --git a/node/consensus/src/lib.rs b/node/consensus/src/lib.rs index 44be5c7176..9c5b9f82f6 100644 --- a/node/consensus/src/lib.rs +++ b/node/consensus/src/lib.rs @@ -50,6 +50,7 @@ use snarkvm::{ puzzle::{Solution, SolutionID}, }, prelude::*, + utilities::LoggableError, }; use aleo_std::StorageMode; @@ -337,13 +338,16 @@ impl Consensus { let solution_id = solution.id(); trace!("Adding unconfirmed solution '{}' to the memory pool...", fmt_id(solution_id)); // Send the unconfirmed solution to the primary. - if let Err(e) = self.primary_sender.send_unconfirmed_solution(solution_id, Data::Object(solution)).await { + if let Err(err) = self.primary_sender.send_unconfirmed_solution(solution_id, Data::Object(solution)).await { // If the BFT is synced, then log the warning. if self.bft.is_synced() { // If error occurs after the first 10 blocks of the epoch, log it as a warning, otherwise ignore. if self.ledger.latest_block_height() % N::NUM_BLOCKS_PER_EPOCH > 10 { - warn!("Failed to add unconfirmed solution '{}' to the memory pool - {e}", fmt_id(solution_id)) - }; + err.log_warning(format!( + "Failed to add unconfirmed solution '{}' to the memory pool", + fmt_id(solution_id) + )); + } } } } @@ -436,15 +440,15 @@ impl Consensus { }; trace!("Adding unconfirmed {tx_type_str} transaction '{}' to the memory pool...", fmt_id(transaction_id)); // Send the unconfirmed transaction to the primary. - if let Err(e) = + if let Err(err) = self.primary_sender.send_unconfirmed_transaction(transaction_id, Data::Object(transaction)).await { // If the BFT is synced, then log the warning. if self.bft.is_synced() { - warn!( - "Failed to add unconfirmed {tx_type_str} transaction '{}' to the memory pool - {e}", + err.log_warning(format!( + "Failed to add unconfirmed {tx_type_str} transaction '{}' to the memory pool", fmt_id(transaction_id) - ); + )); } } } @@ -477,12 +481,12 @@ impl Consensus { // Sleep briefly. tokio::time::sleep(Duration::from_millis(MAX_BATCH_DELAY_IN_MS)).await; // Process the unconfirmed transactions in the memory pool. - if let Err(e) = self_.process_unconfirmed_transactions().await { - warn!("Cannot process unconfirmed transactions - {e}"); + if let Err(err) = self_.process_unconfirmed_transactions().await { + err.log_warning("Cannot process unconfirmed transactions"); } // Process the unconfirmed solutions in the memory pool. - if let Err(e) = self_.process_unconfirmed_solutions().await { - warn!("Cannot process unconfirmed solutions - {e}"); + if let Err(err) = self_.process_unconfirmed_solutions().await { + err.log_warning("Cannot process unconfirmed solutions"); } } }); diff --git a/node/src/client/mod.rs b/node/src/client/mod.rs index e1851c5031..82ec52c2b0 100644 --- a/node/src/client/mod.rs +++ b/node/src/client/mod.rs @@ -47,6 +47,7 @@ use snarkvm::{ store::ConsensusStorage, }, prelude::{VM, block::Transaction}, + utilities::LoggableError, }; use aleo_std::StorageMode; @@ -290,7 +291,7 @@ impl> Client { let has_new_blocks = match self.sync.try_advancing_block_synchronization().await { Ok(val) => val, Err(err) => { - error!("Block synchronization failed - {err}"); + err.log_error("Block synchronization failed"); return; } }; @@ -299,7 +300,7 @@ impl> Client { if has_new_blocks { match self.sync.get_block_locators() { Ok(locators) => self.ping.update_block_locators(locators), - Err(err) => error!("Failed to get block locators: {err}"), + Err(err) => err.log_error("Failed to get block locators"), } } } @@ -331,7 +332,7 @@ impl> Client { // If there are no block requests, but there are pending block responses in the sync pool, // then try to advance the ledger using these pending block responses. - if block_requests.is_empty() { + if !block_requests.is_empty() { let total_requests = self.sync.num_total_block_requests(); let num_outstanding = self.sync.num_outstanding_block_requests(); if total_requests > 0 { diff --git a/node/sync/src/block_sync.rs b/node/sync/src/block_sync.rs index 0bc4e1dba1..2e175e0169 100644 --- a/node/sync/src/block_sync.rs +++ b/node/sync/src/block_sync.rs @@ -887,6 +887,8 @@ impl BlockSync { entry.response = Some(block.clone()); } + trace!("Received a new and valid block response for height {height}"); + // Notify the sync loop that something changed. self.response_notify.notify_one(); @@ -1070,7 +1072,7 @@ impl BlockSync { /// Returns `None` if there are no peers to sync from. /// /// # Locking - /// This function will read-lock `common_ancstors`. + /// This function will read-lock `common_ancestors`. fn find_sync_peers_inner(&self, current_height: u32) -> Option<(IndexMap>, u32)> { // Retrieve the latest ledger height. let latest_ledger_height = self.ledger.latest_block_height(); diff --git a/node/sync/src/block_sync/sync_state.rs b/node/sync/src/block_sync/sync_state.rs index ba6b480555..f37189dd97 100644 --- a/node/sync/src/block_sync/sync_state.rs +++ b/node/sync/src/block_sync/sync_state.rs @@ -138,7 +138,7 @@ impl SyncState { if new_sync_val { let elapsed = - if elapsed < 60 { format!("{elapsed} seconds") } else { format!("{} minutes", elapsed / 60) }; + if elapsed < 60 { format!("{elapsed} second(s)") } else { format!("{} minute(s)", elapsed / 60) }; debug!("Block sync state changed to \"synced\". It took {elapsed} to catch up with the network."); } else {