From c5c738f412e31fa50b6b8948fdcea95923bc8673 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Fri, 12 Dec 2025 11:12:53 -0800 Subject: [PATCH 1/9] feat: add better devnett peering and `--dev-num-clients` flag --- .ci/devnet_ci.sh | 63 ++++++++---- .ci/utils.sh | 3 + cli/src/commands/start.rs | 206 +++++++++++++++++++++++++++----------- cli/src/helpers/dev.rs | 21 ++++ 4 files changed, 211 insertions(+), 82 deletions(-) diff --git a/.ci/devnet_ci.sh b/.ci/devnet_ci.sh index 41f28c0ca0..900554b5dd 100755 --- a/.ci/devnet_ci.sh +++ b/.ci/devnet_ci.sh @@ -16,14 +16,15 @@ network_id=$3 min_height=$4 # The verobsity of snarkos nodes. -NODE_VERBOSITY=1 +NODE_VERBOSITY=3 # Default values if not provided : "${total_validators:=4}" -: "${total_clients:=2}" +: "${total_clients:=4}" # need at least 4 clients, so each validator has at least one client connected to it. : "${network_id:=0}" : "${min_height:=60}" # To likely go past the 100 round garbage collection limit. +# shellcheck source=SCRIPTDIR/utils.sh . ./.ci/utils.sh # Determine network name based on network_id @@ -57,20 +58,9 @@ trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" fail # Flags used by all nodes. common_flags=( --nodisplay --nobanner --noupdater "--network=$network_id" "--verbosity=$NODE_VERBOSITY" - "--dev-num-validators=$total_validators" + "--dev-num-validators=$total_validators" "--dev-num-clients=$total_clients" ) -# Set the trusted peers for the validators as they will not allow connections from unknown peers. -trusted_peers="" -for ((client_index = 0; client_index < total_clients; client_index++)); do - node_index=$((client_index + total_validators)) - if (( client_index == 0 )); then - trusted_peers+="$localhost:$((4130+node_index))" - else - trusted_peers+=",$localhost:$((4130+node_index))" - fi -done - # Start all validator nodes in the background for ((validator_index = 0; validator_index < total_validators; validator_index++)); do snarkos clean "--dev=$validator_index" "--network=$network_id" @@ -78,10 +68,10 @@ for ((validator_index = 0; validator_index < total_validators; validator_index++ log_file="$log_dir/validator-$validator_index.log" if [ $validator_index -eq 0 ]; then snarkos start "${common_flags[@]}" "--dev=$validator_index" \ - --validator "--logfile=$log_file" --metrics --no-dev-txs "--peers=$trusted_peers" & + --validator "--logfile=$log_file" --metrics --no-dev-txs & else snarkos start "${common_flags[@]}" "--dev=$validator_index" \ - --validator "--logfile=$log_file" "--peers=$trusted_peers" & + --validator "--logfile=$log_file" & fi PIDS[validator_index]=$! echo "Started validator $validator_index with PID ${PIDS[$validator_index]}" @@ -111,6 +101,16 @@ done # Ensure all nodes are up and running. wait_for_nodes "$total_validators" "$total_clients" +# Ensure all nodes have at least one peer +echo "ℹ️ Waiting for all nodes to have at least one peer..." +SECONDS=0 +for node_index in $(seq 0 $((total_clients+total_validators))); do + if ! (wait_for_peers "$node_index" 1); then + exit 1 + fi +done +echo "✅ All nodes have at least one peer" + last_seen_consensus_version=0 last_seen_height=0 @@ -118,7 +118,14 @@ last_seen_height=0 function consensus_version_stable() { consensus_version=$(curl -s "http://$localhost:3030/v2/$network_name/consensus_version") height=$(curl -s "http://$localhost:3030/v2/$network_name/block/height/latest") - if (is_integer "$consensus_version") && (is_integer "$height"); then + + if (! is_integer "$consensus_version"); then + echo "❌ Failed to retrieve consensus version: $consensus_version" + return 1 + elif (! is_integer "$height"); then + echo "❌ Failed to retrieve height: $height" + return 1 + else # If the consensus version is greater than the last seen, we update it. if (( consensus_version > last_seen_consensus_version )); then echo "✅ Consensus version updated to $consensus_version" @@ -129,19 +136,21 @@ function consensus_version_stable() { return 0 fi fi - else - echo "❌ Failed to retrieve consensus version or height: $consensus_version, $height" - exit 1 + + last_seen_consensus_version=$consensus_version + last_seen_height=$height fi - last_seen_consensus_version=$consensus_version - last_seen_height=$height + return 1 } # Check consensus versions periodically with a timeout +echo "ℹ️ Waiting for consensus version to stabilize..." total_wait=0 +version_stable=false while (( total_wait < 300 )); do # 5 minutes max if consensus_version_stable; then + version_stable=true break fi @@ -151,7 +160,17 @@ while (( total_wait < 300 )); do # 5 minutes max echo "Waited $total_wait seconds so far..." done +if ! $version_stable; then + echo "❌ Test failed! Consensus version did not stabilize within 5 minutes." + exit 1 +fi + # Creates a test program. + +if ! $version_stable; then + echo "❌ Test failed! Consensus version did not stabilize within 5 minutes." + exit 1 +fi mkdir -p program program_name="test_program.aleo" echo "program ${program_name}; diff --git a/.ci/utils.sh b/.ci/utils.sh index 7cf5617885..26c015c85a 100644 --- a/.ci/utils.sh +++ b/.ci/utils.sh @@ -251,6 +251,9 @@ function wait_for_nodes() { echo "All nodes are ready!" return 0 fi + + # Pause to give the nodes time to start up. + sleep 1 done } diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index c35a9bcd70..4ffa7ebb28 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -62,7 +62,7 @@ use tokio::{ sync::mpsc, task, }; -use tracing::warn; +use tracing::{debug, info, warn}; use ureq::http; /// The recommended minimum number of 'open files' limit for a validator. @@ -70,7 +70,7 @@ use ureq::http; #[cfg(target_family = "unix")] const RECOMMENDED_MIN_NOFILES_LIMIT: u64 = 2048; -/// A mapping of `staker_address` to `(validator_address, withdrawal_address, amount)`. +// A mapping of `staker_address` to `(validator_address, withdrawal_address, amount)`. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct BondedBalances(IndexMap); @@ -82,7 +82,7 @@ impl FromStr for BondedBalances { } } -/// Starts the snarkOS node. +// Starts the snarkOS node. #[derive(Clone, Debug, Parser)] #[command( // Use kebab-case for all arguments (e.g., use the `private-key` flag for the `private_key` field). @@ -256,15 +256,20 @@ pub struct Start { pub dev: Option, /// If development mode is enabled, specify the number of genesis validator. - #[clap(long, group = "dev-flags", default_value_t=DEVELOPMENT_MODE_NUM_GENESIS_COMMITTEE_MEMBERS)] + #[clap(long, group = "dev_flags", default_value_t=DEVELOPMENT_MODE_NUM_GENESIS_COMMITTEE_MEMBERS)] pub dev_num_validators: u16, + /// If development mode is enabled, specify the number of clients. + /// This is only used by validators to automatically populate their set of trusted peers. + #[clap(long, group = "dev_flags")] + pub dev_num_clients: Option, + /// If development mode is enabled, specify whether node 0 should generate traffic to drive the network. - #[clap(long, group = "dev-flag")] + #[clap(long, group = "dev_flag")] pub no_dev_txs: bool, /// If development mode is enabled, specify the custom bonded balances as a JSON object. - #[clap(long, group = "dev-flags")] + #[clap(long, group = "dev_flags")] pub dev_bonded_balances: Option, } @@ -384,43 +389,95 @@ impl Start { } /// Updates the configurations if the node is in development mode. - fn parse_development(&mut self, trusted_peers: &mut Vec, trusted_validators: &mut Vec) { + fn parse_development( + &mut self, + trusted_peers: &mut Vec, + trusted_validators: &mut Vec, + ) -> Result<()> { + // If `--dev` is not set, return early. + let Some(dev) = self.dev else { + return Ok(()); + }; + + // Determine the number of development validators. + let num_validators = self.dev_num_validators; + ensure!(num_validators >= 4, "Value for `dev_num_validators` is too low. Needs to be at least 4."); + // If `--dev` is set, assume the dev nodes are initialized from 0 to `dev`, // and add each of them to the trusted peers. In addition, set the node IP to `4130 + dev`, // and the REST port to `3030 + dev`. + info!("Development mode enabled with index={dev} and num_validators={num_validators}."); + + // Nodes only start as validators if the `--validator` flag is set, because the default mode is "client". + let is_validator = self.validator; - if let Some(dev) = self.dev { - // Add the dev nodes to the trusted peers. - if trusted_peers.is_empty() { - for i in 0..dev { - trusted_peers.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, DEFAULT_NODE_PORT + i))); + // Ensure the node type and `dev_num_validators` are compatible. + if is_validator { + ensure!( + dev < num_validators, + "Development validator index is too high (dev={dev}, dev_num_validators={num_validators})", + ); + } else { + ensure!( + dev >= self.dev_num_validators, + "Development prover/client index is too low (dev={dev}, dev_num_validators={num_validators})", + ); + } + + // Add the dev nodes to the trusted validators. + if trusted_validators.is_empty() && is_validator { + // Validators add all other validators as trusted. + for idx in 0..num_validators { + if idx == dev { + continue; } + trusted_validators.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, MEMORY_POOL_PORT + idx))); } - // Add the dev nodes to the trusted validators. - if trusted_validators.is_empty() { - // To avoid ambiguity, we define the first few nodes to be the trusted validators to connect to. - for i in 0..2 { - // Don't connect to yourself. - if i == dev { - continue; - } + } - trusted_validators - .push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, MEMORY_POOL_PORT + i))); + // Determine if we need to populate `trusted_peers`. + if trusted_peers.is_empty() { + if is_validator { + if let Some(num_clients) = self.dev_num_clients { + // Ensure the clients that added this validator as a trusted peer are able to connect to it. + for client_idx in 0..num_clients { + if get_devnet_validators_for_client(client_idx, num_validators).contains(&dev) { + let node_idx = num_validators + client_idx; + trusted_peers.push(get_devnet_router_address_for_node(node_idx)); + } + } + } else { + warn!( + "Development validator started without trusted peers or `--dev-num-clients`. No clients will be able to connect to it." + ); + } + } else { + // Clients/provers add two validators to connect to. + for validator_idx in get_devnet_validators_for_client(dev, num_validators) { + trusted_peers.push(get_devnet_router_address_for_node(validator_idx)); } } - // Set the node IP to `4130 + dev`. - // - // Note: the `node` flag is an option to detect remote devnet testing. - if self.node.is_none() { - self.node = Some(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, DEFAULT_NODE_PORT + dev))); - } + } else { + debug!("Trusted peers/validators was set manually. Will not populate them with development addresses.") + } - // If the `norest` flag is not set and the REST IP is not already specified set the REST IP to `3030 + dev`. - if !self.norest && self.rest.is_none() { - self.rest = Some(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, DEFAULT_REST_PORT + dev))); - } + // Set the node's listening port to `4130 + dev`. + // + // Note: the `node` flag is an option to detect remote devnet testing. + if self.node.is_none() { + let address = get_devnet_router_address_for_node(dev); + debug!("Setting node address to {address} due to dev={dev}"); + self.node = Some(address); + } + + // If the `norest` flag is not set and the REST IP is not already specified set the REST IP to `3030 + dev`. + if !self.norest && self.rest.is_none() { + let port = DEFAULT_REST_PORT + dev; + debug!("Setting REST port to {port} due to dev={dev}"); + self.rest = Some(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, port))); } + + Ok(()) } /// Returns an alternative genesis block if the node is in development mode. @@ -599,7 +656,7 @@ impl Start { } // Parse the development configurations. - self.parse_development(&mut trusted_peers, &mut trusted_validators); + self.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); // Parse the CDN. let cdn = self.parse_cdn::().with_context(|| "Failed to parse given CDN URL")?; @@ -1065,7 +1122,7 @@ mod tests { let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; let mut config = Start::try_parse_from(["snarkos"].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); let candidate_genesis = config.parse_genesis::().unwrap(); assert_eq!(trusted_peers.len(), 0); assert_eq!(trusted_validators.len(), 0); @@ -1073,78 +1130,89 @@ mod tests { let _config = Start::try_parse_from(["snarkos", "--dev", ""].iter()).unwrap_err(); + // Validator dev mode with default settings. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; - let mut config = Start::try_parse_from(["snarkos", "--dev", "1"].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + let mut config = Start::try_parse_from(["snarkos", "--dev", "1", "--validator"].iter()).unwrap(); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3031").unwrap())); + assert_eq!(trusted_validators.len(), 3); + // Validator dev mode with `--rest` flag. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; - let mut config = Start::try_parse_from(["snarkos", "--dev", "1", "--rest", "127.0.0.1:8080"].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + let mut config = + Start::try_parse_from(["snarkos", "--dev", "1", "--rest", "127.0.0.1:8080", "--validator"].iter()).unwrap(); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); assert_eq!(config.rest, Some(SocketAddr::from_str("127.0.0.1:8080").unwrap())); + assert_eq!(trusted_validators.len(), 3); + // Validator dev mode with `--norest` flag. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; - let mut config = Start::try_parse_from(["snarkos", "--dev", "1", "--norest"].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + let mut config = Start::try_parse_from(["snarkos", "--dev", "1", "--norest", "--validator"].iter()).unwrap(); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); assert!(config.rest.is_none()); + assert_eq!(trusted_validators.len(), 3); + // Client dev node. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; - let mut config = Start::try_parse_from(["snarkos", "--dev", "0"].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + let mut config = Start::try_parse_from(["snarkos", "--dev", "5"].iter()).unwrap(); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); let expected_genesis = config.parse_genesis::().unwrap(); - assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4130").unwrap())); - assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3030").unwrap())); - assert_eq!(trusted_peers.len(), 0); - assert_eq!(trusted_validators.len(), 1); + assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4135").unwrap())); + assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3035").unwrap())); + assert_eq!(trusted_peers.len(), 2); + assert_eq!(trusted_validators.len(), 0); assert!(!config.validator); assert!(!config.prover); assert!(!config.client); assert_ne!(expected_genesis, prod_genesis); + // Validator dev node with `--private-key` flag. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; let mut config = Start::try_parse_from(["snarkos", "--dev", "1", "--validator", "--private-key", ""].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); let genesis = config.parse_genesis::().unwrap(); assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4131").unwrap())); assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3031").unwrap())); - assert_eq!(trusted_peers.len(), 1); - assert_eq!(trusted_validators.len(), 1); + assert_eq!(trusted_peers.len(), 0); + assert_eq!(trusted_validators.len(), 3); assert!(config.validator); assert!(!config.prover); assert!(!config.client); assert_eq!(genesis, expected_genesis); + // Prover dev node with `--private-key` flag. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; let mut config = - Start::try_parse_from(["snarkos", "--dev", "2", "--prover", "--private-key", ""].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + Start::try_parse_from(["snarkos", "--dev", "6", "--prover", "--private-key", ""].iter()).unwrap(); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); let genesis = config.parse_genesis::().unwrap(); - assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4132").unwrap())); - assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3032").unwrap())); + assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4136").unwrap())); + assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3036").unwrap())); assert_eq!(trusted_peers.len(), 2); - assert_eq!(trusted_validators.len(), 2); + assert_eq!(trusted_validators.len(), 0); assert!(!config.validator); assert!(config.prover); assert!(!config.client); assert_eq!(genesis, expected_genesis); + // Client dev node with `--private-key` flag. let mut trusted_peers = vec![]; let mut trusted_validators = vec![]; let mut config = - Start::try_parse_from(["snarkos", "--dev", "3", "--client", "--private-key", ""].iter()).unwrap(); - config.parse_development(&mut trusted_peers, &mut trusted_validators); + Start::try_parse_from(["snarkos", "--dev", "10", "--client", "--private-key", ""].iter()).unwrap(); + config.parse_development(&mut trusted_peers, &mut trusted_validators).unwrap(); let genesis = config.parse_genesis::().unwrap(); - assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4133").unwrap())); - assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3033").unwrap())); - assert_eq!(trusted_peers.len(), 3); - assert_eq!(trusted_validators.len(), 2); + assert_eq!(config.node, Some(SocketAddr::from_str("0.0.0.0:4140").unwrap())); + assert_eq!(config.rest, Some(SocketAddr::from_str("0.0.0.0:3040").unwrap())); + assert_eq!(trusted_peers.len(), 2); + assert_eq!(trusted_validators.len(), 0); assert!(!config.validator); assert!(!config.prover); assert!(config.client); @@ -1188,6 +1256,24 @@ mod tests { assert_eq!(start.validators, Some("IP1,IP2,IP3".to_string())); } + /// Ensure two clients do not connect to the same validators. + #[test] + fn test_parse_development_client_validators() { + let mut client1_config = + Start::try_parse_from(["snarkos", "--dev", "10", "--client", "--private-key", ""].iter()).unwrap(); + let mut trusted_peers1 = vec![]; + let mut trusted_validators1 = vec![]; + client1_config.parse_development(&mut trusted_peers1, &mut trusted_validators1).unwrap(); + + let mut client2_config = + Start::try_parse_from(["snarkos", "--dev", "11", "--client", "--private-key", ""].iter()).unwrap(); + let mut trusted_peers2 = vec![]; + let mut trusted_validators2 = vec![]; + client2_config.parse_development(&mut trusted_peers2, &mut trusted_validators2).unwrap(); + + assert_ne!(trusted_peers1, trusted_peers2); + } + #[test] fn parse_peers_when_ips() { let arg_vec = vec!["snarkos", "start", "--peers", "127.0.0.1:3030,127.0.0.2:3030"]; diff --git a/cli/src/helpers/dev.rs b/cli/src/helpers/dev.rs index 38169912e5..cde7be1aca 100644 --- a/cli/src/helpers/dev.rs +++ b/cli/src/helpers/dev.rs @@ -13,11 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use snarkos_node::{bft::MEMORY_POOL_PORT, router::DEFAULT_NODE_PORT}; + use snarkvm::{console::network::Network, prelude::PrivateKey}; use anyhow::Result; use rand::SeedableRng; use rand_chacha::ChaChaRng; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; /// The development mode RNG seed. pub const DEVELOPMENT_MODE_RNG_SEED: u64 = 1234567890u64; @@ -25,6 +28,9 @@ pub const DEVELOPMENT_MODE_RNG_SEED: u64 = 1234567890u64; /// The development mode number of genesis committee members. pub const DEVELOPMENT_MODE_NUM_GENESIS_COMMITTEE_MEMBERS: u16 = 4; +/// The number of validators a devnet client connects to by default. +pub const DEVNET_NUM_VALIDATORS_PER_CLIENT: u16 = 2; + /// Get the private key for a validator in development mode. pub fn get_development_key(index: u16) -> Result> { // Sample the private key of this node. @@ -37,3 +43,18 @@ pub fn get_development_key(index: u16) -> Result> { PrivateKey::::new(&mut rng) } + +/// Returns the indicies of validators a particular devnet client will connect to. +pub fn get_devnet_validators_for_client(dev: u16, num_validators: u16) -> Vec { + (0..DEVNET_NUM_VALIDATORS_PER_CLIENT).map(|i| (dev + i) % num_validators).collect() +} + +/// Returns the gateway address a particular devnet validator will listen on. +pub fn get_devnet_gateway_address_for_validator(dev: u16) -> SocketAddr { + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, MEMORY_POOL_PORT + dev)) +} + +/// Returns the router address a particular devnet validator will list on. +pub fn get_devnet_router_address_for_node(dev: u16) -> SocketAddr { + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, DEFAULT_NODE_PORT + dev)) +} From c1280fcd36d3f0270af0d59f77fdea809a21555d Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Mon, 15 Dec 2025 18:39:54 -0800 Subject: [PATCH 2/9] ci: correctly test for failed nodes --- .ci/bench_bft_sync.sh | 1 - .ci/bench_cdn_sync.sh | 1 - .ci/bench_p2p_sync.sh | 1 - .ci/bench_rest_api.sh | 4 ---- .ci/db_backup_ci.sh | 1 - .ci/devnet_ci.sh | 1 - .ci/utils.sh | 21 ++++++++++----------- 7 files changed, 10 insertions(+), 20 deletions(-) diff --git a/.ci/bench_bft_sync.sh b/.ci/bench_bft_sync.sh index 9cce21b8ef..3b33cb470b 100755 --- a/.ci/bench_bft_sync.sh +++ b/.ci/bench_bft_sync.sh @@ -42,7 +42,6 @@ function exit_handler() { stop_nodes } trap exit_handler EXIT -trap child_exit_handler CHLD # Define a trap handler that prints a message when an error occurs trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR diff --git a/.ci/bench_cdn_sync.sh b/.ci/bench_cdn_sync.sh index 2b7c1475c4..1c4674fe41 100755 --- a/.ci/bench_cdn_sync.sh +++ b/.ci/bench_cdn_sync.sh @@ -29,7 +29,6 @@ function exit_handler() { stop_nodes } trap exit_handler EXIT -trap child_exit_handler CHLD # Define a trap handler that prints a message when an error occurs. trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR diff --git a/.ci/bench_p2p_sync.sh b/.ci/bench_p2p_sync.sh index dc15b5e52a..1101da3d25 100755 --- a/.ci/bench_p2p_sync.sh +++ b/.ci/bench_p2p_sync.sh @@ -87,7 +87,6 @@ mkdir -p "$log_dir" # Define a trap handler that cleans up all processes on exit. trap stop_nodes EXIT -trap child_exit_handler CHLD # Define a trap handler that prints a message when an error occurs. trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR diff --git a/.ci/bench_rest_api.sh b/.ci/bench_rest_api.sh index 811b1fadad..05a714c881 100755 --- a/.ci/bench_rest_api.sh +++ b/.ci/bench_rest_api.sh @@ -11,9 +11,6 @@ network_id=1 # Adjust this to show more/less log messages log_filter="info,snarkos_node_sync=debug,snarkos_node_tcp=warn,snarkos_node_rest=warn" -max_wait=2400 # Wait for up to 40 minutes -poll_interval=1 # Check block heights every second - . ./.ci/utils.sh branch_name=$(git rev-parse --abbrev-ref HEAD) @@ -31,7 +28,6 @@ mkdir -p "$log_dir" # Define a trap handler that cleans up all processes on exit. trap stop_nodes EXIT -trap child_exit_handler CHLD # Define a trap handler that prints a message when an error occurs. trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR diff --git a/.ci/db_backup_ci.sh b/.ci/db_backup_ci.sh index d07b02839c..39483c64d8 100755 --- a/.ci/db_backup_ci.sh +++ b/.ci/db_backup_ci.sh @@ -23,7 +23,6 @@ jwt[3]="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhbGVvMTJ1eDNnZGF1Y2swdjY # Define a trap handler that cleans up all processes on exit. trap stop_nodes EXIT -trap child_exit_handler CHLD # Define a trap handler that prints a message when an error occurs trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR diff --git a/.ci/devnet_ci.sh b/.ci/devnet_ci.sh index 900554b5dd..34740ae951 100755 --- a/.ci/devnet_ci.sh +++ b/.ci/devnet_ci.sh @@ -50,7 +50,6 @@ function exit_handler() { rmdir program || true } trap exit_handler EXIT -trap child_exit_handler CHLD # Define a trap handler that prints a message when an error occurs trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR diff --git a/.ci/utils.sh b/.ci/utils.sh index 26c015c85a..87ba0788f4 100644 --- a/.ci/utils.sh +++ b/.ci/utils.sh @@ -7,9 +7,6 @@ # Array to store PIDs of all processes declare -a PIDS -# Flag is set true once a node process stopped -node_stopped=false - # How many cores should each node use? # (Should be half of the number of (v)CPUs) # NOTE: when you update this, update TASKSET1/2 as well. @@ -25,15 +22,17 @@ else TASKSET2="taskset -c 8-15" fi -# Handler for a child process exiting -function child_exit_handler() { - # only set to true if this was indeed a node +# Check if any tracked node process has exited. +# Returns 0 if a node stopped, 1 otherwise. +function check_node_stopped() { for i in "${!PIDS[@]}"; do - if [[ "${PIDS[i]}" -eq "$pid" ]]; then - echo "Node #${i} (pid=$pid) exited" - node_stopped=true + local pid="${PIDS[i]}" + if ! kill -0 "$pid" 2>/dev/null; then + echo "Node #${i} (pid=$pid) has exited unexpectedly" + return 0 fi done + return 1 } # Function checking that each node in the given range [start_index, end_index) @@ -242,8 +241,8 @@ function wait_for_nodes() { local total_clients=$2 while true; do - if [ "$node_stopped" = true ]; then - echo "Something went wrong: one more nodes stopped unexpectedly" + if check_node_stopped; then + echo "ERROR: one or more nodes stopped unexpectedly" return 1 fi From 70f7302ec6d4acae90af17f27451f47896829f49 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Tue, 16 Dec 2025 10:31:55 -0800 Subject: [PATCH 3/9] chore: resolve remaining shellcheck warnings in .ci --- .ci/bench_bft_sync.sh | 4 +++- .ci/bench_cdn_sync.sh | 4 +++- .ci/bench_p2p_sync.sh | 1 + .ci/bench_rest_api.sh | 7 ++++--- .ci/db_backup_ci.sh | 10 ++++++---- .ci/generate_ledger.sh | 2 ++ .ci/utils.sh | 5 +++++ 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.ci/bench_bft_sync.sh b/.ci/bench_bft_sync.sh index 3b33cb470b..513e52ed7f 100755 --- a/.ci/bench_bft_sync.sh +++ b/.ci/bench_bft_sync.sh @@ -22,7 +22,8 @@ log_filter="info,snarkos_node_sync=trace,snarkos_node_bft::sync=trace,snarkos_no max_wait=1800 # Wait for up to 30 minutes poll_interval=1 # Check block heights every second -. ./.ci/utils.sh +#shellcheck source=SCRIPTDIR/utils.sh +. .ci/utils.sh branch_name=$(git rev-parse --abbrev-ref HEAD) echo "On branch: ${branch_name}" @@ -38,6 +39,7 @@ log_dir=".logs-$(date +"%Y%m%d%H%M%S")" mkdir -p "$log_dir" # Define a trap handler that cleans up all processes on exit. +# shellcheck disable=SC2329 function exit_handler() { stop_nodes } diff --git a/.ci/bench_cdn_sync.sh b/.ci/bench_cdn_sync.sh index 1c4674fe41..adaae50a01 100755 --- a/.ci/bench_cdn_sync.sh +++ b/.ci/bench_cdn_sync.sh @@ -15,6 +15,7 @@ log_filter="info,snarkos_node_rest=warn,snarkos_node_cdn=debug" max_wait=1800 # Wait for up to 30 minutes poll_interval=1 # Check block heights every second +# shellcheck source=SCRIPTDIR/utils.sh . ./.ci/utils.sh network_name=$(get_network_name $network_id) @@ -25,6 +26,7 @@ log_dir=".logs-$(date +"%Y%m%d%H%M%S")" mkdir -p "$log_dir" # Define a trap handler that cleans up all processes on exit. +# shellcheck disable=SC2329 function exit_handler() { stop_nodes } @@ -41,7 +43,7 @@ args=( "--network=$network_id" --nobanner --noupdater --nodisplay # reduce clutter in the output and hide TUI --rest-rps=1000000 # ensure benchmarks don't fail due to rate limiting - --log-filter=$log_filter # only show the logs we care about + "--log-filter=$log_filter" # only show the logs we care about ) # Spawn the client that will sync the ledger. diff --git a/.ci/bench_p2p_sync.sh b/.ci/bench_p2p_sync.sh index 1101da3d25..26ea1e714f 100755 --- a/.ci/bench_p2p_sync.sh +++ b/.ci/bench_p2p_sync.sh @@ -22,6 +22,7 @@ log_filter="info,snarkos_node::client=trace,snarkos_node_sync=trace,snarkos_node max_wait=2400 # Wait for up to 40 minutes poll_interval=1 # Check block heights every second +# shellcheck source=SCRIPTDIR/utils.sh . ./.ci/utils.sh # Running sums for variance: use sum and sumsq for unbiased sample variance diff --git a/.ci/bench_rest_api.sh b/.ci/bench_rest_api.sh index 05a714c881..62b2bb9356 100755 --- a/.ci/bench_rest_api.sh +++ b/.ci/bench_rest_api.sh @@ -11,6 +11,7 @@ network_id=1 # Adjust this to show more/less log messages log_filter="info,snarkos_node_sync=debug,snarkos_node_tcp=warn,snarkos_node_rest=warn" +#shellcheck source=SCRIPTDIR/utils.sh . ./.ci/utils.sh branch_name=$(git rev-parse --abbrev-ref HEAD) @@ -51,8 +52,8 @@ PIDS[0]=$! # Block until node is running. wait_for_nodes 0 1 -python ./.ci/rest_api_helper.py "get-block" $CORES_PER_NODE 60 -python ./.ci/rest_api_helper.py "block-height" $CORES_PER_NODE 10000 -python ./.ci/rest_api_helper.py "get-latest-block" $CORES_PER_NODE 100 +python ./.ci/rest_api_helper.py "get-block" "$CORES_PER_NODE" 60 +python ./.ci/rest_api_helper.py "block-height" "$CORES_PER_NODE" 10000 +python ./.ci/rest_api_helper.py "get-latest-block" "$CORES_PER_NODE" 100 exit 0 diff --git a/.ci/db_backup_ci.sh b/.ci/db_backup_ci.sh index 39483c64d8..fd02379616 100755 --- a/.ci/db_backup_ci.sh +++ b/.ci/db_backup_ci.sh @@ -1,5 +1,6 @@ #!/bin/bash +#shellcheck source=SCRIPTDIR/utils.sh . ./.ci/utils.sh # Network parameters @@ -55,7 +56,7 @@ function create_checkpoints() { return 0 } -wait_for_nodes "$total_validators" "$total_clients" +wait_for_nodes "$total_validators" 0 # Check heights periodically with a timeout total_wait=0 @@ -88,16 +89,17 @@ while (( total_wait < 600 )); do # 10 minutes max for ((validator_index = 0; validator_index < total_validators; validator_index++)); do # Remove the original ledger if (( num_checkpoints == 1 )); then - snarkos clean --network $network_id --dev $validator_index + snarkos clean "--network=$network_id" "--dev=$validator_index" else suffix="${validator_index}_$((num_checkpoints-2))" - snarkos clean --network $network_id --dev $validator_index --path=/tmp/checkpoint_$suffix + snarkos clean "--network=$network_id" "--dev=validator_index" "--path=/tmp/checkpoint_$suffix" fi # Wait until the cleanup concludes sleep 1 # Restart using the checkpoint suffix="${validator_index}_$((num_checkpoints-1))" - snarkos start --nodisplay --network $network_id --dev $validator_index --dev-num-validators $total_validators --validator --jwt-secret $jwt_secret --jwt-timestamp $jwt_ts --storage /tmp/checkpoint_$suffix & + snarkos start --nodisplay "--network=$network_id" "--dev=$validator_index" "--dev-num-validators=$total_validators" \ + --validator "--jwt-secret=$jwt_secret" "--jwt-timestamp=$jwt_ts" "--storage=/tmp/checkpoint_$suffix" & PIDS[validator_index]=$! echo "Restarted validator $validator_index with PID ${PIDS[$validator_index]}" # Add 1-second delay between starting nodes to avoid hitting rate limits diff --git a/.ci/generate_ledger.sh b/.ci/generate_ledger.sh index 71417f0c70..7c2fa4b1c1 100755 --- a/.ci/generate_ledger.sh +++ b/.ci/generate_ledger.sh @@ -26,6 +26,7 @@ poll_interval=10 : "${min_height:=250}" : "${network_id:=1}" +#shellcheck source=SCRIPTDIR/utils.sh . ./.ci/utils.sh git_commit=$(git rev-parse --short=10 HEAD) @@ -40,6 +41,7 @@ mkdir -p "$log_dir" chmod 755 "$log_dir" # Define a trap handler that cleans up all processes on exit. +#shellcheck disable=SC2329 function exit_handler() { stop_nodes } diff --git a/.ci/utils.sh b/.ci/utils.sh index 87ba0788f4..5cdf2ace56 100644 --- a/.ci/utils.sh +++ b/.ci/utils.sh @@ -10,15 +10,20 @@ declare -a PIDS # How many cores should each node use? # (Should be half of the number of (v)CPUs) # NOTE: when you update this, update TASKSET1/2 as well. +# shellcheck disable=SC2034 CORES_PER_NODE=8 # Tasksets to pin processes to specific CPUs. # This is a no-op on MacOS. if [[ "$(uname)" == "Darwin" ]]; then + # shellcheck disable=SC2034 TASKSET1="" + # shellcheck disable=SC2034 TASKSET2="" else + # shellcheck disable=SC2034 TASKSET1="taskset -c 0-7" + # shellcheck disable=SC2034 TASKSET2="taskset -c 8-15" fi From 3638c7963e862fcd2da03f426105a39aa9f746e8 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Tue, 16 Dec 2025 10:36:28 -0800 Subject: [PATCH 4/9] ci: run shellcheck for all .ci scripts --- .circleci/config.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac5589a612..b28a8f4362 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -614,6 +614,27 @@ jobs: - clear_environment: cache_key: v4.2.0-rust-1.88.0-clippy-cache + lint-scripts: + executor: rust-docker + resource_class: << pipeline.parameters.medium >> + steps: + - checkout + - setup_environment: + cache_key: v4.2.0-rust-1.88.0-lint-scripts-cache + - run: + name: Install shellcheck + command: | + # Fetch from git so we reliably install the same version. + wget https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz + tar -xvf shellcheck-v0.11.0.linux.x86_64.tar.xz + sudo mv shellcheck-v0.11.0/shellcheck /usr/local/bin/ + - run: + name: Lint scripts + command: | + shellcheck -x .ci/*.sh + - clear_environment: + cache_key: v4.2.0-rust-1.88.0-lint-scripts-cache + verify-windows: executor: name: windows/default @@ -637,6 +658,7 @@ workflows: - check-unused-dependencies - check-cargo-audit - check-other-crates + - lint-scripts node-workflow: jobs: From 4f9aece15b1f66914f10d59405193d6128191b2f Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Tue, 16 Dec 2025 15:38:07 -0800 Subject: [PATCH 5/9] ci: pick correct version for cargo-nextest --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b28a8f4362..e1f9abf351 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,7 +155,8 @@ commands: - run: name: Install cargo-nextest command: | - cargo install cargo-nextest --locked || true + # Anything newer requires a Rust toolchain upgrade. + cargo install cargo-nextest@0.9.114 --locked || true - run: name: "Run Tests" From 240f9f9a9f001918c191007e3cc2f1a248ffe9f7 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Tue, 16 Dec 2025 16:10:49 -0800 Subject: [PATCH 6/9] fix(cli): bind development node to 0.0.0.0, not localhost --- cli/src/commands/start.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index 4ffa7ebb28..e47313090b 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -465,7 +465,9 @@ impl Start { // // Note: the `node` flag is an option to detect remote devnet testing. if self.node.is_none() { - let address = get_devnet_router_address_for_node(dev); + // Pick 0.0.0.0 here, not localhost. + let port = get_devnet_router_address_for_node(dev).port(); + let address = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, port)); debug!("Setting node address to {address} due to dev={dev}"); self.node = Some(address); } From e03193024e473d44a73e3d88c68df11a2218a030 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Tue, 16 Dec 2025 16:36:54 -0800 Subject: [PATCH 7/9] ci: remove duplicate check for consensus version --- .ci/devnet_ci.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.ci/devnet_ci.sh b/.ci/devnet_ci.sh index 34740ae951..a62f8d7424 100755 --- a/.ci/devnet_ci.sh +++ b/.ci/devnet_ci.sh @@ -165,11 +165,6 @@ if ! $version_stable; then fi # Creates a test program. - -if ! $version_stable; then - echo "❌ Test failed! Consensus version did not stabilize within 5 minutes." - exit 1 -fi mkdir -p program program_name="test_program.aleo" echo "program ${program_name}; From 05fe710a21518e98729bab71632f4c86c2d9570c Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Wed, 17 Dec 2025 10:18:43 -0800 Subject: [PATCH 8/9] fix(cli): make --peers and --dev-num-validators conflict --- cli/src/commands/start.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index e47313090b..49f755394a 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -261,7 +261,9 @@ pub struct Start { /// If development mode is enabled, specify the number of clients. /// This is only used by validators to automatically populate their set of trusted peers. - #[clap(long, group = "dev_flags")] + /// + /// This option cannot be used while also passing the `--peers` flag. + #[clap(long, group = "dev_flags", conflicts_with = "peers")] pub dev_num_clients: Option, /// If development mode is enabled, specify whether node 0 should generate traffic to drive the network. @@ -1221,6 +1223,15 @@ mod tests { assert_eq!(genesis, expected_genesis); } + /// Tests that you cannot pass the `--dev-num-clients` flag while also passing the `--peers` flag. + #[test] + fn test_parse_development_num_clients_and_peers() { + let result = Start::try_parse_from( + ["snarkos", "--validator", "--dev", "1", "--peers", "127.0.0.1:3030", "--dev-num-clients", "1"].iter(), + ); + assert!(result.is_err()); + } + #[test] fn clap_snarkos_start() { let arg_vec = vec![ From d830b359ccbc0743d4d04a4da2c28ca5146d5ef3 Mon Sep 17 00:00:00 2001 From: Kai Mast Date: Wed, 17 Dec 2025 10:31:59 -0800 Subject: [PATCH 9/9] fix(scripts): correct devnet script --- cli/src/commands/start.rs | 4 ++++ scripts/devnet.sh | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index 49f755394a..ca44cb4c8b 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -435,6 +435,8 @@ impl Start { } trusted_validators.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, MEMORY_POOL_PORT + idx))); } + + debug!("Trusted validators set to: {trusted_validators:?}"); } // Determine if we need to populate `trusted_peers`. @@ -459,6 +461,8 @@ impl Start { trusted_peers.push(get_devnet_router_address_for_node(validator_idx)); } } + + debug!("Trusted peers set to: {trusted_peers:?}"); } else { debug!("Trusted peers/validators was set manually. Will not populate them with development addresses.") } diff --git a/scripts/devnet.sh b/scripts/devnet.sh index aee5b8da11..bb20ff1eb2 100755 --- a/scripts/devnet.sh +++ b/scripts/devnet.sh @@ -74,7 +74,7 @@ if [[ $clear_ledger == "y" ]]; then for ((index = 0; index < $((total_validators + total_clients)); index++)); do # Run 'snarkos clean' for each node in the background - ${binary_path}snarkos clean --network $network_id --dev $index & + "${binary_path}snarkos" clean "--network=$network_id" "--dev=$index" "--dev-num-validators=$total_validators" & # Store the process ID of the background task clean_processes+=($!) @@ -121,7 +121,7 @@ for validator_index in "${validator_indices[@]}"; do fi # Send the command to start the validator to the new window and capture output to the log file - tmux send-keys -t "devnet:$window_index" "${binary_path}snarkos start --nodisplay --network $network_id --dev $validator_index --dev-num-validators $total_validators --validator --logfile $log_file --verbosity $verbosity --metrics --metrics-ip=0.0.0.0:$metrics_port --no-dev-txs" C-m + tmux send-keys -t "devnet:$window_index" "${binary_path}snarkos start --dev-num-clients $total_clients --nodisplay --network $network_id --dev $validator_index --dev-num-validators $total_validators --validator --logfile $log_file --verbosity $verbosity --metrics --metrics-ip=0.0.0.0:$metrics_port --no-dev-txs" C-m done if [ "$total_clients" -ne 0 ]; then