[Fix] Clean up devnet peering and other CI script improvements

.ci/bench_bft_sync.sh

@@ -22,7 +22,8 @@ log_filter="info,snarkos_node_sync=trace,snarkos_node_bft::sync=trace,snarkos_no
 max_wait=1800 # Wait for up to 30 minutes
 poll_interval=1 # Check block heights every second
 
-. ./.ci/utils.sh
+#shellcheck source=SCRIPTDIR/utils.sh
+. .ci/utils.sh
 
 branch_name=$(git rev-parse --abbrev-ref HEAD)
 echo "On branch: ${branch_name}"
@@ -38,11 +39,11 @@ log_dir=".logs-$(date +"%Y%m%d%H%M%S")"
 mkdir -p "$log_dir"
 
 # Define a trap handler that cleans up all processes on exit.
+# shellcheck disable=SC2329
 function exit_handler() {
   stop_nodes
 }
 trap exit_handler EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs 
 trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR

.ci/bench_cdn_sync.sh

@@ -15,6 +15,7 @@ log_filter="info,snarkos_node_rest=warn,snarkos_node_cdn=debug"
 max_wait=1800 # Wait for up to 30 minutes
 poll_interval=1 # Check block heights every second
 
+# shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
 network_name=$(get_network_name $network_id)
@@ -25,11 +26,11 @@ log_dir=".logs-$(date +"%Y%m%d%H%M%S")"
 mkdir -p "$log_dir"
 
 # Define a trap handler that cleans up all processes on exit.
+# shellcheck disable=SC2329
 function exit_handler() {
   stop_nodes
 }
 trap exit_handler EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs.
 trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
@@ -42,7 +43,7 @@ args=(
   "--network=$network_id"
   --nobanner --noupdater --nodisplay # reduce clutter in the output and hide TUI
   --rest-rps=1000000 # ensure benchmarks don't fail due to rate limiting
-  --log-filter=$log_filter # only show the logs we care about
+  "--log-filter=$log_filter" # only show the logs we care about
 )
 
 # Spawn the client that will sync the ledger.

.ci/bench_p2p_sync.sh

@@ -22,6 +22,7 @@ log_filter="info,snarkos_node::client=trace,snarkos_node_sync=trace,snarkos_node
 max_wait=2400 # Wait for up to 40 minutes
 poll_interval=1 # Check block heights every second
 
+# shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
 # Running sums for variance: use sum and sumsq for unbiased sample variance
@@ -87,7 +88,6 @@ mkdir -p "$log_dir"
 
 # Define a trap handler that cleans up all processes on exit.
 trap stop_nodes EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs.
 trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR

.ci/bench_rest_api.sh

@@ -11,9 +11,7 @@ network_id=1
 # Adjust this to show more/less log messages
 log_filter="info,snarkos_node_sync=debug,snarkos_node_tcp=warn,snarkos_node_rest=warn"
 
-max_wait=2400 # Wait for up to 40 minutes
-poll_interval=1 # Check block heights every second
-
+#shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
 branch_name=$(git rev-parse --abbrev-ref HEAD)
@@ -31,7 +29,6 @@ mkdir -p "$log_dir"
 
 # Define a trap handler that cleans up all processes on exit.
 trap stop_nodes EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs.
 trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
@@ -55,8 +52,8 @@ PIDS[0]=$!
 # Block until node is running.
 wait_for_nodes 0 1
 
-python ./.ci/rest_api_helper.py "get-block" $CORES_PER_NODE 60
-python ./.ci/rest_api_helper.py "block-height" $CORES_PER_NODE 10000
-python ./.ci/rest_api_helper.py "get-latest-block" $CORES_PER_NODE 100
+python ./.ci/rest_api_helper.py "get-block" "$CORES_PER_NODE" 60
+python ./.ci/rest_api_helper.py "block-height" "$CORES_PER_NODE" 10000
+python ./.ci/rest_api_helper.py "get-latest-block" "$CORES_PER_NODE" 100
 
 exit 0

.ci/db_backup_ci.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+#shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
 # Network parameters
@@ -23,7 +24,6 @@ jwt[3]="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhbGVvMTJ1eDNnZGF1Y2swdjY
 
 # Define a trap handler that cleans up all processes on exit.
 trap stop_nodes EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs 
 trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
@@ -56,7 +56,7 @@ function create_checkpoints() {
   return 0
 }
 
-wait_for_nodes "$total_validators" "$total_clients"
+wait_for_nodes "$total_validators" 0 
 
 # Check heights periodically with a timeout
 total_wait=0
@@ -89,16 +89,17 @@ while (( total_wait < 600 )); do  # 10 minutes max
       for ((validator_index = 0; validator_index < total_validators; validator_index++)); do
         # Remove the original ledger
         if (( num_checkpoints == 1 )); then
-          snarkos clean --network $network_id --dev $validator_index
+          snarkos clean "--network=$network_id" "--dev=$validator_index"
         else
           suffix="${validator_index}_$((num_checkpoints-2))"
-          snarkos clean --network $network_id --dev $validator_index --path=/tmp/checkpoint_$suffix
+          snarkos clean "--network=$network_id" "--dev=validator_index" "--path=/tmp/checkpoint_$suffix"
         fi
         # Wait until the cleanup concludes
         sleep 1
         # Restart using the checkpoint
         suffix="${validator_index}_$((num_checkpoints-1))"
-        snarkos start --nodisplay --network $network_id --dev $validator_index --dev-num-validators $total_validators --validator --jwt-secret $jwt_secret --jwt-timestamp $jwt_ts --storage /tmp/checkpoint_$suffix &
+        snarkos start --nodisplay "--network=$network_id" "--dev=$validator_index" "--dev-num-validators=$total_validators" \
+          --validator "--jwt-secret=$jwt_secret" "--jwt-timestamp=$jwt_ts" "--storage=/tmp/checkpoint_$suffix" &
         PIDS[validator_index]=$!
         echo "Restarted validator $validator_index with PID ${PIDS[$validator_index]}"
         # Add 1-second delay between starting nodes to avoid hitting rate limits

.ci/devnet_ci.sh

@@ -16,14 +16,15 @@ network_id=$3
 min_height=$4
 
 # The verobsity of snarkos nodes.
-NODE_VERBOSITY=1
+NODE_VERBOSITY=3
 
 # Default values if not provided
 : "${total_validators:=4}"
-: "${total_clients:=2}"
+: "${total_clients:=4}" # need at least 4 clients, so each validator has at least one client connected to it.
 : "${network_id:=0}"
 : "${min_height:=60}" # To likely go past the 100 round garbage collection limit.
 
+# shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
 # Determine network name based on network_id
@@ -49,39 +50,27 @@ function exit_handler() {
   rmdir program || true
 }
 trap exit_handler EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs 
 trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
 
 # Flags used by all nodes.
 common_flags=(
   --nodisplay --nobanner --noupdater "--network=$network_id" "--verbosity=$NODE_VERBOSITY"
-  "--dev-num-validators=$total_validators"
+  "--dev-num-validators=$total_validators"  "--dev-num-clients=$total_clients"
 )
 
-# Set the trusted peers for the validators as they will not allow connections from unknown peers.
-trusted_peers=""
-for ((client_index = 0; client_index < total_clients; client_index++)); do
-  node_index=$((client_index + total_validators))
-  if (( client_index == 0 )); then
-    trusted_peers+="$localhost:$((4130+node_index))"
-  else
-    trusted_peers+=",$localhost:$((4130+node_index))"
-  fi
-done
-
 # Start all validator nodes in the background
 for ((validator_index = 0; validator_index < total_validators; validator_index++)); do
   snarkos clean "--dev=$validator_index" "--network=$network_id"
 
   log_file="$log_dir/validator-$validator_index.log"
   if [ $validator_index -eq 0 ]; then
     snarkos start "${common_flags[@]}" "--dev=$validator_index" \
-      --validator "--logfile=$log_file" --metrics --no-dev-txs "--peers=$trusted_peers" &
+      --validator "--logfile=$log_file" --metrics --no-dev-txs &
   else
     snarkos start "${common_flags[@]}" "--dev=$validator_index" \
-      --validator "--logfile=$log_file" "--peers=$trusted_peers" &
+      --validator "--logfile=$log_file" &
   fi
   PIDS[validator_index]=$!
   echo "Started validator $validator_index with PID ${PIDS[$validator_index]}"
@@ -111,14 +100,31 @@ done
 # Ensure all nodes are up and running.
 wait_for_nodes "$total_validators" "$total_clients"
 
+# Ensure all nodes have at least one peer
+echo "ℹ️ Waiting for all nodes to have at least one peer..."
+SECONDS=0
+for node_index in $(seq 0 $((total_clients+total_validators))); do
+  if ! (wait_for_peers "$node_index" 1); then
+    exit 1
+  fi
+done
+echo "✅ All nodes have at least one peer"
+
 last_seen_consensus_version=0
 last_seen_height=0
 
 # Function checking that the first node reached the latest (unchanging) consensus version.
 function consensus_version_stable() {
   consensus_version=$(curl -s "http://$localhost:3030/v2/$network_name/consensus_version")
   height=$(curl -s "http://$localhost:3030/v2/$network_name/block/height/latest")
-  if (is_integer "$consensus_version") && (is_integer "$height"); then
+
+  if (! is_integer "$consensus_version"); then
+    echo "❌ Failed to retrieve consensus version: $consensus_version"
+    return 1
+  elif (! is_integer "$height"); then
+    echo "❌ Failed to retrieve height: $height"
+    return 1
+  else
     # If the consensus version is greater than the last seen, we update it.
     if (( consensus_version > last_seen_consensus_version )); then
       echo "✅ Consensus version updated to $consensus_version"
@@ -129,19 +135,21 @@ function consensus_version_stable() {
         return 0
       fi
     fi
-  else
-    echo "❌ Failed to retrieve consensus version or height: $consensus_version, $height"
-    exit 1
+
+    last_seen_consensus_version=$consensus_version
+    last_seen_height=$height
   fi
-  last_seen_consensus_version=$consensus_version
-  last_seen_height=$height
+
   return 1
 }
 
 # Check consensus versions periodically with a timeout
+echo "ℹ️ Waiting for consensus version to stabilize..."
 total_wait=0
+version_stable=false
 while (( total_wait < 300 )); do  # 5 minutes max
   if consensus_version_stable; then
+    version_stable=true
     break
   fi
 
@@ -151,6 +159,11 @@ while (( total_wait < 300 )); do  # 5 minutes max
   echo "Waited $total_wait seconds so far..."
 done
 
+if ! $version_stable; then
+  echo "❌ Test failed! Consensus version did not stabilize within 5 minutes."
+  exit 1
+fi
+
 # Creates a test program.
 mkdir -p program
 program_name="test_program.aleo"

.ci/generate_ledger.sh

@@ -26,6 +26,7 @@ poll_interval=10
 : "${min_height:=250}"
 : "${network_id:=1}"
 
+#shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
 git_commit=$(git rev-parse --short=10 HEAD)
@@ -40,6 +41,7 @@ mkdir -p "$log_dir"
 chmod 755 "$log_dir"
 
 # Define a trap handler that cleans up all processes on exit.
+#shellcheck disable=SC2329
 function exit_handler() {
   stop_nodes
 }

.ci/utils.sh

@@ -7,33 +7,37 @@
 # Array to store PIDs of all processes
 declare -a PIDS
 
-# Flag is set true once a node process stopped
-node_stopped=false
-
 # How many cores should each node use?
 # (Should be half of the number of (v)CPUs)
 # NOTE: when you update this, update TASKSET1/2 as well.
+# shellcheck disable=SC2034
 CORES_PER_NODE=8
 
 # Tasksets to pin processes to specific CPUs.
 # This is a no-op on MacOS.
 if [[ "$(uname)" == "Darwin" ]]; then
+  # shellcheck disable=SC2034
   TASKSET1=""
+  # shellcheck disable=SC2034
   TASKSET2=""
 else
+  # shellcheck disable=SC2034
   TASKSET1="taskset -c 0-7"
+  # shellcheck disable=SC2034
   TASKSET2="taskset -c 8-15"
 fi
 
-# Handler for a child process exiting
-function child_exit_handler() {
-  # only set to true if this was indeed a node
+# Check if any tracked node process has exited.
+# Returns 0 if a node stopped, 1 otherwise.
+function check_node_stopped() {
   for i in "${!PIDS[@]}"; do
-    if [[ "${PIDS[i]}" -eq "$pid" ]]; then
-      echo "Node #${i} (pid=$pid) exited"
-      node_stopped=true
+    local pid="${PIDS[i]}"
+    if ! kill -0 "$pid" 2>/dev/null; then
+      echo "Node #${i} (pid=$pid) has exited unexpectedly"
+      return 0
     fi
   done
+  return 1
 }
 
 # Function checking that each node in the given range [start_index, end_index)
@@ -242,15 +246,18 @@ function wait_for_nodes() {
   local total_clients=$2
 
   while true; do
-    if [ "$node_stopped" = true ]; then
-      echo "Something went wrong: one more nodes stopped unexpectedly"
+    if check_node_stopped; then
+      echo "ERROR: one or more nodes stopped unexpectedly"
       return 1
     fi
 
     if check_nodes "$total_validators" "$total_clients"; then
       echo "All nodes are ready!"
       return 0
     fi
+
+    # Pause to give the nodes time to start up.
+    sleep 1
   done
 }