[perf]: backport changes from thunder perf hackathon (#68)

* [perf]: backport changes from thunder perf hackathon

* [misc] Fix std dep order, add comments

* [cli] remove mimalloc from cli

* [fix] smallvec usage

* [misc] Add comments for LMDB flags

* [fix] Switch to Borsh and add small test for OutpointKey

* [fix] make in/out tracking explicit as per suggestion

* [fix] Adjust import ordering

* [fix] clarify counts() use, rm into_parts() and explicit deconstruct

* [fix] explicit From

Author: giaki3003

Cargo.lock

@@ -20,6 +20,36 @@ use util::saturating_pred_level;
 #[global_allocator]
 static GLOBAL: MiMalloc = MiMalloc;
 
+fn configure_mimalloc() {
+    // Tune mimalloc via environment variables
+    // SAFETY: called before any threads are started, so no other thread can
+    // concurrently read or write the process environment.
+    unsafe {
+        // Limit how many abandoned pages stay cached before returning them to the OS.
+        std::env::set_var("MIMALLOC_ABANDONED_PAGE_LIMIT", "4");
+        // Reset abandoned pages so their physical memory is released quickly.
+        std::env::set_var("MIMALLOC_ABANDONED_PAGE_RESET", "1");
+        // Cap the number of huge arenas we keep around to bound reserved memory.
+        std::env::set_var("MIMALLOC_ARENA_LIMIT", "4");
+        // Allow mimalloc to allocate from any NUMA node for better balance.
+        std::env::set_var("MIMALLOC_USE_NUMA_NODES", "all");
+        // Commit new pages immediately to avoid first-use page fault latency.
+        std::env::set_var("MIMALLOC_EAGER_COMMIT", "1");
+        // Commit entire regions up front rather than piecemeal to reduce faults.
+        std::env::set_var("MIMALLOC_EAGER_REGION_COMMIT", "1");
+        // Keep up to 32 segments cached for quick reuse before releasing to the OS.
+        std::env::set_var("MIMALLOC_SEGMENT_CACHE", "32");
+        // Prefer allocating from large OS pages (2MB) when the system supports it.
+        std::env::set_var("MIMALLOC_LARGE_OS_PAGES", "1");
+        // Reserve a handful of huge OS pages at startup to back large heaps.
+        std::env::set_var("MIMALLOC_RESERVE_HUGE_OS_PAGES", "4");
+        // Keep freed pages committed instead of resetting to avoid extra madvise calls.
+        std::env::set_var("MIMALLOC_PAGE_RESET", "0");
+        // Likewise, keep whole segments committed to reduce release/reacquire churn.
+        std::env::set_var("MIMALLOC_SEGMENT_RESET", "0");
+    }
+}
+
 /// The empty string target `""` can be used to set a default level.
 fn targets_directive_str<'a, Targets>(targets: Targets) -> String
 where
@@ -167,6 +197,8 @@ fn run_egui_app(
 }
 
 fn main() -> anyhow::Result<()> {
+    // Configure the allocator before Tokio spins up worker threads.
+    configure_mimalloc();
     let cli = cli::Cli::parse();
     let config = cli.get_config()?;
     let (line_buffer, _rolling_log_guard) = set_tracing_subscriber(

app/main.rs

@@ -41,6 +41,7 @@ rayon = "1.7.0"
 rcgen = "0.13.2"
 rustls = { version = "0.23.21", default-features = false, features = ["ring"] }
 rustreexo = { workspace = true, features = ["with-serde"] }
+smallvec = { version = "1.13.2", features = ["write"] }
 semver = { version = "1.0.25", features = ["serde"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }

lib/Cargo.toml

@@ -137,6 +137,7 @@ where
         // let _ = std::fs::remove_dir_all(&env_path);
         std::fs::create_dir_all(&env_path)?;
         let env = {
+            use heed::EnvFlags;
             let mut env_open_opts = heed::EnvOpenOptions::new();
             env_open_opts
                 .map_size(128 * 1024 * 1024 * 1024) // 128 GB
@@ -146,6 +147,28 @@ where
                         + MemPool::NUM_DBS
                         + Net::NUM_DBS,
                 );
+            // Apply LMDB "fast" flags consistent with our benchmark setup:
+            // - WRITE_MAP lets us write directly into the memory map instead of
+            //   copying into LMDB's page buffer, reducing syscall overhead for
+            //   write-heavy workloads.
+            // - MAP_ASYNC hands dirty-page flushing to the kernel so commits do
+            //   not block waiting for msync, keeping latencies tight.
+            // - NO_SYNC and NO_META_SYNC skip fsync calls for data and
+            //   metadata; this trades durability for throughput, which is
+            //   acceptable here because the state can be reconstructed from the
+            //   canonical chain if a crash occurs.
+            // - NO_READ_AHEAD disables kernel readahead that would otherwise
+            //   touch cold pages we immediately overwrite, improving random
+            //   access behaviour on SSDs used in testing.
+            // - NO_TLS stops LMDB from relying on thread-local storage for
+            //   reader slots so transactions can be moved across Tokio tasks.
+            let fast_flags = EnvFlags::WRITE_MAP
+                | EnvFlags::MAP_ASYNC
+                | EnvFlags::NO_SYNC
+                | EnvFlags::NO_META_SYNC
+                | EnvFlags::NO_READ_AHEAD
+                | EnvFlags::NO_TLS;
+            unsafe { env_open_opts.flags(fast_flags) };
             unsafe { Env::open(&env_open_opts, &env_path) }
                 .map_err(EnvError::from)?
         };

lib/node/mod.rs

@@ -93,15 +93,23 @@ fn connect_tip_(
     two_way_peg_data: &mainchain::TwoWayPegData,
 ) -> Result<(), Error> {
     let block_hash = header.hash();
-    let (_fees, merkle_root): (bitcoin::Amount, MerkleRoot) =
-        state.validate_block(rwtxn, header, body)?;
+    let prevalidated = state.prevalidate_block(rwtxn, header, body)?;
     if tracing::enabled!(tracing::Level::DEBUG) {
         let height = state.try_get_height(rwtxn)?;
-        let _: MerkleRoot = state.connect_block(rwtxn, header, body)?;
-        tracing::debug!(?height, %merkle_root, %block_hash,
-                            "connected body")
+        let merkle_root = state.connect_prevalidated_block(
+            rwtxn,
+            header,
+            body,
+            prevalidated,
+        )?;
+        tracing::debug!(?height, %merkle_root, %block_hash, "connected body")
     } else {
-        let _: MerkleRoot = state.connect_block(rwtxn, header, body)?;
+        let _: MerkleRoot = state.connect_prevalidated_block(
+            rwtxn,
+            header,
+            body,
+            prevalidated,
+        )?;
     }
     let () = state.connect_two_way_peg_data(rwtxn, two_way_peg_data)?;
     let accumulator = state.get_accumulator(rwtxn)?;