Refactor encryption

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.wasm32-unknown-unknown]
+rustflags = ['--cfg', 'getrandom_backend="wasm_js"']

.clippy.toml

@@ -0,0 +1,2 @@
+# We want to know ASAP if we have a needless_pass_by_mut or needless_pass_by_value,  etc added to the public API
+avoid-breaking-exported-api = false

.github/workflows/ci.yml

@@ -31,11 +31,9 @@ jobs:
     - name: Run tests
       run: |
         cargo check --all-targets
-        cargo check --all-targets --no-default-features --features tokio
-        cargo check --all-targets --no-default-features --features async-std
-        cargo test --features js_interop_tests
-        cargo test --no-default-features --features js_interop_tests,tokio
-        cargo test --no-default-features --features js_interop_tests,async-std
+        cargo check --all-targets --no-default-features
+        cargo test --features js_tests
+        cargo test --no-default-features --features js_tests
         cargo test --benches
 
   build-extra:
@@ -48,15 +46,13 @@ jobs:
           targets: wasm32-unknown-unknown
       - name: Build WASM
         run: |
-          cargo build --target=wasm32-unknown-unknown --no-default-features --features wasm-bindgen,tokio
-          cargo build --target=wasm32-unknown-unknown --no-default-features --features wasm-bindgen,async-std
+          cargo build --target=wasm32-unknown-unknown --no-default-features --features wasm-bindgen
       - name: Build release
         run: |
-          cargo build --release --no-default-features --features tokio
-          cargo build --release --no-default-features --features async-std
+          cargo build --release --no-default-features
       - name: Build examples
         run: |
-          cargo build --example replication 
+          cargo build --example replication
 
   lint:
     runs-on: ubuntu-latest

.rustfmt.toml

@@ -0,0 +1,5 @@
+# groups 'use' statements by crate
+imports_granularity = "crate"
+# formats code within doc tests
+# requires: cargo +nightly fmt (otherwise rustfmt will warn, but pass)
+format_code_in_doc_comments = true

CHANGELOG.md

@@ -4,7 +4,10 @@ All notable changes to this Rust implementation of hypercore-protocol will be do
 
 ### unreleased
 
-* TODO: Add changes here as they happen
+BIG CHANGES:
+* Encryption and framing of streams has been moved out of this crate into `hypercore_handshake` and `uint24le_framing` respectively. This had big impacts on the public API. Now `Protocol::new` just takes a `impl CipherTrait` argument.
+* Remove dependence on `hypercore` instead we use `hypercore_schema`.
+* Bumped to edition 2024.
 
 ### 0.6.1
 

Cargo.toml

@@ -10,7 +10,7 @@ authors = [
 documentation = "https://docs.rs/hypercore-protocol"
 repository = "https://github.com/datrs/hypercore-protocol-rs"
 readme = "README.md"
-edition = "2021"
+edition = "2024"
 keywords = ["dat", "p2p", "replication", "hypercore", "protocol"]
 categories = [
   "asynchronous",
@@ -26,54 +26,60 @@ bench = false
 
 [dependencies]
 async-channel = "1"
-snow = { version = "0.9", features = ["risky-raw-split"] }
-bytes = "1"
+snow = { version = "0.10", features = ["risky-raw-split"] }
 rand = "0.8"
 blake2 = "0.10"
 hex = "0.4"
-async-trait = "0.1"
 tracing = "0.1"
 pretty-hash = "0.4"
 futures-timer = "3"
 futures-lite = "1"
 sha2 = "0.10"
 curve25519-dalek = "4"
 crypto_secretstream = "0.2"
+futures = "0.3.31"
+compact-encoding = "2"
+thiserror = "2.0.12"
+hypercore_handshake = "0.6.0"
 
-[dependencies.hypercore]
-version = "0.14.0"
-default-features = false
+[dev-dependencies.hypercore]
+features = ["shared-core"]
+version = "0.16.0"
 
+[dependencies.hypercore_schema]
+version = "0.2.0"
 
 [dev-dependencies]
 async-std = { version = "1.12.0", features = ["attributes", "unstable"] }
 async-compat = "0.2.1"
 tokio = { version = "1.27.0", features = ["macros", "net", "process", "rt", "rt-multi-thread", "sync", "time"] }
-env_logger = "0.7.1"
 anyhow = "1.0.28"
 instant = "0.1"
 criterion = { version = "0.4", features = ["async_std"] }
 pretty-bytes = "0.2.2"
 duplexify = "1.1.0"
 sluice = "0.5.4"
 futures = "0.3.13"
-log = "0.4"
-test-log = { version = "0.2.11", default-features = false, features = ["trace"] }
-tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] }
+tracing-subscriber = { version = "0.3.19", features = ["env-filter", "fmt"] }
+tracing-tree = "0.4.0"
+tokio-util = { version = "0.7.14", features = ["compat"] }
+uint24le_framing = { version = "0.2.0" }
+
+[dev-dependencies.rusty_nodejs_repl]
+version = "0.4.0"
+features = ["serde", "integration_utils"]
 
 [features]
-default = ["tokio", "sparse"]
 wasm-bindgen = [
   "futures-timer/wasm-bindgen"
 ]
-sparse = ["hypercore/sparse"]
-cache = ["hypercore/cache"]
-tokio = ["hypercore/tokio"]
-async-std = ["hypercore/async-std"]
 # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore
 # to verify that this crate works. To run them, use:
-# cargo test --features js_interop_tests
-js_interop_tests = []
+# cargo test --features js_tests
+js_tests = []
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+getrandom = { version = "0.3", features = ["wasm_js"] }
 
 [profile.bench]
 # debug = true

README.md

@@ -72,10 +72,10 @@ node examples-nodejs/run.js node
 
 ## Development
 
-To test interoperability with Javascript, enable the `js_interop_tests` feature:
+To test interoperability with Javascript, enable the `js_tests` feature:
 
 ```bash
-cargo test --features js_interop_tests
+cargo test --features js_tests
 ```
 
 Run benches with:

benches/pipe.rs

@@ -1,24 +1,27 @@
+#[path = "../src/test_utils.rs"]
+mod test_utils;
 use async_std::task;
-use criterion::{criterion_group, criterion_main, Criterion, Throughput};
-use futures::io::{AsyncRead, AsyncWrite};
+#[path = "../tests/_util.rs"]
+mod _util;
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
 use futures::stream::StreamExt;
-use hypercore_protocol::{schema::*, Duplex};
-use hypercore_protocol::{Channel, Event, Message, Protocol, ProtocolBuilder};
-use log::*;
+use hypercore_protocol::{Channel, Event, Message, Protocol, schema::*};
+use hypercore_schema::DataBlock;
 use pretty_bytes::converter::convert as pretty_bytes;
-use sluice::pipe::pipe;
-use std::io::Result;
-use std::time::Instant;
+use std::{io::Result, time::Instant};
+use tracing::{debug, error};
+
+use crate::_util::create_pair;
 
 const COUNT: u64 = 1000;
 const SIZE: u64 = 100;
 const CONNS: u64 = 10;
 
 fn bench_throughput(c: &mut Criterion) {
-    env_logger::from_env(env_logger::Env::default().default_filter_or("error")).init();
+    test_utils::log();
     let mut group = c.benchmark_group("pipe");
     group.sample_size(10);
-    group.throughput(Throughput::Bytes(SIZE * COUNT * CONNS as u64));
+    group.throughput(Throughput::Bytes(SIZE * COUNT * CONNS));
     group.bench_function("pipe_echo", |b| {
         b.iter(|| {
             task::block_on(async move {
@@ -38,17 +41,7 @@ criterion_group!(benches, bench_throughput);
 criterion_main!(benches);
 
 async fn run_echo(i: u64) -> Result<()> {
-    // let cap: usize = SIZE as usize * 10;
-    let (ar, bw) = pipe();
-    let (br, aw) = pipe();
-
-    let encrypted = true;
-    let a = ProtocolBuilder::new(true)
-        .encrypted(encrypted)
-        .connect_rw(ar, aw);
-    let b = ProtocolBuilder::new(false)
-        .encrypted(encrypted)
-        .connect_rw(br, bw);
+    let (a, b) = create_pair();
     let ta = task::spawn(async move { onconnection(i, a).await });
     let tb = task::spawn(async move { onconnection(i, b).await });
     ta.await?;
@@ -58,11 +51,7 @@ async fn run_echo(i: u64) -> Result<()> {
 
 // The onconnection handler is called for each incoming connection (if server)
 // or once when connected (if client).
-async fn onconnection<R, W>(i: u64, mut protocol: Protocol<Duplex<R, W>>) -> Result<u64>
-where
-    R: AsyncRead + Send + Unpin + 'static,
-    W: AsyncWrite + Send + Unpin + 'static,
-{
+async fn onconnection(i: u64, mut protocol: Protocol) -> Result<u64> {
     let key = [0u8; 32];
     let is_initiator = protocol.is_initiator();
     // let mut len: u64 = 0;
@@ -72,7 +61,7 @@ where
                 debug!("[{}] EVENT {:?}", is_initiator, event);
                 match event {
                     Event::Handshake(_) => {
-                        protocol.open(key.clone()).await?;
+                        protocol.open(key).await?;
                     }
                     Event::DiscoveryKey(_dkey) => {}
                     Event::Channel(channel) => {
@@ -92,7 +81,7 @@ where
             }
             Some(Err(err)) => {
                 error!("ERROR {:?}", err);
-                return Err(err.into());
+                return Err(err);
             }
             None => return Ok(0),
         }
@@ -127,20 +116,17 @@ async fn on_channel_init(i: u64, mut channel: Channel) -> Result<u64> {
     let start = std::time::Instant::now();
 
     while let Some(message) = channel.next().await {
-        match message {
-            Message::Data(mut data) => {
-                len += value_len(&data);
-                debug!("[a] recv {}", index(&data));
-                if index(&data) >= COUNT {
-                    debug!("close at {}", index(&data));
-                    channel.close().await?;
-                    break;
-                } else {
-                    increment_index(&mut data);
-                    channel.send(Message::Data(data)).await?;
-                }
+        if let Message::Data(mut data) = message {
+            len += value_len(&data);
+            debug!("[a] recv {}", index(&data));
+            if index(&data) >= COUNT {
+                debug!("close at {}", index(&data));
+                channel.close().await?;
+                break;
+            } else {
+                increment_index(&mut data);
+                channel.send(Message::Data(data)).await?;
             }
-            _ => {}
         }
     }
     // let bytes = (COUNT * SIZE) as f64;
@@ -149,8 +135,6 @@ async fn on_channel_init(i: u64, mut channel: Channel) -> Result<u64> {
 }
 
 fn msg_data(index: u64, value: Vec<u8>) -> Message {
-    use hypercore::DataBlock;
-
     Message::Data(Data {
         request: index,
         fork: 0,