feat: improve copy algorithm (#464)

Improves the `copy` algorithm to read multiple blobs at once (allowing
to read small unused data as well), like in `prune` or `forget`. This
should make `copy` much faster for remote sources.

As a side effect, the `PackSizer` has been refactored and now also
allows fixed_size packs. This allowed to simplify some methods for
copying a bit.

Also progress now works much better and integration tests for `copy`
have been added.

Author: aawsome

crates/core/src/archiver/file_archiver.rs

@@ -13,7 +13,7 @@ use crate::{
     },
     blob::{
         BlobId, BlobType, DataId,
-        packer::{Packer, PackerStats},
+        packer::{PackSizer, Packer, PackerStats},
     },
     chunker::ChunkIter,
     crypto::hasher::hash,
@@ -62,13 +62,9 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> {
         indexer: SharedIndexer<BE>,
         config: &ConfigFile,
     ) -> RusticResult<Self> {
-        let data_packer = Packer::new(
-            be,
-            BlobType::Data,
-            indexer,
-            config,
-            index.total_size(BlobType::Data),
-        )?;
+        let pack_sizer =
+            PackSizer::from_config(config, BlobType::Data, index.total_size(BlobType::Data));
+        let data_packer = Packer::new(be, BlobType::Data, indexer, pack_sizer)?;
 
         Ok(Self {
             index,

crates/core/src/archiver/tree_archiver.rs

@@ -8,7 +8,7 @@ use crate::{
     backend::{decrypt::DecryptWriteBackend, node::Node},
     blob::{
         BlobType,
-        packer::Packer,
+        packer::{PackSizer, Packer},
         tree::{Tree, TreeId},
     },
     error::{ErrorKind, RusticError, RusticResult},
@@ -66,13 +66,9 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> TreeArchiver<'a, BE, I> {
         config: &ConfigFile,
         summary: SnapshotSummary,
     ) -> RusticResult<Self> {
-        let tree_packer = Packer::new(
-            be,
-            BlobType::Tree,
-            indexer,
-            config,
-            index.total_size(BlobType::Tree),
-        )?;
+        let pack_sizer =
+            PackSizer::from_config(config, BlobType::Tree, index.total_size(BlobType::Tree));
+        let tree_packer = Packer::new(be, BlobType::Tree, indexer, pack_sizer)?;
 
         Ok(Self {
             tree: Tree::new(),

crates/core/src/blob.rs

@@ -166,14 +166,34 @@ impl BlobLocation {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq, Eq)]
 pub struct BlobLocations<T> {
     pub offset: u32,
     pub length: u32,
     pub blobs: Vec<(BlobLocation, T)>,
 }
 
+impl<T: Eq + PartialEq> PartialOrd<Self> for BlobLocations<T> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<T: Eq> Ord for BlobLocations<T> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.offset.cmp(&other.offset)
+    }
+}
+
 impl<T> BlobLocations<T> {
+    pub fn length(&self) -> u32 {
+        self.blobs.iter().map(|bl| bl.0.length).sum()
+    }
+
+    pub fn data_length(&self) -> u32 {
+        self.blobs.iter().map(|bl| bl.0.data_length()).sum()
+    }
+
     pub fn from_blob_location(location: BlobLocation, target: T) -> Self {
         Self {
             offset: location.offset,