feat: store content digest metadata for uploaded objects. (stjude-rust-labs#27)

* feat: store content digest metadata for uploaded objects.

This commit adds calculating SHA-256 content digests for uploaded objects,
using a metadata header to store a `Content-Digest` header value.

It also adds `get_content_digest` to retrieve the content digest of a given
URL, which issues a `HEAD` request and returns either a `Content-Digest` header
value (directly or via the metadata header) or a strong `ETag` header.

Adds the `--hash-algorithm` option to `cloud-copy` to specifying either `none`,
`sha256`, or `blake3` for calculating content digests.

* chore: update CHANGELOG.

* chore: code review feedback.

* Update src/lib.rs

Co-authored-by: Clay McLeod <[email protected]>

---------

Co-authored-by: Clay McLeod <[email protected]>

Author: peterhuene

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 #### Added
 
+* Added `get_content_digest` function for retrieving content digests ([#27](https://github.com/stjude-rust-labs/cloud-copy/pull/27)).
+* Added content digest metadata for uploads, defaulting to SHA-256 ([#27](https://github.com/stjude-rust-labs/cloud-copy/pull/27)).
+* Added `--hash-algorithm` to `cloud-copy` CLI for specifying the algorithm to
+  use for attaching content digest metadata to uploaded objects ([#27](https://github.com/stjude-rust-labs/cloud-copy/pull/27)).
 * Added support for Azure Shared Key authentication ([#26](https://github.com/stjude-rust-labs/cloud-copy/pull/26)).
 
 #### Changed

Cargo.lock

@@ -31,6 +31,7 @@ cli = [
 [dependencies]
 anyhow = { version = "1.0.99", optional = true }
 base64 = "0.22.1"
+blake3 = { version = "1.8.2", features = ["mmap", "rayon"] }
 byte-unit = { version = "5.1.6", optional = true }
 bytes = "1.10.1"
 chrono = { version = "0.4.41", features = ["serde"] }

Cargo.toml

@@ -110,5 +110,14 @@ pub trait StorageBackend {
     fn walk(&self, url: Url) -> impl Future<Output = Result<Vec<String>>> + Send;
 
     /// Creates a new upload.
-    fn new_upload(&self, url: Url) -> impl Future<Output = Result<Self::Upload>> + Send;
+    ///
+    /// If `digest` is `Some`, it is expected to be a `Content-Digest` header
+    /// value.
+    ///
+    /// See: <https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Digest>
+    fn new_upload(
+        &self,
+        url: Url,
+        digest: Option<String>,
+    ) -> impl Future<Output = Result<Self::Upload>> + Send;
 }

src/backend.rs

@@ -77,6 +77,14 @@ const AZURE_BLOB_TYPE: &str = "BlockBlob";
 /// The name of the root container.
 const AZURE_ROOT_CONTAINER: &str = "$root";
 
+/// The Azure content digest header name.
+///
+/// This is the content digest for the entire blob.
+///
+/// Note: the metadata name *must* be a legal C# identifier (i.e. cannot contain
+/// `-`).
+pub(crate) const AZURE_CONTENT_DIGEST_HEADER: &str = "x-ms-meta-content_digest";
+
 /// Inserts the authentication header to the request.
 fn insert_authentication_header(auth: &AzureAuthConfig, request: &mut Request) -> Result<()> {
     let signer = RequestSigner::new(auth);
@@ -251,6 +259,8 @@ pub struct AzureBlobUpload {
     url: Url,
     /// The Azure block id.
     block_id: Arc<String>,
+    /// The content digest header value of the blob.
+    digest: Option<String>,
     /// The channel for sending progress updates.
     events: Option<broadcast::Sender<TransferEvent>>,
 }
@@ -262,13 +272,15 @@ impl AzureBlobUpload {
         client: HttpClient,
         url: Url,
         block_id: Arc<String>,
+        digest: Option<String>,
         events: Option<broadcast::Sender<TransferEvent>>,
     ) -> Self {
         Self {
             config,
             client,
             url,
             block_id,
+            digest,
             events,
         }
     }
@@ -367,6 +379,15 @@ impl Upload for AzureBlobUpload {
             .body(body)
             .build()?;
 
+        if let Some(digest) = &self.digest {
+            request.headers_mut().insert(
+                AZURE_CONTENT_DIGEST_HEADER,
+                digest
+                    .try_into()
+                    .expect("invalid content digest header value"),
+            );
+        }
+
         if let Some(auth) = self.config.azure().auth() {
             insert_authentication_header(auth, &mut request)?;
         }
@@ -778,7 +799,7 @@ impl StorageBackend for AzureBlobStorageBackend {
         Ok(paths)
     }
 
-    async fn new_upload(&self, url: Url) -> Result<Self::Upload> {
+    async fn new_upload(&self, url: Url, digest: Option<String>) -> Result<Self::Upload> {
         debug_assert!(
             Self::is_supported_url(&self.config, &url),
             "{url} is not a supported Azure URL",
@@ -800,6 +821,7 @@ impl StorageBackend for AzureBlobStorageBackend {
             self.client.clone(),
             url,
             Arc::new(Alphanumeric::new(16).to_string()),
+            digest,
             self.events.clone(),
         ))
     }

src/backend/azure.rs

@@ -214,7 +214,7 @@ impl StorageBackend for GenericStorageBackend {
         Ok(Vec::default())
     }
 
-    async fn new_upload(&self, _: Url) -> Result<Self::Upload> {
+    async fn new_upload(&self, _: Url, _: Option<String>) -> Result<Self::Upload> {
         panic!("generic storage backend cannot be used for uploading");
     }
 }

src/backend/generic.rs

@@ -60,8 +60,15 @@ const MAX_FILE_SIZE: u64 = MAX_PART_SIZE * 1024;
 const GOOGLE_DATE_HEADER: &str = "x-goog-date";
 
 /// The Google content SHA256 header name.
+///
+/// This is the SHA256 of each upload part for multipart uploads.
 const GOOGLE_CONTENT_SHA256_HEADER: &str = "x-goog-content-sha256";
 
+/// The Google content digest header name.
+///
+/// This is the content digest for the entire object.
+pub(crate) const GOOGLE_CONTENT_DIGEST_HEADER: &str = "x-goog-meta-content-digest";
+
 /// Represents a Google-specific copy operation error.
 #[derive(Debug, thiserror::Error)]
 pub enum GoogleError {
@@ -760,7 +767,7 @@ impl StorageBackend for GoogleStorageBackend {
         Ok(paths)
     }
 
-    async fn new_upload(&self, url: Url) -> Result<Self::Upload> {
+    async fn new_upload(&self, url: Url, digest: Option<String>) -> Result<Self::Upload> {
         // See: https://cloud.google.com/storage/docs/xml-api/post-object-multipart
 
         debug_assert!(
@@ -796,6 +803,15 @@ impl StorageBackend for GoogleStorageBackend {
             .header(GOOGLE_CONTENT_SHA256_HEADER, sha256_hex_string([]))
             .build()?;
 
+        if let Some(digest) = digest {
+            request.headers_mut().insert(
+                GOOGLE_CONTENT_DIGEST_HEADER,
+                digest
+                    .try_into()
+                    .expect("invalid content digest header value"),
+            );
+        }
+
         if let Some(auth) = self.config.google().auth() {
             insert_authentication_header(auth, date, &mut request)?;
         }

src/backend/google.rs

@@ -61,8 +61,15 @@ const MAX_FILE_SIZE: u64 = MAX_PART_SIZE * 1024;
 const AWS_DATE_HEADER: &str = "x-amz-date";
 
 /// The AWS content SHA256 header name.
+///
+/// This is the SHA256 of each upload part for multipart uploads.
 const AWS_CONTENT_SHA256_HEADER: &str = "x-amz-content-sha256";
 
+/// The AWS content digest header name.
+///
+/// This is the content digest for the entire object.
+pub(crate) const AWS_CONTENT_DIGEST_HEADER: &str = "x-amz-meta-content-digest";
+
 /// Represents a S3-specific copy operation error.
 #[derive(Debug, thiserror::Error)]
 pub enum S3Error {
@@ -841,7 +848,7 @@ impl StorageBackend for S3StorageBackend {
         Ok(paths)
     }
 
-    async fn new_upload(&self, url: Url) -> Result<Self::Upload> {
+    async fn new_upload(&self, url: Url, digest: Option<String>) -> Result<Self::Upload> {
         // See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html
 
         debug_assert!(
@@ -874,6 +881,15 @@ impl StorageBackend for S3StorageBackend {
             .header(AWS_CONTENT_SHA256_HEADER, sha256_hex_string([]))
             .build()?;
 
+        if let Some(digest) = digest {
+            request.headers_mut().insert(
+                AWS_CONTENT_DIGEST_HEADER,
+                digest
+                    .try_into()
+                    .expect("invalid content digest header value"),
+            );
+        }
+
         if let Some(auth) = self.config.s3().auth() {
             insert_authentication_header(auth, date, &mut request)?;
         }