feat: implement emulator tests. (stjude-rust-labs#4)

* feat: implement emulator tests.

This commit implements tests for the Azure Storage and AWS S3 backends to use
local storage emulators for tests.

Also fixes a panic due to a missing `break` in `cloud-copy`'s main function
that was caused by introducing a loop to drive the main future to completion
upon cancellation.

Note that Google Cloud Storage tests were not added as there currently isn't an
officially supported emulator.

* fix: remove `macos` from test matrix until it supports Docker.

* chore: fix CI.

* chore: code review feedback.

Include how long the retry waited in the warning.

* chore: code review feedback.

* Removed the test environment variables in favor of proper configuration.
* The configuration now enables additional support for Azurite/Localstack and
  does not attempt to rewrite every URL (`az` and `s3` schemed URLs are
  rewritten when enabled, though).
* Updated the URLs in the tests to use Azurite/Localstack where appropriate.

Author: peterhuene

.github/workflows/CI.yml

@@ -31,9 +31,29 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
+      - name: Install Azure CLI
+        run: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+      - name: Start LocalStack (AWS S3)
+        run: |
+          container=$(docker create -p 4566:4566 localstack/localstack:s3-latest)
+          docker start $container
+          sleep 5
+          docker logs $container
+      - name: Start Azurite (Azure Storage)
+        run: |
+          container=$(docker create -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite -l /data --blobHost 0.0.0.0 --loose)
+          docker start $container
+          sleep 5
+          docker logs $container
+      - name: Create test AWS S3 bucket
+        run: |
+          AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=${DEFAULT_REGION:-$AWS_DEFAULT_REGION} aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket cloud-copy-test
+      - name: Create test Azure Storage container
+        run: |
+          az storage container create --name cloud-copy-test --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1"
       - name: Update Rust
         run: rustup update stable && rustup default stable
       - run: cargo test --all-features

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 #### Added
 
+* Added testing with emulators Azurite and localstack ([#4](https://github.com/stjude-rust-labs/cloud-copy/pull/4)).
 * Added initial implementation of `cloud-copy` ([#1](https://github.com/stjude-rust-labs/cloud-copy/pull/1)).
 
 #### Changed
@@ -17,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 #### Fixed
 
+* Fixed a panic when running `cloud-copy` ([#4](https://github.com/stjude-rust-labs/cloud-copy/pull/4)).
 * Fixed remote content modification check with ranged downloads to use the
   `if-match` header ([#3](https://github.com/stjude-rust-labs/cloud-copy/pull/3)).
 * Fixed graceful cancellation (i.e. SIGINT) to cancel operations without

Cargo.toml

@@ -41,6 +41,9 @@ url = { version = "2.5.4", features = ["serde"] }
 urlencoding = "2.1.3"
 walkdir = "2.5.0"
 
+[dev-dependencies]
+anyhow = { version = "1.0.98" }
+
 [lints.rust]
 missing_docs = "warn"
 nonstandard-style = "warn"

README.md

@@ -127,7 +127,47 @@ path to copy the file or directory to.
 
 ## 🧠 Running Automated Tests
 
-This section coming soon!
+Automated tests rely on having the following cloud service emulators installed:
+
+* [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage)
+  for Azure Blob Storage.
+* [Localstack](https://github.com/localstack/localstack) for AWS S3.
+
+Use the following command to run Azurite in a Docker container:
+
+```bash
+docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite -l /data --blobHost 0.0.0.0 --loose
+```
+
+Use the following command to run Localstack in a Docker container:
+
+```bash
+docker run -p 4566:4566 localstack/localstack:s3-latest
+```
+
+The tests expect a container/bucket with the name `cloud-copy-test` to be
+present.
+
+To create the container with Azurite, use the Azure CLI:
+
+```bash
+az storage container create --name cloud-copy-test --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1"
+```
+
+To create the bucket with Localstack, use the `awslocal` tool:
+
+```bash
+awslocal s3api create-bucket --bucket cloud-copy-test
+```
+
+Finally, run the tests:
+
+```bash
+cargo test --all
+```
+
+Note: the Azure tests expect `*.blob.core.windows.net.localhost` to resolve to
+`127.0.0.1`.
 
 ## ✅ Submitting Pull Requests
 

src/backend.rs

@@ -50,6 +50,16 @@ pub trait StorageBackend {
     /// Gets the block size given the size of a file.
     fn block_size(&self, file_size: u64) -> Result<u64>;
 
+    /// Whether or not the URL is supported by this backend.
+    fn is_supported_url(config: &Config, url: &Url) -> bool;
+
+    /// Rewrites the given URL.
+    ///
+    /// If the URL is using a cloud-specific scheme, the URL is rewritten to a `https` schemed URL.
+    ///
+    /// Otherwise, the given URL is returned as-is.
+    fn rewrite_url(&self, url: Url) -> Result<Url>;
+
     /// Joins segments to a URL to form a new URL.
     fn join_url<'a>(&self, url: Url, segments: impl Iterator<Item = &'a str>) -> Result<Url>;
 

src/backend/azure.rs

@@ -36,6 +36,9 @@ use crate::streams::TransferStream;
 /// The Azure Blob Storage domain suffix.
 const AZURE_BLOB_STORAGE_ROOT_DOMAIN: &str = "blob.core.windows.net";
 
+/// The Azurite root domain suffix.
+const AZURITE_ROOT_DOMAIN: &str = "blob.core.windows.net.localhost";
+
 /// The default block size in bytes (4 MiB).
 const DEFAULT_BLOCK_SIZE: u64 = 4 * ONE_MEBIBYTE;
 
@@ -94,66 +97,6 @@ pub enum AzureError {
     BlobNameMissing,
 }
 
-/// Determines if the given URL is an Azure Blob Storage URL.
-pub fn is_azure_url(url: &Url) -> bool {
-    match url.scheme() {
-        "az" => true,
-        "https" => {
-            let Some(domain) = url.domain() else {
-                return false;
-            };
-
-            // Virtual host style URL of the form https://<account>.blob.core.windows.net/<container>/<path>
-            let Some((_, domain)) = domain.split_once('.') else {
-                return false;
-            };
-            domain.eq_ignore_ascii_case(AZURE_BLOB_STORAGE_ROOT_DOMAIN)
-        }
-        _ => false,
-    }
-}
-
-/// Rewrites an Azure Blob Storage URL (az://) into a HTTPS URL.
-///
-/// If the URL is not `az` schemed, the given URL is returned as-is.
-pub fn rewrite_url(url: Url) -> Result<Url> {
-    match url.scheme() {
-        "az" => {
-            let account = url.host_str().ok_or(AzureError::InvalidScheme)?;
-
-            if url.path() == "/" {
-                return Err(AzureError::InvalidScheme.into());
-            }
-
-            match (url.query(), url.fragment()) {
-                (None, None) => format!(
-                    "https://{account}.{AZURE_BLOB_STORAGE_ROOT_DOMAIN}{path}",
-                    path = url.path()
-                ),
-                (None, Some(fragment)) => {
-                    format!(
-                        "https://{account}.{AZURE_BLOB_STORAGE_ROOT_DOMAIN}{path}#{fragment}",
-                        path = url.path()
-                    )
-                }
-                (Some(query), None) => format!(
-                    "https://{account}.{AZURE_BLOB_STORAGE_ROOT_DOMAIN}{path}?{query}",
-                    path = url.path()
-                ),
-                (Some(query), Some(fragment)) => {
-                    format!(
-                        "https://{account}.{AZURE_BLOB_STORAGE_ROOT_DOMAIN}{path}?{query}#{fragment}",
-                        path = url.path()
-                    )
-                }
-            }
-            .parse()
-            .map_err(|_| AzureError::InvalidScheme.into())
-        }
-        _ => Ok(url),
-    }
-}
-
 /// Represents information about a blob.
 #[derive(Debug, Deserialize)]
 struct Blob {
@@ -421,6 +364,69 @@ impl StorageBackend for AzureBlobStorageBackend {
         Ok(block_size)
     }
 
+    fn is_supported_url(config: &Config, url: &Url) -> bool {
+        match url.scheme() {
+            "az" => true,
+            "http" | "https" => {
+                let Some(domain) = url.domain() else {
+                    return false;
+                };
+
+                // Virtual host style URL of the form https://<account>.blob.core.windows.net/<container>/<path>
+                let Some((_, domain)) = domain.split_once('.') else {
+                    return false;
+                };
+
+                domain.eq_ignore_ascii_case(AZURE_BLOB_STORAGE_ROOT_DOMAIN)
+                    | (config.azure.use_azurite && domain.eq_ignore_ascii_case(AZURITE_ROOT_DOMAIN))
+            }
+            _ => false,
+        }
+    }
+
+    fn rewrite_url(&self, url: Url) -> Result<Url> {
+        match url.scheme() {
+            "az" => {
+                let account = url.host_str().ok_or(AzureError::InvalidScheme)?;
+
+                if url.path() == "/" {
+                    return Err(AzureError::InvalidScheme.into());
+                }
+
+                let (scheme, root, port) = if self.config.azure.use_azurite {
+                    ("http", AZURITE_ROOT_DOMAIN, ":10000")
+                } else {
+                    ("https", AZURE_BLOB_STORAGE_ROOT_DOMAIN, "")
+                };
+
+                match (url.query(), url.fragment()) {
+                    (None, None) => {
+                        format!("{scheme}://{account}.{root}{port}{path}", path = url.path())
+                    }
+                    (None, Some(fragment)) => {
+                        format!(
+                            "{scheme}://{account}.{root}{port}{path}#{fragment}",
+                            path = url.path()
+                        )
+                    }
+                    (Some(query), None) => format!(
+                        "{scheme}://{account}.{root}{port}{path}?{query}",
+                        path = url.path()
+                    ),
+                    (Some(query), Some(fragment)) => {
+                        format!(
+                            "{scheme}://{account}.{root}{port}{path}?{query}#{fragment}",
+                            path = url.path()
+                        )
+                    }
+                }
+                .parse()
+                .map_err(|_| AzureError::InvalidScheme.into())
+            }
+            _ => Ok(url),
+        }
+    }
+
     fn join_url<'a>(&self, mut url: Url, segments: impl Iterator<Item = &'a str>) -> Result<Url> {
         let mut segments = segments.peekable();
 
@@ -446,8 +452,9 @@ impl StorageBackend for AzureBlobStorageBackend {
 
     async fn head(&self, url: Url) -> Result<Response> {
         debug_assert!(
-            is_azure_url(&url) && url.scheme() == "https",
-            "expected Azure HTTPS URL"
+            Self::is_supported_url(&self.config, &url),
+            "{url} is not a supported Azure URL",
+            url = url.as_str()
         );
 
         debug!("sending HEAD request for `{url}`", url = url.display());
@@ -470,8 +477,9 @@ impl StorageBackend for AzureBlobStorageBackend {
 
     async fn get(&self, url: Url) -> Result<Response> {
         debug_assert!(
-            is_azure_url(&url) && url.scheme() == "https",
-            "expected Azure HTTPS URL"
+            Self::is_supported_url(&self.config, &url),
+            "{url} is not a supported Azure URL",
+            url = url.as_str()
         );
 
         debug!("sending GET request for `{url}`", url = url.display());
@@ -494,8 +502,9 @@ impl StorageBackend for AzureBlobStorageBackend {
 
     async fn get_range(&self, url: Url, etag: &str, range: Range<u64>) -> Result<Response> {
         debug_assert!(
-            is_azure_url(&url) && url.scheme() == "https",
-            "expected Azure HTTPS URL"
+            Self::is_supported_url(&self.config, &url),
+            "{url} is not a supported Azure URL",
+            url = url.as_str()
         );
 
         debug!(
@@ -541,8 +550,9 @@ impl StorageBackend for AzureBlobStorageBackend {
 
     async fn walk(&self, url: Url) -> Result<Vec<String>> {
         debug_assert!(
-            is_azure_url(&url) && url.scheme() == "https",
-            "expected Azure HTTPS URL"
+            Self::is_supported_url(&self.config, &url),
+            "{url} is not a supported Azure URL",
+            url = url.as_str()
         );
 
         debug!("walking `{url}` as a directory", url = url.display());
@@ -671,8 +681,9 @@ impl StorageBackend for AzureBlobStorageBackend {
 
     async fn new_upload(&self, url: Url) -> Result<Self::Upload> {
         debug_assert!(
-            is_azure_url(&url) && url.scheme() == "https",
-            "expected Azure HTTPS URL"
+            Self::is_supported_url(&self.config, &url),
+            "{url} is not a supported Azure URL",
+            url = url.as_str()
         );
 
         Ok(AzureBlobUpload::new(

src/backend/generic.rs

@@ -108,6 +108,14 @@ impl StorageBackend for GenericStorageBackend {
         Ok(4 * 1024 * 1024)
     }
 
+    fn is_supported_url(_: &Config, _: &Url) -> bool {
+        true
+    }
+
+    fn rewrite_url(&self, url: Url) -> Result<Url> {
+        Ok(url)
+    }
+
     fn join_url<'a>(&self, mut url: Url, segments: impl Iterator<Item = &'a str>) -> Result<Url> {
         // Append on the segments
         {