Chunk size in config file, back to 1mb default

Author: FayKn

cryptify-back-end/src/config.rs

@@ -10,6 +10,7 @@ pub struct RawCryptifyConfig {
     smtp_credentials: Option<(String, String)>,
     allowed_origins: String,
     pkg_url: String,
+    chunk_size: Option<u64>,
     s3_endpoint: Option<String>,
     s3_access_key: Option<String>,
     s3_secret_key: Option<String>,
@@ -28,6 +29,7 @@ pub struct CryptifyConfig {
     smtp_credentials: Option<(String, String)>,
     allowed_origins: String,
     pkg_url: String,
+    chunk_size: u64,
 
     s3_endpoint: Option<String>,
     s3_access_key: Option<String>,
@@ -37,7 +39,10 @@ pub struct CryptifyConfig {
 }
 
 impl From<RawCryptifyConfig> for CryptifyConfig {
-    fn from(config: RawCryptifyConfig) -> Self {
+    fn from(mut config: RawCryptifyConfig) -> Self {
+        // deoption chunk_size to default value if not set
+        let chunk_size = config.chunk_size.take().unwrap_or(1024 * 1024); // 1 MB default for backward compatibility with older configs and front-ends
+        
         CryptifyConfig {
             server_url: config.server_url,
             data_dir: config.data_dir,
@@ -50,6 +55,7 @@ impl From<RawCryptifyConfig> for CryptifyConfig {
             smtp_credentials: config.smtp_credentials,
             allowed_origins: config.allowed_origins,
             pkg_url: config.pkg_url,
+            chunk_size,
             s3_endpoint: config.s3_endpoint,
             s3_access_key: config.s3_access_key,
             s3_secret_key: config.s3_secret_key,
@@ -91,6 +97,10 @@ impl CryptifyConfig {
     pub fn pkg_url(&self) -> &str {
         &self.pkg_url
     }
+    
+    pub fn chunk_size(&self) -> u64 {
+        self.chunk_size
+    }
 
     pub fn s3_endpoint(&self) -> Option<&str> {
         self.s3_endpoint.as_deref()

cryptify-back-end/src/main.rs

@@ -38,8 +38,6 @@ use rocket_cors::{AllowedOrigins, CorsOptions};
 use serde::{Deserialize, Serialize};
 use store::{FileState, Store};
 
-const CHUNK_SIZE: u64 = (1024 * 1024)*5; // 5 MB
-
 #[derive(Serialize, Deserialize)]
 struct InitBody {
     recipient: String,
@@ -301,7 +299,7 @@ async fn upload_chunk(
         )));
     }
 
-    if end - start > CHUNK_SIZE {
+    if end - start > config.chunk_size() {
         return Err(Error::BadRequest(Some(
             "File chunk too large; the maximum is 1 MB".to_owned(),
         )));
@@ -329,7 +327,7 @@ async fn upload_chunk(
 
         let byte_stream = data.into_inner();
 
-        let part_number = (start / CHUNK_SIZE + 1) as i32;
+        let part_number = (start / config.chunk_size() + 1) as i32;
         let upload_part_res = client
             .upload_part()
             .bucket(bucket_name)
@@ -469,7 +467,7 @@ async fn upload_finalize(
             .into_async_read()
             .compat();
     } else {
-        // read with the s3 client so it's the same type as above but still works for local files
+        // read with the s3 client, so it's the same type as above but still works for local files
         file = aws_sdk_s3::primitives::ByteStream::from_path(Path::new(config.data_dir()).join(uuid))
             .await
             .map_err(|_| Error::InternalServerError(Some("could not open file".to_string())))?
@@ -500,7 +498,7 @@ async fn upload_finalize(
 }
 
 #[get("/<uuid>")]
-async fn s3fileServer(
+async fn s3_file_server(
     _config: &State<CryptifyConfig>,
     _store: &State<Store>,
     s3_client: &State<Option<store::S3Client>>,
@@ -575,8 +573,8 @@ async fn rocket() -> _ {
 
     // conditionally mount the file download routes
     if s3_client.is_some() {
-        rocket = rocket.mount("/filedownload", routes![s3fileServer]);
-        if CHUNK_SIZE < 5 * 1024 * 1024 {
+        rocket = rocket.mount("/filedownload", routes![s3_file_server]);
+        if config.chunk_size() < 5 * 1024 * 1024 { // 5 MB is the usual minimum for S3 multipart uploads
             log::warn!("S3 storage is enabled, but CHUNK_SIZE is less than 5 MB. This may lead to failing uploads.");
         }
     } else {

cryptify-front-end/src/Constants.tsx

@@ -3,6 +3,7 @@ import {browserName, browserVersion, isMobile} from "react-device-detect";
 type ConfigFile = {
     PKG_URL?: string;
     BACKEND_URL?: string;
+    UPLOAD_CHUNK_SIZE?: number;
 };
 const rawConfig: unknown = (window as any).__APP_CONFIG__;
 const configFile: ConfigFile = rawConfig && typeof rawConfig === "object" ? (rawConfig as ConfigFile) : {};
@@ -12,7 +13,7 @@ export const MAX_UPLOAD_SIZE: number = 2 * 1000 * 1000 * 1000;
 
 // 1Mb chunks
 export const FILEREAD_CHUNK_SIZE: number = 1024 * 1024;
-export const UPLOAD_CHUNK_SIZE: number = (1024 * 1024)*5;
+export const UPLOAD_CHUNK_SIZE: number = configFile.UPLOAD_CHUNK_SIZE ?? 1024 * 1024;
 
 // progress bar smooth time in seconds.
 export const SMOOTH_TIME: number = 2;