feat: fix droplet manifest generation

libraries/droplet/.gitignore

@@ -19,3 +19,7 @@ target/
 # Added by cargo
 
 /target
+
+perf.data
+flamegraph.svg
+*.json

libraries/droplet/Cargo.lock

@@ -219,7 +219,7 @@ dependencies = [
 
 [[package]]
 name = "droplet-rs"
-version = "0.12.3"
+version = "0.13.0"
 dependencies = [
  "anyhow",
  "async-trait",

libraries/droplet/Cargo.toml

@@ -2,7 +2,7 @@
 edition = "2021"
 authors = ["Drop-OSS"]
 name = "droplet-rs"
-version = "0.12.3"
+version = "0.13.0"
 license = "AGPL-3.0-only"
 description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop"
 

libraries/droplet/src/manifest.rs

@@ -55,26 +55,25 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
 
     let required_single_file = backend.require_whole_files();
 
-    let files = backend.list_files().await?;
+    let mut files = backend.list_files().await?;
+    files.sort_by(|a, b| a.size.cmp(&b.size));
     // Filepath to chunk data
     let mut chunks: Vec<Vec<(VersionFile, u64, u64)>> = Vec::new();
     let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new();
 
     log_sfn("organizing files into chunks...".to_string());
 
-    for version_file in files {
+    if required_single_file {
-        // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on
+        for version_file in files {
-        if required_single_file && version_file.size >= CHUNK_SIZE {
+            if version_file.size >= CHUNK_SIZE {
-            let size = version_file.size;
+                let size = version_file.size;
-            chunks.push(vec![(version_file, 0, size)]);
+                chunks.push(vec![(version_file, 0, size)]);
 
-            continue;
+                continue;
-        }
+            }
 
-        let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::<u64>();
+            let mut current_size = current_chunk.iter().map(|v| v.2).sum::<u64>();
 
-        // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be
-        if required_single_file {
             let size = version_file.size;
             current_chunk.push((version_file, 0, size));
 
@@ -88,43 +87,36 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
 
             continue;
         }
+    } else {
+        for version_file in files {
+            let current_size = current_chunk.iter().map(|v| v.2).sum::<u64>();
 
-        // Otherwise we calculate how much of the file we need, then use that much
+            if version_file.size + current_size < CHUNK_SIZE + WIGGLE {
-        let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size;
+                let size = version_file.size;
-        if version_file.size >= remaining_budget {
+                current_chunk.push((version_file, 0, size));
-            let remaining_budget = CHUNK_SIZE - current_size;
-            current_chunk.push((version_file.clone(), 0, remaining_budget));
 
-            let new_chunk = std::mem::take(&mut current_chunk);
+                continue;
-            chunks.push(new_chunk);
-
-            let remaining_size = version_file.size - remaining_budget;
-            let mut running_offset = remaining_budget;
-            // Do everything but the last one
-            while running_offset < remaining_size {
-                let chunk_size = CHUNK_SIZE.min(remaining_size);
-                let chunk = vec![(version_file.clone(), running_offset, chunk_size)];
-                if chunk_size == CHUNK_SIZE {
-                    chunks.push(chunk);
-                } else {
-                    current_chunk = chunk;
-                }
-                running_offset += chunk_size;
             }
 
-            continue;
+            // Fill up current chunk
-        } else {
+            let remaining = CHUNK_SIZE - current_size;
-            let size = version_file.size;
+            current_chunk.push((version_file.clone(), 0, remaining));
-            current_chunk.push((version_file, 0, size));
+            chunks.push(std::mem::take(&mut current_chunk));
-            current_size += size;
-        }
 
-        if current_size >= CHUNK_SIZE {
+            // This is our offset in our current file
-            // Pop current and add, then reset
+            let mut offset = remaining;
-            let new_chunk = std::mem::take(&mut current_chunk);
+            while offset < version_file.size {
-            chunks.push(new_chunk);
+                let length = CHUNK_SIZE.min(version_file.size - offset);
+                offset += length;
+                if length == CHUNK_SIZE {
+                    chunks.push(vec![(version_file.clone(), offset, length)]);
+                } else {
+                    current_chunk.push((version_file.clone(), offset, length));
+                }
+            }
         }
     }
+
     if !current_chunk.is_empty() {
         chunks.push(current_chunk);
     }
@@ -191,8 +183,9 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
 
             send_log
                 .send(format!(
-                    "created chunk of size {} from {} files (index {})",
+                    "created chunk of size {} ({}b) from {} files (index {})",
                     format_size(chunk_length, BINARY),
+                    chunk_length,
                     chunk_data.files.len(),
                     index
                 ))