feat: manifest generator

libraries/droplet/Cargo.lock

@@ -98,6 +98,21 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
+
 [[package]]
 name = "bytes"
 version = "1.11.0"
@@ -119,6 +134,25 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.8.0"
@@ -162,6 +196,16 @@ dependencies = [
  "powerfmt",
 ]
 
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
 [[package]]
 name = "displaydoc"
 version = "0.2.5"
@@ -181,11 +225,16 @@ dependencies = [
  "async-trait",
  "dyn-clone",
  "hex",
+ "humansize",
  "rcgen",
  "ring",
+ "serde",
+ "serde_json",
+ "sha2",
  "time",
  "time-macros",
  "tokio",
+ "uuid",
  "webpki",
  "x509-parser 0.17.0",
 ]
@@ -196,6 +245,16 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
 
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -207,18 +266,49 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
 [[package]]
 name = "hex"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "humansize"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
+dependencies = [
+ "libm",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
+[[package]]
+name = "js-sys"
+version = "0.3.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"
+dependencies = [
+ "once_cell",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -231,6 +321,12 @@ version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
+[[package]]
+name = "libm"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
+
 [[package]]
 name = "memchr"
 version = "2.7.4"
@@ -316,6 +412,12 @@ dependencies = [
  "asn1-rs 0.7.1",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
 [[package]]
 name = "pem"
 version = "3.0.5"
@@ -356,6 +458,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
 [[package]]
 name = "rcgen"
 version = "0.13.2"
@@ -378,7 +486,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "libc",
  "untrusted",
  "windows-sys 0.52.0",
@@ -400,25 +508,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
 
 [[package]]
-name = "serde"
+name = "rustversion"
-version = "1.0.219"
+version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "ryu"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.219"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.145"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -541,6 +695,12 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "typenum"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.18"
@@ -553,12 +713,83 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
+[[package]]
+name = "uuid"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
+dependencies = [
+ "getrandom 0.3.4",
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasip2"
+version = "1.0.1+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"
+dependencies = [
+ "unicode-ident",
+]
+
 [[package]]
 name = "webpki"
 version = "0.22.4"
@@ -657,6 +888,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "wit-bindgen"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
+
 [[package]]
 name = "x509-parser"
 version = "0.16.0"

libraries/droplet/Cargo.toml

@@ -16,6 +16,11 @@ dyn-clone = "1.0.20"
 tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] }
 anyhow = "1.0.100"
 async-trait = "0.1.89"
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.145"
+humansize = "2.1.3"
+uuid = { version = "1.19.0", features = ["v4"] }
+sha2 = "0.10.9"
 
 [dependencies.x509-parser]
 version = "0.17.0"

libraries/droplet/src/lib.rs

@@ -3,3 +3,4 @@
 pub mod file_utils;
 pub mod ssl;
 pub mod versions;
+pub mod manifest;

libraries/droplet/src/manifest.rs

@@ -0,0 +1,193 @@
+use std::{collections::HashMap, path::Path};
+
+use anyhow::anyhow;
+use hex::ToHex as _;
+use humansize::{BINARY, format_size};
+use serde::Serialize;
+use serde_json::json;
+use sha2::{Digest as _, Sha256};
+use tokio::io::AsyncReadExt as _;
+
+#[derive(Serialize)]
+struct FileEntry {
+    filename: String,
+    start: usize,
+    length: usize,
+    permissions: u32,
+}
+
+#[derive(Serialize)]
+struct ChunkData {
+    files: Vec<FileEntry>,
+    checksum: String,
+}
+
+#[derive(Serialize)]
+struct Manifest {
+  version: String,
+  chunks: HashMap<String, ChunkData>,
+  size: u64,
+}
+
+const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
+const WIGGLE: u64 = 1024 * 1024 * 1;
+
+use crate::versions::{create_backend_constructor, types::VersionFile};
+
+pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
+    dir: &Path,
+    progress_sfn: V,
+    log_sfn: T,
+) -> anyhow::Result<String> {
+    let mut backend =
+        create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?;
+
+    let required_single_file = true; //backend.require_whole_files();
+
+    let files = backend.list_files().await?;
+    // Filepath to chunk data
+    let mut chunks: Vec<Vec<(VersionFile, u64, u64)>> = Vec::new();
+    let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new();
+
+    log_sfn(format!("organizing files into chunks...",));
+
+    for version_file in files {
+        // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on
+        if required_single_file && version_file.size >= CHUNK_SIZE {
+            let size = version_file.size;
+            chunks.push(vec![(version_file, 0, size)]);
+
+            continue;
+        }
+
+        let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::<u64>();
+
+        // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be
+        if required_single_file {
+            let size = version_file.size.try_into().unwrap();
+            current_chunk.push((version_file, 0, size));
+
+            current_size += size;
+
+            if current_size >= CHUNK_SIZE {
+                // Pop current and add, then reset
+                let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
+                chunks.push(new_chunk);
+            }
+
+            continue;
+        }
+
+        // Otherwise we calculate how much of the file we need, then use that much
+        let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size;
+        if version_file.size >= remaining_budget {
+            let remaining_budget = CHUNK_SIZE - current_size;
+            current_chunk.push((version_file.clone(), 0, remaining_budget));
+
+            let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
+            chunks.push(new_chunk);
+
+            let remaining_size = version_file.size - remaining_budget;
+            let mut running_offset = remaining_budget;
+            // Do everything but the last one
+            while running_offset < remaining_size {
+                let chunk_size = CHUNK_SIZE.min(remaining_size);
+                let chunk = vec![(version_file.clone(), running_offset, chunk_size)];
+                if chunk_size == CHUNK_SIZE {
+                    chunks.push(chunk);
+                } else {
+                    current_chunk = chunk;
+                }
+                running_offset += chunk_size;
+            }
+
+            continue;
+        } else {
+            let size = version_file.size;
+            current_chunk.push((version_file, 0, size));
+            current_size += size;
+        }
+
+        if current_size >= CHUNK_SIZE {
+            // Pop current and add, then reset
+            let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
+            chunks.push(new_chunk);
+        }
+    }
+    if current_chunk.len() > 0 {
+        chunks.push(current_chunk);
+    }
+
+    log_sfn(format!(
+        "organized into {} chunks, generating checksums...",
+        chunks.len()
+    ));
+
+    let mut manifest: HashMap<String, ChunkData> = HashMap::new();
+    let mut total_manifest_length = 0;
+
+    let mut read_buf = vec![0; 1024 * 1024 * 64];
+
+    let chunk_len = chunks.len();
+    for (index, chunk) in chunks.into_iter().enumerate() {
+        let uuid = uuid::Uuid::new_v4().to_string();
+        let mut hasher = Sha256::new();
+
+        let mut chunk_data = ChunkData {
+            files: Vec::new(),
+            checksum: String::new(),
+        };
+
+        let mut chunk_length = 0;
+
+        for (file, start, length) in chunk {
+            log_sfn(format!(
+                "reading {} from {} to {}, {}",
+                file.relative_filename,
+                start,
+                start + length,
+                format_size(length, BINARY)
+            ));
+            let mut reader = backend.reader(&file, start, start + length).await?;
+
+            loop {
+                let amount = reader.read(&mut read_buf).await?;
+                if amount == 0 {
+                    break;
+                }
+                hasher.update(&read_buf[0..amount]);
+            }
+
+            chunk_length += length;
+
+            chunk_data.files.push(FileEntry {
+                filename: file.relative_filename,
+                start: start.try_into().unwrap(),
+                length: length.try_into().unwrap(),
+                permissions: file.permission,
+            });
+        }
+
+        log_sfn(format!(
+            "created chunk of size {} ({}/{})",
+            format_size(chunk_length, BINARY),
+            index,
+            chunk_len
+        ));
+        total_manifest_length += chunk_length;
+
+        let hash: String = hasher.finalize().encode_hex();
+        chunk_data.checksum = hash;
+        manifest.insert(uuid, chunk_data);
+
+        let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32;
+        progress_sfn(progress);
+    }
+
+    Ok(json!(Manifest {
+        version: "2".to_string(),
+        chunks: manifest,
+        size: total_manifest_length
+    })
+    .to_string())
+}