Ryan/droplet
1
0
Fork 0
mirror of https://github.com/Drop-OSS/droplet.git synced 2026-07-27 02:14:27 +10:00
Code Issues Packages Projects Releases Wiki Activity

fix: move manifest generation to droplet-rs

Browse Source
This commit is contained in:
DecDuck
2025-12-13 15:28:35 +11:00
parent b8a7dc169c
commit 2dfb4b90ad
2 changed files with 11 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+7 -2
View File
@@ -505,18 +505,23 @@ dependencies = [
[[package]] [[package]]
name = "droplet-rs" name = "droplet-rs"
version = "0.9.2" version = "0.10.0"
source = "git+https://github.com/Drop-OSS/droplet-rs.git#d8f37886d479d8d9fec7e51523628e863306dddb" source = "git+https://github.com/Drop-OSS/droplet-rs.git#d7ac90eb2a25544868c8e447d8438f97111c8947"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
"dyn-clone", "dyn-clone",
"hex", "hex",
"humansize",
"rcgen", "rcgen",
"ring", "ring",
"serde",
"serde_json",
"sha2",
"time", "time",
"time-macros", "time-macros",
"tokio", "tokio",
"uuid",
"webpki", "webpki",
"x509-parser 0.17.0", "x509-parser 0.17.0",
] ]
src/manifest.rs
+3 -187
View File
@@ -1,7 +1,7 @@
use std::{collections::HashMap, sync::Arc, thread}; use std::{collections::HashMap, path::PathBuf, sync::Arc, thread};
use anyhow::anyhow; use anyhow::anyhow;
use droplet_rs::versions::types::VersionFile; use droplet_rs::{manifest::generate_manifest_rusty, versions::types::VersionFile};
use hashing_reader::HashingReader; use hashing_reader::HashingReader;
use hex::ToHex; use hex::ToHex;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
@@ -17,23 +17,6 @@ use uuid::Uuid;
use crate::version::create_backend_for_path; use crate::version::create_backend_for_path;
const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
const WIGGLE: u64 = 1024 * 1024 * 1;
#[derive(Serialize)]
struct FileEntry {
filename: String,
start: usize,
length: usize,
permissions: u32,
}
#[derive(Serialize)]
struct ChunkData {
files: Vec<FileEntry>,
checksum: String,
}
#[napi] #[napi]
pub fn call_alt_thread_func(tsfn: Arc<ThreadsafeFunction<()>>) -> Result<(), String> { pub fn call_alt_thread_func(tsfn: Arc<ThreadsafeFunction<()>>) -> Result<(), String> {
let tsfn_cloned = tsfn.clone(); let tsfn_cloned = tsfn.clone();
@@ -43,13 +26,6 @@ pub fn call_alt_thread_func(tsfn: Arc<ThreadsafeFunction<()>>) -> Result<(), Str
Ok(()) Ok(())
} }
#[derive(Serialize)]
struct Manifest {
version: String,
chunks: HashMap<String, ChunkData>,
size: u64,
}
#[napi] #[napi]
pub async fn generate_manifest( pub async fn generate_manifest(
dir: String, dir: String,
@@ -57,7 +33,7 @@ pub async fn generate_manifest(
log_sfn: ThreadsafeFunction<String>, log_sfn: ThreadsafeFunction<String>,
) -> anyhow::Result<String> { ) -> anyhow::Result<String> {
generate_manifest_rusty( generate_manifest_rusty(
dir, &PathBuf::from(dir),
|progress| { |progress| {
progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking); progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
}, },
@@ -67,163 +43,3 @@ pub async fn generate_manifest(
) )
.await .await
} }
pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
dir: String,
progress_sfn: V,
log_sfn: T,
) -> anyhow::Result<String> {
let mut backend =
create_backend_for_path(dir).ok_or(anyhow!("Could not create backend for path."))?;
let required_single_file = true; //backend.require_whole_files();
let files = backend.list_files().await?;
// Filepath to chunk data
let mut chunks: Vec<Vec<(VersionFile, u64, u64)>> = Vec::new();
let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new();
log_sfn(format!("organizing files into chunks...",));
for version_file in files {
// If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on
if required_single_file && version_file.size >= CHUNK_SIZE {
let size = version_file.size;
chunks.push(vec![(version_file, 0, size)]);
continue;
}
let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::<u64>();
// If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be
if required_single_file {
let size = version_file.size.try_into().unwrap();
current_chunk.push((version_file, 0, size));
current_size += size;
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
chunks.push(new_chunk);
}
continue;
}
// Otherwise we calculate how much of the file we need, then use that much
let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size;
if version_file.size >= remaining_budget {
let remaining_budget = CHUNK_SIZE - current_size;
current_chunk.push((version_file.clone(), 0, remaining_budget));
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
chunks.push(new_chunk);
let remaining_size = version_file.size - remaining_budget;
let mut running_offset = remaining_budget;
// Do everything but the last one
while running_offset < remaining_size {
let chunk_size = CHUNK_SIZE.min(remaining_size);
let chunk = vec![(version_file.clone(), running_offset, chunk_size)];
if chunk_size == CHUNK_SIZE {
chunks.push(chunk);
} else {
current_chunk = chunk;
}
running_offset += chunk_size;
}
continue;
} else {
let size = version_file.size;
current_chunk.push((version_file, 0, size));
current_size += size;
}
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
chunks.push(new_chunk);
}
}
if current_chunk.len() > 0 {
chunks.push(current_chunk);
}
log_sfn(format!(
"organized into {} chunks, generating checksums...",
chunks.len()
));
let mut manifest: HashMap<String, ChunkData> = HashMap::new();
let mut total_manifest_length = 0;
let mut read_buf = vec![0; 1024 * 1024 * 64];
let chunk_len = chunks.len();
for (index, chunk) in chunks.into_iter().enumerate() {
let uuid = uuid::Uuid::new_v4().to_string();
let mut hasher = Sha256::new();
let mut chunk_data = ChunkData {
files: Vec::new(),
checksum: String::new(),
};
let mut chunk_length = 0;
for (file, start, length) in chunk {
log_sfn(format!(
"reading {} from {} to {}, {}",
file.relative_filename,
start,
start + length,
format_size(length, BINARY)
));
let mut reader = backend.reader(&file, start, start + length).await?;
loop {
let amount = reader.read(&mut read_buf).await?;
if amount == 0 {
break;
}
hasher.update(&read_buf[0..amount]);
}
chunk_length += length;
chunk_data.files.push(FileEntry {
filename: file.relative_filename,
start: start.try_into().unwrap(),
length: length.try_into().unwrap(),
permissions: file.permission,
});
}
log_sfn(format!(
"created chunk of size {} ({}/{})",
format_size(chunk_length, BINARY),
index,
chunk_len
));
total_manifest_length += chunk_length;
let hash: String = hasher.finalize().encode_hex();
chunk_data.checksum = hash;
manifest.insert(uuid, chunk_data);
let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32;
progress_sfn(progress);
}
Ok(
json!(Manifest {
version: "2".to_string(),
chunks: manifest,
size: total_manifest_length
})
.to_string(),
)
}