diff --git a/Cargo.lock b/Cargo.lock index cfec974..cc2caf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,18 +505,23 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.9.2" -source = "git+https://github.com/Drop-OSS/droplet-rs.git#d8f37886d479d8d9fec7e51523628e863306dddb" +version = "0.10.0" +source = "git+https://github.com/Drop-OSS/droplet-rs.git#d7ac90eb2a25544868c8e447d8438f97111c8947" dependencies = [ "anyhow", "async-trait", "dyn-clone", "hex", + "humansize", "rcgen", "ring", + "serde", + "serde_json", + "sha2", "time", "time-macros", "tokio", + "uuid", "webpki", "x509-parser 0.17.0", ] diff --git a/src/manifest.rs b/src/manifest.rs index a4c71af..82bc991 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,7 +1,7 @@ -use std::{collections::HashMap, sync::Arc, thread}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, thread}; use anyhow::anyhow; -use droplet_rs::versions::types::VersionFile; +use droplet_rs::{manifest::generate_manifest_rusty, versions::types::VersionFile}; use hashing_reader::HashingReader; use hex::ToHex; use humansize::{format_size, BINARY}; @@ -17,23 +17,6 @@ use uuid::Uuid; use crate::version::create_backend_for_path; -const CHUNK_SIZE: u64 = 1024 * 1024 * 64; -const WIGGLE: u64 = 1024 * 1024 * 1; - -#[derive(Serialize)] -struct FileEntry { - filename: String, - start: usize, - length: usize, - permissions: u32, -} - -#[derive(Serialize)] -struct ChunkData { - files: Vec, - checksum: String, -} - #[napi] pub fn call_alt_thread_func(tsfn: Arc>) -> Result<(), String> { let tsfn_cloned = tsfn.clone(); @@ -43,13 +26,6 @@ pub fn call_alt_thread_func(tsfn: Arc>) -> Result<(), Str Ok(()) } -#[derive(Serialize)] -struct Manifest { - version: String, - chunks: HashMap, - size: u64, -} - #[napi] pub async fn generate_manifest( dir: String, @@ -57,7 +33,7 @@ pub async fn generate_manifest( log_sfn: ThreadsafeFunction, ) -> anyhow::Result { generate_manifest_rusty( - dir, + &PathBuf::from(dir), |progress| { progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking); }, @@ -66,164 +42,4 @@ pub async fn generate_manifest( }, ) .await -} - -pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( - dir: String, - progress_sfn: V, - log_sfn: T, -) -> anyhow::Result { - let mut backend = - create_backend_for_path(dir).ok_or(anyhow!("Could not create backend for path."))?; - - let required_single_file = true; //backend.require_whole_files(); - - let files = backend.list_files().await?; - // Filepath to chunk data - let mut chunks: Vec> = Vec::new(); - let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); - - log_sfn(format!("organizing files into chunks...",)); - - for version_file in files { - // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on - if required_single_file && version_file.size >= CHUNK_SIZE { - let size = version_file.size; - chunks.push(vec![(version_file, 0, size)]); - - continue; - } - - let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::(); - - // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be - if required_single_file { - let size = version_file.size.try_into().unwrap(); - current_chunk.push((version_file, 0, size)); - - current_size += size; - - if current_size >= CHUNK_SIZE { - // Pop current and add, then reset - let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); - chunks.push(new_chunk); - } - - continue; - } - - // Otherwise we calculate how much of the file we need, then use that much - let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size; - if version_file.size >= remaining_budget { - let remaining_budget = CHUNK_SIZE - current_size; - current_chunk.push((version_file.clone(), 0, remaining_budget)); - - let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); - chunks.push(new_chunk); - - let remaining_size = version_file.size - remaining_budget; - let mut running_offset = remaining_budget; - // Do everything but the last one - while running_offset < remaining_size { - let chunk_size = CHUNK_SIZE.min(remaining_size); - let chunk = vec![(version_file.clone(), running_offset, chunk_size)]; - if chunk_size == CHUNK_SIZE { - chunks.push(chunk); - } else { - current_chunk = chunk; - } - running_offset += chunk_size; - } - - continue; - } else { - let size = version_file.size; - current_chunk.push((version_file, 0, size)); - current_size += size; - } - - if current_size >= CHUNK_SIZE { - // Pop current and add, then reset - let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); - chunks.push(new_chunk); - } - } - if current_chunk.len() > 0 { - chunks.push(current_chunk); - } - - log_sfn(format!( - "organized into {} chunks, generating checksums...", - chunks.len() - )); - - let mut manifest: HashMap = HashMap::new(); - let mut total_manifest_length = 0; - - let mut read_buf = vec![0; 1024 * 1024 * 64]; - - let chunk_len = chunks.len(); - for (index, chunk) in chunks.into_iter().enumerate() { - let uuid = uuid::Uuid::new_v4().to_string(); - let mut hasher = Sha256::new(); - - let mut chunk_data = ChunkData { - files: Vec::new(), - checksum: String::new(), - }; - - let mut chunk_length = 0; - - for (file, start, length) in chunk { - log_sfn(format!( - "reading {} from {} to {}, {}", - file.relative_filename, - start, - start + length, - format_size(length, BINARY) - )); - let mut reader = backend.reader(&file, start, start + length).await?; - - loop { - let amount = reader.read(&mut read_buf).await?; - if amount == 0 { - break; - } - hasher.update(&read_buf[0..amount]); - } - - chunk_length += length; - - chunk_data.files.push(FileEntry { - filename: file.relative_filename, - start: start.try_into().unwrap(), - length: length.try_into().unwrap(), - permissions: file.permission, - }); - } - - log_sfn(format!( - "created chunk of size {} ({}/{})", - format_size(chunk_length, BINARY), - index, - chunk_len - )); - total_manifest_length += chunk_length; - - let hash: String = hasher.finalize().encode_hex(); - chunk_data.checksum = hash; - manifest.insert(uuid, chunk_data); - - let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32; - progress_sfn(progress); - } - - Ok( - json!(Manifest { - version: "2".to_string(), - chunks: manifest, - size: total_manifest_length - }) - .to_string(), - ) -} +} \ No newline at end of file