Ryan/droplet
1
0
Fork 0
mirror of https://github.com/Drop-OSS/droplet.git synced 2025-11-17 02:01:19 +10:00
Code Issues Packages Projects Releases Wiki Activity

hashmap chunks for speed

Browse Source
This commit is contained in:
DecDuck
2024-10-13 15:02:04 +11:00
parent 0ddd263d74
commit 43c25f0caa
1 changed files with 7 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/manifest.rs
View File

@ -1,8 +1,5 @@
use std::{ use std::{
fs::File, collections::HashMap, fs::File, io::{BufRead, BufReader}, path::Path, thread
io::{BufRead, BufReader},
path::Path,
thread,
}; };
#[cfg(unix)] #[cfg(unix)]
@ -19,11 +16,10 @@ use uuid::Uuid;
use crate::file_utils::list_files; use crate::file_utils::list_files;
const CHUNK_SIZE: usize = 1024 * 1024 * 16; const CHUNK_SIZE: usize = 1024 * 1024 * 128;
#[derive(Serialize)] #[derive(Serialize)]
struct Chunk { struct Chunk {
id: String,
permissions: u32, permissions: u32,
file_name: String, file_name: String,
chunk_index: u32, chunk_index: u32,
@ -65,7 +61,7 @@ pub fn generate_manifest(
let base_dir = Path::new(&dir); let base_dir = Path::new(&dir);
let files = list_files(base_dir); let files = list_files(base_dir);
let mut chunks: Vec<Chunk> = Vec::new(); let mut chunks: HashMap<String, Chunk> = HashMap::new();
let total: i32 = files.len() as i32; let total: i32 = files.len() as i32;
let mut i: i32 = 0; let mut i: i32 = 0;
@ -100,7 +96,6 @@ pub fn generate_manifest(
let checksum_string = hex::encode(checksum.to_le_bytes()); let checksum_string = hex::encode(checksum.to_le_bytes());
let chunk = Chunk { let chunk = Chunk {
id: chunk_id.to_string(),
chunk_index: chunk_index, chunk_index: chunk_index,
permissions: permissions, permissions: permissions,
file_name: relative.to_str().unwrap().to_string(), file_name: relative.to_str().unwrap().to_string(),
@ -108,7 +103,10 @@ pub fn generate_manifest(
length: length, length: length,
}; };
chunks.push(chunk); let original = chunks.insert(chunk_id.to_string(), chunk);
if original.is_some() {
panic!("UUID collision");
}
let log_str = format!( let log_str = format!(
"Processed chunk {} for {}", "Processed chunk {} for {}",