Ryan/droplet
1
0
Fork 0
mirror of https://github.com/Drop-OSS/droplet.git synced 2025-11-10 04:22:16 +10:00
Code Issues Packages Projects Releases Wiki Activity

got rid of thread utils, and made the processing properly async

Browse Source
This commit is contained in:
DecDuck
2024-10-11 16:33:11 +11:00
parent 3e68f0524e
commit 73e8f9b42c
1 changed files with 72 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
src/manifest.rs
View File

@ -1,7 +1,8 @@
use std::{
fs::File,
io::{BufRead, BufReader},
path::Path, thread,
path::Path,
thread,
};
#[cfg(unix)]
@ -47,67 +48,87 @@ pub fn generate_manifest(
dir: String,
progress: JsFunction,
log: JsFunction,
) -> Result<String, Error> {
let base_dir = Path::new(&dir);
let files = list_files(base_dir);
callback: JsFunction,
) -> Result<(), Error> {
let progress_sfn: ThreadsafeFunction<i32, ErrorStrategy::CalleeHandled> = progress
.create_threadsafe_function(0, |ctx| ctx.env.create_int32(ctx.value).map(|v| vec![v]))
.unwrap();
let log_sfn: ThreadsafeFunction<String, ErrorStrategy::CalleeHandled> = log
.create_threadsafe_function(0, |ctx| ctx.env.create_string_from_std(ctx.value).map(|v| vec![v]))
.unwrap();
let callback_sfn: ThreadsafeFunction<String, ErrorStrategy::CalleeHandled> = callback
.create_threadsafe_function(0, |ctx| ctx.env.create_string_from_std(ctx.value).map(|v| vec![v]))
.unwrap();
let mut chunks: Vec<Chunk> = Vec::new();
thread::spawn(move || {
let base_dir = Path::new(&dir);
let files = list_files(base_dir);
let total: i32 = files.len() as i32;
let mut i: i32 = 0;
for file_path in files {
let file = File::open(file_path.clone()).unwrap();
let relative = file_path.strip_prefix(base_dir).unwrap();
let permission_object = file.try_clone().unwrap().metadata().unwrap().permissions();
let permissions = {
let mut perm = 0;
#[cfg(unix)]
{
perm = permission_object.mode();
}
perm
};
let mut chunks: Vec<Chunk> = Vec::new();
let mut reader = BufReader::with_capacity(CHUNK_SIZE, file);
let total: i32 = files.len() as i32;
let mut i: i32 = 0;
let mut chunk_index = 0;
loop {
let mut buffer: Vec<u8> = Vec::new();
reader.fill_buf().unwrap().clone_into(&mut buffer);
let length = buffer.len();
if length == 0 {
break;
}
let chunk_id = Uuid::new_v4();
let checksum = gxhash128(&buffer, 0);
let checksum_string = hex::encode(checksum.to_le_bytes());
let chunk = Chunk {
id: chunk_id.to_string(),
chunk_index: chunk_index,
permissions: permissions,
file_name: relative.to_str().unwrap().to_string(),
checksum: checksum_string,
for file_path in files {
let file = File::open(file_path.clone()).unwrap();
let relative = file_path.strip_prefix(base_dir).unwrap();
let permission_object = file.try_clone().unwrap().metadata().unwrap().permissions();
let permissions = {
let mut perm = 0;
#[cfg(unix)]
{
perm = permission_object.mode();
}
perm
};
chunks.push(chunk);
let mut reader = BufReader::with_capacity(CHUNK_SIZE, file);
log
.call1::<String, ()>(format!(
let mut chunk_index = 0;
loop {
let mut buffer: Vec<u8> = Vec::new();
reader.fill_buf().unwrap().clone_into(&mut buffer);
let length = buffer.len();
if length == 0 {
break;
}
let chunk_id = Uuid::new_v4();
let checksum = gxhash128(&buffer, 0);
let checksum_string = hex::encode(checksum.to_le_bytes());
let chunk = Chunk {
id: chunk_id.to_string(),
chunk_index: chunk_index,
permissions: permissions,
file_name: relative.to_str().unwrap().to_string(),
checksum: checksum_string,
};
chunks.push(chunk);
let log_str = format!(
"Processed chunk {} for {}",
chunk_index,
relative.to_str().unwrap()
))
.unwrap();
reader.consume(length);
chunk_index += 1;
);
log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
reader.consume(length);
chunk_index += 1;
}
i += 1;
let progress = i * 100 / total;
progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
}
i += 1;
progress.call1::<i32, ()>(i * 100 / total).unwrap();
}
callback_sfn.call(
Ok(json!(chunks).to_string()),
ThreadsafeFunctionCallMode::Blocking,
);
});
Ok(json!(chunks).to_string())
return Ok(());
}