Ryan/drop
1
0
Fork 0
mirror of https://github.com/Drop-OSS/drop.git synced 2026-06-22 04:11:32 +10:00
Code Issues Packages Projects Releases Wiki Activity

fix: progress reporting

Browse Source
This commit is contained in:
DecDuck
2025-12-13 21:20:57 +11:00
parent 62ff8be3c0
commit ed0b7bcf42
7 changed files with 27 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/droplet/Cargo.lock
Generated
-12
View File
@@ -233,10 +233,8 @@ dependencies = [
"serde_json",
"sha2",
"time",
"time-macros",
"tokio",
"uuid",
"webpki",
"x509-parser 0.17.0",
]
@@ -904,16 +902,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "webpki"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "windows-link"
version = "0.2.1"
libraries/droplet/Cargo.toml
-2
View File
@@ -8,9 +8,7 @@ description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performa
[dependencies]
hex = "0.4.3"
time-macros = "0.2.22"
time = "0.3.41"
webpki = "0.22.4"
ring = "0.17.14"
dyn-clone = "1.0.20"
tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "sync", "macros", "rt-multi-thread"] }
libraries/droplet/src/main.rs
+4 -2
View File
@@ -1,7 +1,7 @@
use std::path::PathBuf;
use droplet_rs::manifest::generate_manifest_rusty;
use tokio::runtime::{Handle, Runtime};
use tokio::runtime::Handle;
#[tokio::main]
pub async fn main() {
@@ -9,7 +9,9 @@ pub async fn main() {
println!("using {} workers", metrics.num_workers());
generate_manifest_rusty(
&PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/Savage Resurrection"),
|_| {},
|progress| {
println!("PROGRESS: {}", progress)
},
|message| {
println!("{}", message);
},
libraries/droplet/src/manifest.rs
+17 -24
View File
@@ -39,11 +39,11 @@ struct Manifest {
}
const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
const WIGGLE: u64 = 1024 * 1024 * 1;
const WIGGLE: u64 = 1024 * 1024;
use crate::versions::{create_backend_constructor, types::VersionFile};
pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
dir: &Path,
progress_sfn: V,
log_sfn: T,
@@ -58,7 +58,7 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
let mut chunks: Vec<Vec<(VersionFile, u64, u64)>> = Vec::new();
let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new();
log_sfn(format!("organizing files into chunks...",));
log_sfn("organizing files into chunks...".to_string());
for version_file in files {
// If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on
@@ -73,14 +73,14 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
// If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be
if required_single_file {
let size = version_file.size.try_into().unwrap();
let size = version_file.size;
current_chunk.push((version_file, 0, size));
current_size += size;
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
let new_chunk = std::mem::take(&mut current_chunk);
chunks.push(new_chunk);
}
@@ -93,7 +93,7 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
let remaining_budget = CHUNK_SIZE - current_size;
current_chunk.push((version_file.clone(), 0, remaining_budget));
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
let new_chunk = std::mem::take(&mut current_chunk);
chunks.push(new_chunk);
let remaining_size = version_file.size - remaining_budget;
@@ -119,11 +119,11 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
let new_chunk = std::mem::take(&mut current_chunk);
chunks.push(new_chunk);
}
}
if current_chunk.len() > 0 {
if !current_chunk.is_empty() {
chunks.push(current_chunk);
}
@@ -140,12 +140,13 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
let futures: FuturesUnordered<impl Future<Output = Result<(), Error>>> =
FuturesUnordered::new();
let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16);
let chunks_length = chunks.len();
for (index, chunk) in chunks.into_iter().enumerate() {
let send_log = send_log.clone();
let backend = backend.clone();
let total_manifest_length = total_manifest_length.clone();
let manifest = manifest.clone();
futures.push((async move || -> Result<(), Error> {
futures.push(async move {
let mut read_buf = vec![0; 1024 * 1024 * 64];
let uuid = uuid::Uuid::new_v4().to_string();
@@ -159,17 +160,6 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
let mut chunk_length = 0;
for (file, start, length) in chunk {
/*
send_log
.send(format!(
"reading {} from {} to {}, {}",
file.relative_filename,
start,
start + length,
format_size(length, BINARY)
))
.await?;
*/
let mut reader = {
let mut backend_lock = backend.lock().await;
let reader = backend_lock.reader(&file, start, start + length).await?;
@@ -196,8 +186,9 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
send_log
.send(format!(
"created chunk of size {} (index {})",
"created chunk of size {} from {} files (index {})",
format_size(chunk_length, BINARY),
chunk_data.files.len(),
index
))
.await?;
@@ -212,13 +203,17 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
};
Ok(())
})());
});
}
drop(send_log);
join!(
async move {
let mut current_progress = 0f32;
let total_progress = chunks_length as f32;
while let Some(message) = recieve_log.recv().await {
log_sfn(message);
current_progress += 1.0f32;
progress_sfn((current_progress / total_progress) * 100.0f32)
}
},
futures.collect::<Vec<Result<(), Error>>>()
@@ -226,8 +221,6 @@ pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
let manifest = manifest.lock().await;
let manifest = manifest.clone();
let manifest_size = size_of_val(&manifest);
println!("manifest uses {} bytes", manifest_size);
Ok(json!(Manifest {
version: "2".to_string(),
libraries/droplet/src/versions/backends.rs
+3 -4
View File
@@ -159,8 +159,8 @@ impl VersionBackend for ZipVersionBackend {
let raw_result = String::from_utf8(result.stdout)?;
let files = raw_result
.split("\n")
.filter(|v| v.len() > 0)
.map(|v| v.split(" ").filter(|v| v.len() > 0));
.filter(|v| !v.is_empty())
.map(|v| v.split(" ").filter(|v| !v.is_empty()));
let mut results = Vec::new();
for file in files {
@@ -179,8 +179,7 @@ impl VersionBackend for ZipVersionBackend {
}
results.push(VersionFile {
relative_filename: name
.into_iter()
.map(|v| *v)
.into_iter().copied()
.fold(String::new(), |a, b| a + b + " ")
.trim_end()
.to_owned(),
libraries/droplet/src/versions/mod.rs
+1 -2
View File
@@ -41,8 +41,7 @@ pub fn create_backend_constructor<'a>(
if let Some(extension) = path.extension().and_then(|v| v.to_str()) {
let supported = SUPPORTED_FILE_EXTENSIONS
.iter()
.find(|v| ***v == *extension)
.is_some();
.any(|v| **v == *extension);
if supported {
let buf = path.to_path_buf();
return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
libraries/droplet/src/versions/types.rs
+2 -2
View File
@@ -1,8 +1,8 @@
use std::{fmt::Debug, io::Read};
use std::fmt::Debug;
use async_trait::async_trait;
use dyn_clone::DynClone;
use tokio::io::{self, AsyncRead};
use tokio::io::AsyncRead;
#[derive(Debug, Clone)]
pub struct VersionFile {