fixed multi-chunk downloads

This commit is contained in:
DecDuck
2024-10-25 10:28:58 +11:00
parent 4779383fc9
commit 2ec351f20e
5 changed files with 21 additions and 11 deletions

1
src-tauri/Cargo.lock generated
View File

@ -1028,6 +1028,7 @@ dependencies = [
"rayon", "rayon",
"reqwest", "reqwest",
"rustbreak", "rustbreak",
"rustix",
"serde", "serde",
"serde_json", "serde_json",
"structured-logger", "structured-logger",

View File

@ -40,6 +40,7 @@ http = "1.1.0"
tokio = { version = "1.40.0", features = ["rt", "tokio-macros"] } tokio = { version = "1.40.0", features = ["rt", "tokio-macros"] }
versions = { version = "6.3.2", features = ["serde"] } versions = { version = "6.3.2", features = ["serde"] }
urlencoding = "2.1.3" urlencoding = "2.1.3"
rustix = "0.38.37"
[dependencies.uuid] [dependencies.uuid]
version = "1.10.0" version = "1.10.0"

View File

@ -3,8 +3,8 @@ use crate::db::DatabaseImpls;
use crate::downloads::manifest::DropDownloadContext; use crate::downloads::manifest::DropDownloadContext;
use crate::DB; use crate::DB;
use log::info; use log::info;
use std::{fs::OpenOptions, io::{BufWriter, Seek, SeekFrom, Write}};
use urlencoding::encode; use urlencoding::encode;
use std::io::{BufWriter, Seek, SeekFrom, Write};
pub fn download_game_chunk(ctx: DropDownloadContext) { pub fn download_game_chunk(ctx: DropDownloadContext) {
let base_url = DB.fetch_base_url(); let base_url = DB.fetch_base_url();
@ -14,7 +14,10 @@ pub fn download_game_chunk(ctx: DropDownloadContext) {
.join(&format!( .join(&format!(
"/api/v1/client/chunk?id={}&version={}&name={}&chunk={}", "/api/v1/client/chunk?id={}&version={}&name={}&chunk={}",
// Encode the parts we don't trust // Encode the parts we don't trust
ctx.game_id, encode(&ctx.version), encode(&ctx.file_name), ctx.index ctx.game_id,
encode(&ctx.version),
encode(&ctx.file_name),
ctx.index
)) ))
.unwrap(); .unwrap();
@ -26,16 +29,15 @@ pub fn download_game_chunk(ctx: DropDownloadContext) {
.send() .send()
.unwrap(); .unwrap();
let mut file_lock = ctx.file.lock().unwrap(); let mut file = OpenOptions::new().write(true).open(ctx.path).unwrap();
if ctx.offset != 0 { if ctx.offset != 0 {
file_lock file
.seek(SeekFrom::Start(ctx.offset)) .seek(SeekFrom::Start(ctx.offset))
.expect("Failed to seek to file offset"); .expect("Failed to seek to file offset");
} }
let mut stream = BufWriter::with_capacity(1024 * 1024, file_lock.try_clone().unwrap()); let mut stream = BufWriter::with_capacity(1024 * 1024, file);
drop(file_lock);
response.copy_to(&mut stream).unwrap(); response.copy_to(&mut stream).unwrap();
} }

View File

@ -5,6 +5,7 @@ use crate::downloads::manifest::{DropDownloadContext, DropManifest};
use crate::downloads::progress::ProgressChecker; use crate::downloads::progress::ProgressChecker;
use crate::{AppState, DB}; use crate::{AppState, DB};
use log::info; use log::info;
use rustix::fs::{fallocate, FallocateFlags};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::fs::{create_dir_all, File}; use std::fs::{create_dir_all, File};
use std::path::Path; use std::path::Path;
@ -64,7 +65,7 @@ impl GameDownloadManager {
} }
self.ensure_manifest_exists().await self.ensure_manifest_exists().await
} }
pub fn begin_download( pub fn begin_download(
&self, &self,
max_threads: usize, max_threads: usize,
@ -143,7 +144,7 @@ pub fn generate_job_contexts(
let container = path.parent().unwrap(); let container = path.parent().unwrap();
create_dir_all(container).unwrap(); create_dir_all(container).unwrap();
let file = Arc::new(Mutex::new(File::create(path).unwrap())); let file = File::create(path.clone()).unwrap();
let mut running_offset = 0; let mut running_offset = 0;
for i in 0..chunk.ids.len() { for i in 0..chunk.ids.len() {
@ -153,10 +154,12 @@ pub fn generate_job_contexts(
offset: running_offset, offset: running_offset,
index: i, index: i,
game_id: game_id.to_string(), game_id: game_id.to_string(),
file: file.clone(), path: path.clone(),
}); });
running_offset += chunk.lengths[i] as u64; running_offset += chunk.lengths[i] as u64;
} }
fallocate(file, FallocateFlags::empty(), 0, running_offset);
} }
contexts contexts
} }
@ -184,7 +187,9 @@ pub async fn start_game_download(
let contexts = generate_job_contexts(&local_manifest, game_version.clone(), game_id); let contexts = generate_job_contexts(&local_manifest, game_version.clone(), game_id);
let _ = download_manager.begin_download(max_threads, contexts); download_manager
.begin_download(max_threads, contexts)
.unwrap();
Ok(()) Ok(())
} }

View File

@ -1,5 +1,6 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::File; use std::fs::File;
use std::path::PathBuf;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -19,5 +20,5 @@ pub struct DropDownloadContext {
pub index: usize, pub index: usize,
pub offset: u64, pub offset: u64,
pub game_id: String, pub game_id: String,
pub file: Arc<Mutex<File>> pub path: PathBuf
} }