feat: Download validation

Signed-off-by: quexeky <git@quexeky.dev>

components/GameStatusButton.vue

@@ -83,7 +83,8 @@ const emit = defineEmits<{
 const showDropdown = computed(
   () =>
     props.status.type === GameStatusEnum.Installed ||
-    props.status.type === GameStatusEnum.SetupRequired
+    props.status.type === GameStatusEnum.SetupRequired ||
+    props.status.type === GameStatusEnum.PartiallyInstalled
 );
 
 const styles: { [key in GameStatusEnum]: string } = {

pages/queue.vue

@@ -91,7 +91,12 @@
 <script setup lang="ts">
 import { ServerIcon, XMarkIcon } from "@heroicons/vue/20/solid";
 import { invoke } from "@tauri-apps/api/core";
-import type { DownloadableMetadata, Game, GameStatus } from "~/types";
+import { GameStatusEnum, type DownloadableMetadata, type Game, type GameStatus } from "~/types";
+
+// const actionNames = {
+//   [GameStatusEnum.Downloading]: "downloading",
+//   [GameStatusEnum.Verifying]: "verifying",
+// }
 
 const windowWidth = ref(window.innerWidth);
 window.addEventListener("resize", (event) => {

src-tauri/src/download_manager/download_manager_builder.rs

@@ -18,7 +18,12 @@ use crate::{
 
 use super::{
     download_manager::{DownloadManager, DownloadManagerSignal, DownloadManagerStatus},
-    downloadable::Downloadable, util::{download_thread_control_flag::{DownloadThreadControl, DownloadThreadControlFlag}, progress_object::ProgressObject, queue::Queue},
+    downloadable::Downloadable,
+    util::{
+        download_thread_control_flag::{DownloadThreadControl, DownloadThreadControlFlag},
+        progress_object::ProgressObject,
+        queue::Queue,
+    },
 };
 
 pub type DownloadAgent = Arc<Box<dyn Downloadable + Send + Sync>>;
@@ -243,16 +248,29 @@ impl DownloadManagerBuilder {
                 // Ok(true) is for completed and exited properly
                 Ok(true) => {
                     debug!("download {:?} has completed", download_agent.metadata());
-                    download_agent.on_complete(&app_handle);
+                    match download_agent.validate() {
-                    sender
+                        Ok(true) => {
-                        .send(DownloadManagerSignal::Completed(download_agent.metadata()))
+                            download_agent.on_complete(&app_handle);
-                        .unwrap();
+                            sender
+                                .send(DownloadManagerSignal::Completed(download_agent.metadata()))
+                                .unwrap();
+                        }
+                        Ok(false) => {
+                            download_agent.on_incomplete(&app_handle);
+                        }
+                        Err(e) => {
+                            error!(
+                                "download {:?} has validation error {}",
+                                download_agent.metadata(),
+                                &e
+                            );
+                        }
+                    }
                 }
                 // Ok(false) is for incomplete but exited properly
                 Ok(false) => {
                     debug!("Donwload agent finished incomplete");
                     download_agent.on_incomplete(&app_handle);
-                    
                 }
                 Err(e) => {
                     error!("download {:?} has error {}", download_agent.metadata(), &e);

src-tauri/src/download_manager/downloadable.rs

@@ -15,6 +15,7 @@ pub trait Downloadable: Send + Sync {
     fn download(&self, app_handle: &AppHandle) -> Result<bool, ApplicationDownloadError>;
     fn progress(&self) -> Arc<ProgressObject>;
     fn control_flag(&self) -> DownloadThreadControl;
+    fn validate(&self) -> Result<bool, ApplicationDownloadError>;
     fn status(&self) -> DownloadStatus;
     fn metadata(&self) -> DownloadableMetadata;
     fn on_initialised(&self, app_handle: &AppHandle);

src-tauri/src/download_manager/util/progress_object.rs

@@ -86,6 +86,13 @@ impl ProgressObject {
             .map(|instance| instance.load(Ordering::Relaxed))
             .sum()
     }
+    pub fn reset(&self, size: usize) {
+        self.set_time_now();
+        self.set_size(size);
+        self.bytes_last_update.store(0, Ordering::Release);
+        self.rolling.reset();
+        self.progress_instances.lock().unwrap().iter().for_each(|x| x.store(0, Ordering::Release));
+    }
     pub fn get_max(&self) -> usize {
         *self.max.lock().unwrap()
     }

src-tauri/src/download_manager/util/rolling_progress_updates.rs

@@ -30,4 +30,7 @@ impl<const S: usize> RollingProgressWindow<S> {
             .sum::<usize>()
             / S
     }
+    pub fn reset(&self) {
+        self.window.iter().for_each(|x| x.store(0, Ordering::Release));
+    }
 }

src-tauri/src/games/downloads/download_agent.rs

@@ -12,6 +12,7 @@ use crate::download_manager::util::progress_object::{ProgressHandle, ProgressObj
 use crate::error::application_download_error::ApplicationDownloadError;
 use crate::error::remote_access_error::RemoteAccessError;
 use crate::games::downloads::manifest::{DropDownloadContext, DropManifest};
+use crate::games::downloads::validate::game_validate_logic;
 use crate::games::library::{
     on_game_complete, on_game_incomplete, push_game_update, GameUpdateEvent,
 };
@@ -354,7 +355,11 @@ impl GameDownloadAgent {
             .map(|x| {
                 (
                     x.checksum.clone(),
-                    context_map_lock.get(&x.checksum).cloned().or(Some(false)).unwrap(),
+                    context_map_lock
+                        .get(&x.checksum)
+                        .cloned()
+                        .or(Some(false))
+                        .unwrap(),
                 )
             })
             .collect::<Vec<(String, bool)>>();
@@ -442,4 +447,14 @@ impl Downloadable for GameDownloadAgent {
     fn status(&self) -> DownloadStatus {
         self.status.lock().unwrap().clone()
     }
+
+    fn validate(&self) -> Result<bool, ApplicationDownloadError> {
+        game_validate_logic(
+            &self.stored_manifest,
+            self.contexts.lock().unwrap().clone(),
+            self.progress.clone(),
+            self.sender.clone(),
+            &self.control_flag,
+        )
+    }
 }

src-tauri/src/games/downloads/download_logic.rs

@@ -5,11 +5,14 @@ use crate::download_manager::util::progress_object::ProgressHandle;
 use crate::error::application_download_error::ApplicationDownloadError;
 use crate::error::remote_access_error::RemoteAccessError;
 use crate::games::downloads::drop_data::DropData;
-use crate::games::downloads::manifest::DropDownloadContext;
+use crate::games::downloads::manifest::{DropDownloadContext, DropManifest};
+use crate::remote::auth::generate_authorization_header;
+use crate::remote::requests::make_request;
 use log::{debug, warn};
 use md5::{Context, Digest};
 use native_model::Decode;
 use reqwest::blocking::{RequestBuilder, Response};
+use reqwest::Client;
 
 use std::fs::{set_permissions, Permissions};
 use std::io::{copy, ErrorKind, Read};
@@ -204,84 +207,3 @@ pub fn download_game_chunk(
 
     Ok(true)
 }
-
-pub fn validate(path: PathBuf) -> Result<Vec<String>, ApplicationDownloadError> {
-    let mut dropdata = File::open(path.join(".dropdata")).unwrap();
-    let mut buf = Vec::new();
-    dropdata.read_to_end(&mut buf);
-    let manifest: DropData = native_model::rmp_serde_1_3::RmpSerde::decode(buf).unwrap();
-    let completed_contexts = manifest.get_completed_contexts();
-    todo!()
-}
-
-pub fn validate_game_chunk(
-    ctx: &DropDownloadContext,
-    control_flag: &DownloadThreadControl,
-    progress: ProgressHandle,
-) -> Result<bool, ApplicationDownloadError> {
-    debug!(
-        "Starting chunk validation {}, {}, {} #{}",
-        ctx.file_name, ctx.index, ctx.offset, ctx.checksum
-    );
-    // If we're paused
-    if control_flag.get() == DownloadThreadControlFlag::Stop {
-        progress.set(0);
-        return Ok(false);
-    }
-
-    let mut source = File::open(&ctx.path).unwrap();
-
-    if ctx.offset != 0 {
-        source
-            .seek(SeekFrom::Start(ctx.offset))
-            .expect("Failed to seek to file offset");
-    }
-
-    let mut hasher = md5::Context::new();
-
-    let completed = validate_copy(&mut source, &mut hasher, control_flag, progress).unwrap();
-    if !completed {
-        return Ok(false);
-    };
-
-    let res = hex::encode(hasher.compute().0);
-    if res != ctx.checksum {
-        return Err(ApplicationDownloadError::Checksum);
-    }
-
-    debug!(
-        "Successfully finished verification #{}, copied {} bytes",
-        ctx.checksum, ctx.length
-    );
-
-    Ok(true)
-}
-
-fn validate_copy(
-    source: &mut File,
-    dest: &mut Context,
-    control_flag: &DownloadThreadControl,
-    progress: ProgressHandle,
-) -> Result<bool, io::Error> {
-    let copy_buf_size = 512;
-    let mut copy_buf = vec![0; copy_buf_size];
-    let mut buf_writer = BufWriter::with_capacity(1024 * 1024, dest);
-
-    loop {
-        if control_flag.get() == DownloadThreadControlFlag::Stop {
-            buf_writer.flush()?;
-            return Ok(false);
-        }
-
-        let bytes_read = source.read(&mut copy_buf)?;
-
-        buf_writer.write_all(&copy_buf[0..bytes_read])?;
-        progress.add(bytes_read);
-
-        if bytes_read == 0 {
-            break;
-        }
-    }
-    buf_writer.flush()?;
-    Ok(true)
-}

src-tauri/src/games/downloads/drop_data.rs

@@ -17,8 +17,8 @@ pub mod v1 {
     #[derive(Serialize, Deserialize, Debug)]
     #[native_model(id = 9, version = 1, with = native_model::rmp_serde_1_3::RmpSerde)]
     pub struct DropData {
-        game_id: String,
+        pub game_id: String,
-        game_version: String,
+        pub game_version: String,
         pub contexts: Mutex<Vec<(String, bool)>>,
         pub base_path: PathBuf,
     }

src-tauri/src/games/downloads/mod.rs

@@ -3,3 +3,4 @@ pub mod download_agent;
 mod download_logic;
 mod manifest;
 mod drop_data;
+pub mod validate;

src-tauri/src/games/downloads/validate.rs

@@ -0,0 +1,198 @@
+use std::{
+    fs::File,
+    io::{self, BufWriter, Read, Seek, SeekFrom, Write},
+    path::{Path, PathBuf},
+    sync::{mpsc::Sender, Arc},
+};
+
+use log::{debug, error, info};
+use md5::Context;
+use native_model::Decode;
+use rayon::ThreadPoolBuilder;
+
+use crate::{
+    database::db::borrow_db_checked,
+    download_manager::{
+        download_manager::DownloadManagerSignal,
+        downloadable::Downloadable,
+        util::{
+            download_thread_control_flag::{DownloadThreadControl, DownloadThreadControlFlag},
+            progress_object::{ProgressHandle, ProgressObject},
+        },
+    },
+    error::{
+        application_download_error::ApplicationDownloadError,
+        remote_access_error::RemoteAccessError,
+    },
+    games::downloads::{
+        drop_data::DropData,
+        manifest::{DropDownloadContext, DropManifest},
+    },
+    remote::{auth::generate_authorization_header, requests::make_request},
+};
+
+pub fn game_validate_logic(
+    dropdata: &DropData,
+    contexts: Vec<DropDownloadContext>,
+    progress: Arc<ProgressObject>,
+    sender: Sender<DownloadManagerSignal>,
+    control_flag: &DownloadThreadControl,
+) -> Result<bool, ApplicationDownloadError> {
+    progress.reset(contexts.len());
+    let max_download_threads = borrow_db_checked().settings.max_download_threads;
+
+    debug!(
+        "validating game: {} with {} threads",
+        dropdata.game_id, max_download_threads
+    );
+    let pool = ThreadPoolBuilder::new()
+        .num_threads(max_download_threads)
+        .build()
+        .unwrap();
+
+    debug!("{:#?}", contexts);
+    let invalid_chunks = Arc::new(boxcar::Vec::new());
+    pool.scope(|scope| {
+        let client = &reqwest::blocking::Client::new();
+        for (index, context) in contexts.iter().enumerate() {
+            let client = client.clone();
+
+            let current_progress = progress.get(index);
+            let progress_handle = ProgressHandle::new(current_progress, progress.clone());
+            let invalid_chunks_scoped = invalid_chunks.clone();
+            let sender = sender.clone();
+
+            let request = match make_request(
+                &client,
+                &["/api/v1/client/chunk"],
+                &[
+                    ("id", &context.game_id),
+                    ("version", &context.version),
+                    ("name", &context.file_name),
+                    ("chunk", &context.index.to_string()),
+                ],
+                |r| r.header("Authorization", generate_authorization_header()),
+            ) {
+                Ok(request) => request,
+                Err(e) => {
+                    sender
+                        .send(DownloadManagerSignal::Error(
+                            ApplicationDownloadError::Communication(e),
+                        ))
+                        .unwrap();
+                    continue;
+                }
+            };
+
+            scope.spawn(move |_| {
+                match validate_game_chunk(context, control_flag, progress_handle) {
+                    Ok(true) => {
+                        debug!(
+                            "Finished context #{} with checksum {}",
+                            index, context.checksum
+                        );
+                    }
+                    Ok(false) => {
+                        debug!(
+                            "Didn't finish context #{} with checksum {}",
+                            index, &context.checksum
+                        );
+                        invalid_chunks_scoped.push(context.checksum.clone());
+                    }
+                    Err(e) => {
+                        error!("{}", e);
+                        sender.send(DownloadManagerSignal::Error(e)).unwrap();
+                    }
+                }
+            });
+        }
+    });
+
+    // If there are any contexts left which are false
+    if !invalid_chunks.is_empty() {
+        info!(
+            "validation of game id {} failed for chunks {:?}",
+            dropdata.game_id.clone(),
+            invalid_chunks
+        );
+        return Ok(false);
+    }
+
+    Ok(true)
+}
+
+pub fn validate_game_chunk(
+    ctx: &DropDownloadContext,
+    control_flag: &DownloadThreadControl,
+    progress: ProgressHandle,
+) -> Result<bool, ApplicationDownloadError> {
+    debug!(
+        "Starting chunk validation {}, {}, {} #{}",
+        ctx.file_name, ctx.index, ctx.offset, ctx.checksum
+    );
+    // If we're paused
+    if control_flag.get() == DownloadThreadControlFlag::Stop {
+        progress.set(0);
+        return Ok(false);
+    }
+
+    let mut source = File::open(&ctx.path).unwrap();
+
+    if ctx.offset != 0 {
+        source
+            .seek(SeekFrom::Start(ctx.offset))
+            .expect("Failed to seek to file offset");
+    }
+
+    let mut hasher = md5::Context::new();
+
+    let completed = validate_copy(&mut source, &mut hasher, control_flag, progress).unwrap();
+    if !completed {
+        return Ok(false);
+    };
+
+    let res = hex::encode(hasher.compute().0);
+    if res != ctx.checksum {
+        println!(
+            "Checksum failed. Correct: {}, actual: {}",
+            &ctx.checksum, &res
+        );
+        return Ok(false);
+    }
+
+    debug!(
+        "Successfully finished verification #{}, copied {} bytes",
+        ctx.checksum, ctx.length
+    );
+
+    Ok(true)
+}
+
+fn validate_copy(
+    source: &mut File,
+    dest: &mut Context,
+    control_flag: &DownloadThreadControl,
+    progress: ProgressHandle,
+) -> Result<bool, io::Error> {
+    let copy_buf_size = 512;
+    let mut copy_buf = vec![0; copy_buf_size];
+    let mut buf_writer = BufWriter::with_capacity(1024 * 1024, dest);
+
+    loop {
+        if control_flag.get() == DownloadThreadControlFlag::Stop {
+            buf_writer.flush()?;
+            return Ok(false);
+        }
+
+        let bytes_read = source.read(&mut copy_buf)?;
+
+        buf_writer.write_all(&copy_buf[0..bytes_read])?;
+        progress.add(bytes_read);
+
+        if bytes_read == 0 {
+            break;
+        }
+    }
+    buf_writer.flush()?;
+    Ok(true)
+}

src-tauri/src/games/library.rs

@@ -313,6 +313,10 @@ pub fn uninstall_game_logic(meta: DownloadableMetadata, app_handle: &AppHandle)
             version_name,
             install_dir,
         } => Some((version_name, install_dir)),
+        GameDownloadStatus::PartiallyInstalled {
+            version_name,
+            install_dir,
+        } => Some((version_name, install_dir)),
         _ => None,
     } {
         db_handle
@@ -353,6 +357,8 @@ pub fn uninstall_game_logic(meta: DownloadableMetadata, app_handle: &AppHandle)
                 );
             }
         });
+    } else {
+        warn!("invalid previous state for uninstall, failing silently.")
     }
 }