Collections & download stability, UI (#130)

* feat: different local path in dev #73

* feat: better error output for downloads

* feat: collections in library view

* feat: improve download manager reliability

* feat: new download UI, more stable downloads

* fix: clippy

* fix: only show admin link if user is admin

* feat: check for libs before building

src-tauri/src/games/downloads/download_agent.rs

@@ -23,7 +23,7 @@ use crate::remote::utils::{DROP_CLIENT_ASYNC, DROP_CLIENT_SYNC};
 use log::{debug, error, info, warn};
 use rayon::ThreadPoolBuilder;
 use std::collections::{HashMap, HashSet};
-use std::fs::{create_dir_all, OpenOptions};
+use std::fs::{OpenOptions, create_dir_all};
 use std::path::{Path, PathBuf};
 use std::sync::mpsc::Sender;
 use std::sync::{Arc, Mutex};
@@ -39,6 +39,7 @@ use super::drop_data::DropData;
 static RETRY_COUNT: usize = 3;
 
 const TARGET_BUCKET_SIZE: usize = 63 * 1000 * 1000;
+const MAX_FILES_PER_BUCKET: usize = (1024 / 4) - 1;
 
 pub struct GameDownloadAgent {
     pub id: String,
@@ -83,6 +84,8 @@ impl GameDownloadAgent {
         let stored_manifest =
             DropData::generate(id.clone(), version.clone(), data_base_dir_path.clone());
 
+        let context_lock = stored_manifest.contexts.lock().unwrap().clone();
+
         let result = Self {
             id,
             version,
@@ -105,7 +108,14 @@ impl GameDownloadAgent {
             .as_ref()
             .unwrap()
             .values()
-            .map(|e| e.lengths.iter().sum::<usize>())
+            .map(|e| {
+                e.lengths
+                    .iter()
+                    .enumerate()
+                    .filter(|(i, _)| *context_lock.get(&e.checksums[*i]).unwrap_or(&false))
+                    .map(|(_, v)| v)
+                    .sum::<usize>()
+            })
             .sum::<usize>() as u64;
 
         let available_space = get_disk_available(data_base_dir_path)? as u64;
@@ -298,7 +308,8 @@ impl GameDownloadAgent {
                         drops: vec![],
                     });
 
-                if *current_bucket_size + length >= TARGET_BUCKET_SIZE
+                if (*current_bucket_size + length >= TARGET_BUCKET_SIZE
+                    || current_bucket.drops.len() >= MAX_FILES_PER_BUCKET)
                     && !current_bucket.drops.is_empty()
                 {
                     // Move current bucket into list and make a new one
@@ -367,7 +378,8 @@ impl GameDownloadAgent {
             .iter()
             .map(|e| &e.version)
             .collect::<HashSet<_>>()
-            .into_iter().cloned()
+            .into_iter()
+            .cloned()
             .collect::<Vec<String>>();
 
         info!("downloading across these versions: {versions:?}");
@@ -460,6 +472,7 @@ impl GameDownloadAgent {
                                     ApplicationDownloadError::Communication(_)
                                         | ApplicationDownloadError::Checksum
                                         | ApplicationDownloadError::Lock
+                                        | ApplicationDownloadError::IoError(_)
                                 );
 
                                 if i == RETRY_COUNT - 1 || !retry {
@@ -626,8 +639,17 @@ impl Downloadable for GameDownloadAgent {
         }
     }
 
-    fn on_initialised(&self, _app_handle: &tauri::AppHandle) {
+    fn on_queued(&self, app_handle: &tauri::AppHandle) {
         *self.status.lock().unwrap() = DownloadStatus::Queued;
+        let mut db_lock = borrow_db_mut_checked();
+        let status = ApplicationTransientStatus::Queued {
+            version_name: self.version.clone(),
+        };
+        db_lock
+            .applications
+            .transient_statuses
+            .insert(self.metadata(), status.clone());
+        push_game_update(app_handle, &self.id, None, (None, Some(status)));
     }
 
     fn on_error(&self, app_handle: &tauri::AppHandle, error: &ApplicationDownloadError) {
@@ -636,7 +658,7 @@ impl Downloadable for GameDownloadAgent {
             .emit("download_error", error.to_string())
             .unwrap();
 
-        error!("error while managing download: {error}");
+        error!("error while managing download: {error:?}");
 
         let mut handle = borrow_db_mut_checked();
         handle
@@ -662,15 +684,8 @@ impl Downloadable for GameDownloadAgent {
     }
 
     fn on_cancelled(&self, app_handle: &tauri::AppHandle) {
+        info!("cancelled {}", self.id);
         self.cancel(app_handle);
-        /*
-           on_game_incomplete(
-               &self.metadata(),
-               self.dropdata.base_path.to_string_lossy().to_string(),
-               app_handle,
-           )
-           .unwrap();
-        */
     }
 
     fn status(&self) -> DownloadStatus {

src-tauri/src/games/downloads/download_logic.rs

@@ -9,7 +9,7 @@ use crate::games::downloads::manifest::{ChunkBody, DownloadBucket, DownloadConte
 use crate::remote::auth::generate_authorization_header;
 use crate::remote::requests::generate_url;
 use crate::remote::utils::DROP_CLIENT_SYNC;
-use log::{info, warn};
+use log::{debug, info, warn};
 use md5::{Context, Digest};
 use reqwest::blocking::Response;
 
@@ -18,6 +18,7 @@ use std::io::Read;
 #[cfg(unix)]
 use std::os::unix::fs::PermissionsExt;
 use std::sync::Arc;
+use std::time::Instant;
 use std::{
     fs::{File, OpenOptions},
     io::{self, BufWriter, Seek, SeekFrom, Write},
@@ -25,6 +26,7 @@ use std::{
 };
 
 static MAX_PACKET_LENGTH: usize = 4096 * 4;
+static BUMP_SIZE: usize = 4096 * 16;
 
 pub struct DropWriter<W: Write> {
     hasher: Context,
@@ -79,6 +81,8 @@ pub struct DropDownloadPipeline<'a, R: Read, W: Write> {
     pub drops: Vec<DownloadDrop>,
     pub destination: Vec<DropWriter<W>>,
     pub control_flag: &'a DownloadThreadControl,
+    #[allow(dead_code)]
+    progress: ProgressHandle,
 }
 
 impl<'a> DropDownloadPipeline<'a, Response, File> {
@@ -96,6 +100,7 @@ impl<'a> DropDownloadPipeline<'a, Response, File> {
                 .try_collect()?,
             drops,
             control_flag,
+            progress,
         })
     }
 
@@ -111,13 +116,24 @@ impl<'a> DropDownloadPipeline<'a, Response, File> {
             if drop.start != 0 {
                 destination.seek(SeekFrom::Start(drop.start.try_into().unwrap()))?;
             }
+            let mut last_bump = 0;
             loop {
                 let size = MAX_PACKET_LENGTH.min(remaining);
-                self.source.read_exact(&mut copy_buffer[0..size])?;
+                let size = self.source.read(&mut copy_buffer[0..size]).inspect_err(|_| {
+                    info!("got error from {}", drop.filename);
+                })?;
                 remaining -= size;
+                last_bump += size;
 
                 destination.write_all(&copy_buffer[0..size])?;
 
+                if last_bump > BUMP_SIZE {
+                    last_bump -= BUMP_SIZE;
+                    if self.control_flag.get() == DownloadThreadControlFlag::Stop {
+                        return Ok(false);
+                    }
+                }
+
                 if remaining == 0 {
                     break;
                 };
@@ -131,6 +147,13 @@ impl<'a> DropDownloadPipeline<'a, Response, File> {
         Ok(true)
     }
 
+    #[allow(dead_code)]
+    fn debug_skip_checksum(self) {
+        self.destination
+            .into_iter()
+            .for_each(|mut e| e.flush().unwrap());
+    }
+
     fn finish(self) -> Result<Vec<Digest>, io::Error> {
         let checksums = self
             .destination
@@ -153,6 +176,8 @@ pub fn download_game_bucket(
         return Ok(false);
     }
 
+    let start = Instant::now();
+
     let header = generate_authorization_header();
 
     let url = generate_url(&["/api/v2/client/chunk"], &[])
@@ -195,9 +220,7 @@ pub fn download_game_bucket(
     for (i, raw_length) in lengths.split(",").enumerate() {
         let length = raw_length.parse::<usize>().unwrap_or(0);
         let Some(drop) = bucket.drops.get(i) else {
-            warn!(
-                "invalid number of Content-Lengths recieved: {i}, {lengths}"
-            );
+            warn!("invalid number of Content-Lengths recieved: {i}, {lengths}");
             return Err(ApplicationDownloadError::DownloadError);
         };
         if drop.length != length {
@@ -209,6 +232,10 @@ pub fn download_game_bucket(
         }
     }
 
+    let timestep = start.elapsed().as_millis();
+
+    debug!("took {}ms to start downloading", timestep);
+
     let mut pipeline =
         DropDownloadPipeline::new(response, bucket.drops.clone(), control_flag, progress)
             .map_err(|e| ApplicationDownloadError::IoError(Arc::new(e)))?;