chore: Mostly finished s3 config

cli/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2024"
 
 [dependencies]
 anyhow = "1.0.100"
-aws-sdk-s3 = "1.120.0"
+async-trait = "0.1.89"
 chrono = "0.4.43"
 clap = { version = "4.5.54", features = ["derive"] }
 console = "0.16.2"
@@ -14,9 +14,12 @@ droplet-rs = { git = "https://github.com/Drop-OSS/droplet-rs.git", version = "0.
 fern = { version = "0.7.1", features = ["colored"] }
 indicatif = "0.18.3"
 log = "0.4.29"
+rand = "0.9.2"
 reqwest = { version = "0.13.1", features = ["json"] }
+rust-s3 = "0.37.1"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.148"
 tokio = { version = "1.48.0", features = ["macros"] }
+tokio-util = "0.7.18"
 url = "2.5.8"
 webbrowser = "1.0.6"

cli/src/cli.rs

@@ -28,9 +28,9 @@ pub enum Commands {
 }
 #[derive(Args)]
 pub struct UploadInfo {
-    /// Sets
+    /// Identifies the specific upload style that will be used for the set depot
     pub upload_style: UploadStyle,
-    /// Path of new version
+    /// Relative path to new version files
     #[arg(short, long)]
     pub path: PathBuf,
     /// ID of game to attach to
@@ -44,5 +44,4 @@ pub struct UploadInfo {
 #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
 pub enum UploadStyle {
     S3,
-    Nginx,
 }

cli/src/commands/upload/interface.rs

@@ -2,33 +2,39 @@ use std::path::Path;
 
 use crate::{
     cli::UploadInfo,
-    commands::upload::{uploadable::Uploadable, void::VoidUploadable},
+    commands::upload::{s3::S3, uploadable::Uploadable},
+    config::Config,
     manifest::generate_manifest,
 };
 use log::info;
 
-pub async fn upload(info: &UploadInfo) -> anyhow::Result<()> {
+pub async fn upload(info: &UploadInfo, config: Config) -> anyhow::Result<()> {
     let game_id = &info.game_id;
     let path = &info.path;
     let version_id = &info.version_id;
 
     let manifest = generate_manifest(&Path::new(path)).await?;
     let mut uploader: Box<dyn Uploadable> = match info.upload_style {
-        crate::cli::UploadStyle::S3 => Box::new(VoidUploadable::new()),
-        crate::cli::UploadStyle::Nginx => Box::new(VoidUploadable::new()),
+        crate::cli::UploadStyle::S3 => Box::new(S3::new(
+            config
+                .get_active_s3()
+                .ok_or(anyhow::Error::msg("Could not get active S3 value"))?,
+        )?),
     };
     info!("Uploading chunks");
     for (id, data) in &manifest.chunks {
         info!("Uploading chunk id {id}");
-        uploader.upload_chunk(game_id, version_id, id, data)?;
+        uploader.upload_chunk(game_id, version_id, id, data).await?;
     }
     info!("Finished uploading chunks");
 
     info!("Uploading manifest");
-    uploader.upload_manifest(manifest, game_id, version_id)?;
+    uploader
+        .upload_manifest(manifest, game_id, version_id)
+        .await?;
 
     info!("Uploading speedtest");
-    uploader.upload_speedtest(game_id, version_id)?;
+    uploader.upload_speedtest().await?;
 
     Ok(())
 }

cli/src/commands/upload/mod.rs

@@ -1,4 +1,5 @@
 pub mod interface;
 pub mod s3;
 pub mod uploadable;
-pub mod void;
+pub mod void;
+pub mod speedtest;

cli/src/commands/upload/s3.rs

@@ -1,10 +1,29 @@
+use crate::{
+    commands::upload::{
+        speedtest::{SPEEDTEST_PATH, Speedtest},
+        uploadable::Uploadable,
+    },
+    config::S3Config,
+};
+use async_trait::async_trait;
 use droplet_rs::manifest::{ChunkData, Manifest};
+use s3::{Bucket, creds::Credentials};
+use serde_json::json;
+use std::{ops::Deref, path::PathBuf};
 
-use crate::commands::upload::uploadable::Uploadable;
-
-pub type S3 = aws_sdk_s3::Client;
+pub struct S3 {
+    bucket: s3::Bucket,
+}
+impl S3 {
+    pub fn new(config: &S3Config) -> anyhow::Result<Self> {
+        Ok(Self {
+            bucket: config.generate_bucket()?,
+        })
+    }
+}
+#[async_trait]
 impl Uploadable for S3 {
-    fn upload_chunk(
+    async fn upload_chunk(
         &mut self,
         id: &String,
         version: &String,
@@ -13,16 +32,39 @@ impl Uploadable for S3 {
     ) -> anyhow::Result<()> {
         todo!()
     }
-    fn upload_speedtest(&mut self, game_id: &String, version_id: &String) -> anyhow::Result<()> {
-        todo!()
+    async fn upload_speedtest(&mut self) -> anyhow::Result<()> {
+        if self.object_exists(SPEEDTEST_PATH).await? {
+            return Ok(());
+        }
+        let mut speedtest = Speedtest::new();
+        self.put_object_stream(&mut speedtest, SPEEDTEST_PATH)
+            .await?;
+        Ok(())
     }
 
-    fn upload_manifest(
+    async fn upload_manifest(
         &mut self,
         manifest: Manifest,
         game_id: &String,
         version_id: &String,
     ) -> anyhow::Result<()> {
-        todo!()
+        self.put_object(
+            PathBuf::from(game_id)
+                .join(version_id)
+                .join("manifest.json")
+                .to_string_lossy()
+                .to_string(),
+            json!(manifest).to_string().as_bytes(),
+        )
+        .await?;
+        Ok(())
+    }
+}
+
+impl Deref for S3 {
+    type Target = Bucket;
+
+    fn deref(&self) -> &Self::Target {
+        &self.bucket
     }
 }

cli/src/commands/upload/speedtest.rs

@@ -0,0 +1,33 @@
+use rand::{RngCore, SeedableRng, rng, rngs::StdRng};
+use tokio::io::AsyncRead;
+
+#[derive(Clone, Debug)]
+pub struct Speedtest {
+    core: rand::rngs::StdRng,
+    to_write: usize,
+}
+pub const SPEEDTEST_BYTES: usize = 64 * 1024 * 1024;
+pub const SPEEDTEST_PATH: &str = "speedtest";
+
+impl AsyncRead for Speedtest {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        _cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        let mut s = self;
+        let to_write = buf.remaining().min(s.to_write);
+        s.to_write = s.to_write.saturating_sub(to_write);
+        let fill_slice = buf.initialize_unfilled_to(to_write);
+        s.core.fill_bytes(fill_slice);
+        std::task::Poll::Ready(Ok(()))
+    }
+}
+impl Speedtest {
+    pub fn new() -> Self {
+        Self {
+            core: StdRng::from_rng(&mut rng()),
+            to_write: SPEEDTEST_BYTES,
+        }
+    }
+}

cli/src/commands/upload/uploadable.rs

@@ -1,20 +1,15 @@
+use async_trait::async_trait;
 use droplet_rs::manifest::{ChunkData, Manifest};
 
+#[async_trait]
 pub trait Uploadable {
-    fn upload_chunk(
+    async fn upload_chunk(
         &mut self,
         id: &String,
         version: &String,
         chunk_id: &String,
         chunk: &ChunkData,
     ) -> anyhow::Result<()>;
-    fn upload_speedtest(&mut self, game_id: &String, version_id: &String) -> anyhow::Result<()>;
-    fn upload_manifest(&mut self, manifest: Manifest, game_id: &String, version_id: &String) -> anyhow::Result<()>;
-}
-pub enum UploadableConfig {
-    S3 {
-        api_secret: String,
-        api_key_identifier: String,
-        region: String,
-    },
+    async fn upload_speedtest(&mut self) -> anyhow::Result<()>;
+    async fn upload_manifest(&mut self, manifest: Manifest, game_id: &String, version_id: &String) -> anyhow::Result<()>;
 }

cli/src/commands/upload/void.rs

@@ -1,11 +1,13 @@
+use async_trait::async_trait;
 use droplet_rs::manifest::{ChunkData, Manifest};
 use log::warn;
 
 use crate::commands::upload::uploadable::Uploadable;
 
 pub struct VoidUploadable;
+#[async_trait]
 impl Uploadable for VoidUploadable {
-    fn upload_chunk(
+    async fn upload_chunk(
         &mut self,
         _id: &String,
         _version: &String,
@@ -16,12 +18,12 @@ impl Uploadable for VoidUploadable {
         Ok(())
     }
 
-    fn upload_speedtest(&mut self, _game_id: &String, _version_id: &String) -> anyhow::Result<()> {
+    async fn upload_speedtest(&mut self) -> anyhow::Result<()> {
         warn!("Uploading speedtest to VoidUploader");
         Ok(())
     }
 
-    fn upload_manifest(
+    async fn upload_manifest(
         &mut self,
         _manifest: Manifest,
         _game_id: &String,

cli/src/config.rs

@@ -0,0 +1,67 @@
+use std::str::FromStr;
+
+use s3::{Bucket, Region, creds::Credentials};
+
+pub struct Config {
+    items: Vec<ConfigItem>,
+    active_s3: Option<String>,
+}
+pub struct ConfigItem {
+    name: String,
+    config_option: ConfigOption,
+}
+enum ConfigOption {
+    S3(S3Config),
+}
+pub struct S3Config {
+    secret_key: String,
+    key_id: String,
+    region: String,
+    bucket_name: String,
+    endpoint: Option<String>,
+}
+impl Config {
+    pub fn new() -> Self {
+        Self {
+            items: Vec::new(),
+            active_s3: None,
+        }
+    }
+    pub fn get_active_s3(&self) -> Option<&S3Config> {
+        if let Some(active_s3) = &self.active_s3 {
+            self.items
+                .iter()
+                .filter_map(|item| {
+                    if item.name == *active_s3 {
+                        match &item.config_option {
+                            ConfigOption::S3(s3_config) => Some(s3_config),
+                        }
+                    } else {
+                        None
+                    }
+                })
+                .next()
+        } else {
+            None
+        }
+    }
+}
+impl S3Config {
+    pub fn generate_bucket(&self) -> anyhow::Result<s3::Bucket> {
+        let credentials =
+            Credentials::new(Some(&self.key_id), Some(&self.secret_key), None, None, None)?;
+
+        let region = if let Some(endpoint) = &self.endpoint {
+            Region::Custom {
+                region: self.region.clone(),
+                endpoint: endpoint.clone(),
+            }
+        } else {
+            Region::from_str(&self.region)?
+        };
+
+        let bucket = Bucket::new(&self.bucket_name, region, credentials)?;
+
+        Ok(*bucket)
+    }
+}

cli/src/main.rs

@@ -1,6 +1,6 @@
 use crate::{
     cli::{Cli, Commands},
-    commands::{configure::interactive_configure, upload},
+    commands::{configure::interactive_configure, upload}, config::Config,
 };
 use clap::Parser;
 use fern::colors::{Color, ColoredLevelConfig};
@@ -8,15 +8,16 @@ use log::LevelFilter;
 use std::env;
 use std::fs;
 use std::io;
-
 mod cli;
 mod commands;
+mod config;
 mod manifest;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     configure_logging()?;
     let cli = Cli::parse();
+    let config = Config::new();
     match &cli.command {
         Commands::Configure { url, token } => {
             if let Some(token) = token {
@@ -25,7 +26,7 @@ async fn main() -> anyhow::Result<()> {
             }
         }
         Commands::Upload(info) => {
-            upload::interface::upload(info).await?;
+            upload::interface::upload(info, config).await?;
         }
     };
 
@@ -34,7 +35,6 @@ async fn main() -> anyhow::Result<()> {
 
 pub fn configure_logging() -> anyhow::Result<()> {
     let log_level = env::var("RUST_LOG")
-        .or_else(|_| env::var("LOG_LEVEL"))
         .unwrap_or_else(|_| "info".to_string())
         .parse::<LevelFilter>()?;
 
@@ -45,27 +45,24 @@ pub fn configure_logging() -> anyhow::Result<()> {
     let colors = ColoredLevelConfig::new()
         .error(Color::Red)
         .warn(Color::Yellow)
-        .info(Color::Green)
-        .debug(Color::Blue)
+        .info(Color::Blue)
+        .debug(Color::Green)
         .trace(Color::Magenta);
 
     fern::Dispatch::new()
         .chain(
-            // Console output with colors and formatting
             fern::Dispatch::new()
                 .format(move |out, message, record| {
                     out.finish(format_args!(
-                        "[{}] {} {} - {}",
-                        chrono::Local::now().format("%Y-%m-%d %H:%M:%S%.3f"),
+                        "[{}] {}: {}",
+                        chrono::Local::now().format("%H:%M:%S%.3f"),
                         colors.color(record.level()),
-                        record.target(),
                         message
                     ))
                 })
                 .chain(io::stdout()),
         )
         .chain(
-            // File output without colors and with formatting
             fern::Dispatch::new()
                 .format(|out, message, record| {
                     out.finish(format_args!(

cli/src/manifest.rs

@@ -1,6 +1,4 @@
 use std::{
-    fs,
-    io::{Read, Seek},
     path::Path,
 };