feat: libarchive backend

libraries/droplet/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "libarchive-rust"]
+	path = libarchive-rust
+	url = git@github.com:Drop-OSS/libarchive-rust.git

libraries/droplet/Cargo.lock

@@ -2,6 +2,21 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "addr2line"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
 [[package]]
 name = "anyhow"
 version = "1.0.100"
@@ -48,8 +63,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
- "synstructure",
+ "synstructure 0.13.1",
 ]
 
 [[package]]
@@ -60,8 +75,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
- "synstructure",
+ "synstructure 0.13.1",
 ]
 
 [[package]]
@@ -72,7 +87,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -83,7 +98,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -92,6 +107,21 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
+[[package]]
+name = "backtrace"
+version = "0.3.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
+dependencies = [
+ "addr2line",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+ "windows-link",
+]
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -214,7 +244,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -228,11 +258,13 @@ dependencies = [
  "getrandom 0.3.4",
  "hex",
  "humansize",
+ "libarchive",
  "rcgen",
  "ring",
  "serde",
  "serde_json",
  "sha2",
+ "speedometer",
  "time",
  "tokio",
  "uuid",
@@ -245,6 +277,28 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
 
+[[package]]
+name = "failure"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
+dependencies = [
+ "backtrace",
+ "failure_derive",
+]
+
+[[package]]
+name = "failure_derive"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "synstructure 0.12.6",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.31"
@@ -301,7 +355,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -367,6 +421,12 @@ dependencies = [
  "wasip2",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.32.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -405,10 +465,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
-name = "libc"
+name = "libarchive"
-version = "0.2.171"
+version = "0.1.1"
+dependencies = [
+ "libarchive3-sys",
+ "libc",
+]
+
+[[package]]
+name = "libarchive3-sys"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
+checksum = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7"
+dependencies = [
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.182"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
 
 [[package]]
 name = "libm"
@@ -428,6 +506,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+]
+
 [[package]]
 name = "mio"
 version = "1.1.0"
@@ -483,6 +570,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "object"
+version = "0.37.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "oid-registry"
 version = "0.7.1"
@@ -529,6 +625,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -587,6 +689,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
+
 [[package]]
 name = "rusticata-macros"
 version = "4.1.0"
@@ -641,7 +749,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -690,16 +798,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
-name = "syn"
+name = "speedometer"
-version = "2.0.100"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
+checksum = "2789736092fa21b44baf8590acb4b360cb91f0f597bd6c1f1741ca9644c95c1e"
+dependencies = [
+ "failure",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "synstructure"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "unicode-xid",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.13.1"
@@ -708,7 +848,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -737,7 +877,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -748,7 +888,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -805,7 +945,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -820,6 +960,12 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -890,7 +1036,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
  "wasm-bindgen-shared",
 ]
 

libraries/droplet/Cargo.toml

@@ -11,7 +11,15 @@ hex = "0.4.3"
 time = "0.3.41"
 ring = "0.17.14"
 dyn-clone = "1.0.20"
-tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "rt", "rt-multi-thread", "macros", "sync"] }
+tokio = { version = "^1.48.0", features = [
+    "process",
+    "fs",
+    "io-util",
+    "rt",
+    "rt-multi-thread",
+    "macros",
+    "sync",
+] }
 anyhow = "1.0.100"
 async-trait = "0.1.89"
 serde = { version = "1.0.228", features = ["derive"] }
@@ -21,6 +29,8 @@ uuid = { version = "1.19.0", features = ["v4"] }
 sha2 = "0.10.9"
 futures = "0.3.31"
 getrandom = "0.3.4"
+libarchive = { version = "*", path = "./libarchive-rust" }
+speedometer = "0.2.2"
 
 [dependencies.x509-parser]
 version = "0.17.0"

libraries/droplet/README.md

@@ -1,2 +1,8 @@
 # droplet-rs
-Core rust functionality for droplet
+
+A Rust-based library for utilities and functionality required both by the server, client, and other tools.
+
+## manifest generation
+`droplet-rs` contains the manifest generation code, held in `manifest.rs`.
+
+It also includes the version backends to provide a unified read-write interface to files.

libraries/droplet/src/lib.rs

@@ -5,6 +5,9 @@ pub mod file_utils;
 pub mod ssl;
 pub mod versions;
 pub mod manifest;
+pub mod vm;
+
+extern crate libarchive;
 
 #[cfg(test)]
 pub mod tests;

libraries/droplet/src/main.rs

@@ -22,6 +22,8 @@ pub async fn main() {
     .await
     .unwrap();
 
+    return;
+
     // Sanity checks
     for (_, chunk_data) in manifest.chunks {
         for file in chunk_data.files {

libraries/droplet/src/manifest.rs

@@ -1,11 +1,15 @@
 use std::{
-    collections::HashMap, future::Future, path::Path, sync::{
+    collections::HashMap,
-        Arc, atomic::{AtomicU64, Ordering}
+    future::Future,
-    }
+    mem,
+    path::Path,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
 };
 
-use anyhow::{anyhow, Error};
+use anyhow::anyhow;
-use futures::{stream::FuturesUnordered, StreamExt};
 use hex::ToHex as _;
 use humansize::{format_size, BINARY};
 use serde::{Deserialize, Serialize};
@@ -14,6 +18,7 @@ use tokio::{
     io::AsyncReadExt as _,
     join,
     sync::{Mutex, Semaphore},
+    task::JoinSet,
 };
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -39,18 +44,21 @@ pub struct Manifest {
     pub key: [u8; 16],
 }
 
-const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
+const CHUNK_SIZE: u64 = 1024 * 1024; //* 64;
 const MAX_FILE_COUNT: usize = 512;
 
-use crate::versions::{create_backend_constructor, types::VersionFile};
+use crate::versions::{
+    create_backend_constructor,
+    types::{VersionBackend, VersionFile},
+};
 
 pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
     dir: &Path,
     progress_sfn: V,
     log_sfn: T,
-    reader_semaphore: Option<&Semaphore>,
+    reader_semaphore: Option<Arc<Semaphore>>,
 ) -> anyhow::Result<Manifest> {
-    let mut backend =
+    let backend =
         create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?;
 
     let required_single_file = backend.require_whole_files();
@@ -137,18 +145,19 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
     let manifest: Arc<Mutex<HashMap<String, ChunkData>>> = Arc::new(Mutex::new(HashMap::new()));
     let total_manifest_length = Arc::new(AtomicU64::new(0));
 
-    let backend = Arc::new(Mutex::new(backend));
+    // SAFETY: we .join_all() the futures using this
+    let backend: &'static Box<dyn VersionBackend + Send + Sync> =
+        unsafe { mem::transmute(&backend) };
 
-    let futures: FuturesUnordered<impl Future<Output = Result<(), Error>>> =
+    let mut futures: JoinSet<Result<(), anyhow::Error>> = JoinSet::new();
-        FuturesUnordered::new();
     let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16);
     let chunks_length = chunks.len();
     for (index, chunk) in chunks.into_iter().enumerate() {
         let send_log = send_log.clone();
-        let backend = backend.clone();
         let total_manifest_length = total_manifest_length.clone();
         let manifest = manifest.clone();
-        futures.push(async move {
+        let reader_semaphore = reader_semaphore.clone();
+        futures.spawn(async move {
             let mut read_buf = vec![0; 1024 * 1024 * 64];
 
             let uuid = uuid::Uuid::new_v4().to_string();
@@ -164,6 +173,8 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
 
             let mut chunk_length = 0;
 
+            println!("starting chunk {}", index);
+
             for (file, start, length) in chunk {
                 let permit = if let Some(reader_semaphore) = &reader_semaphore {
                     Some(reader_semaphore.acquire().await?)
@@ -171,18 +182,20 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
                     None
                 };
 
-                let mut reader = {
+                let mut reader = backend.reader(&file, start, start + length).await?;
-                    let mut backend_lock = backend.lock().await;
+
-                    let reader = backend_lock.reader(&file, start, start + length).await?;
+                let mut total = 0;
-                    reader
-                };
 
                 loop {
                     let amount = reader.read(&mut read_buf).await?;
                     if amount == 0 {
                         break;
                     }
+                    total += amount;
                     hasher.update(&read_buf[0..amount]);
+                    if total as u64 > length {
+                        panic!("read too much: target {}, got {}", length, total);
+                    }
                 }
 
                 chunk_length += length;
@@ -230,7 +243,7 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
                 progress_sfn((current_progress / total_progress) * 100.0f32)
             }
         },
-        futures.collect::<Vec<Result<(), Error>>>()
+        futures.join_all()
     );
 
     let manifest = manifest.lock().await;

libraries/droplet/src/versions/archive_backend.rs

@@ -0,0 +1,115 @@
+use std::{path::PathBuf, task::Poll};
+
+use anyhow::{anyhow};
+use async_trait::async_trait;
+use libarchive::{
+    archive::{Entry, FileType, ReadCompression, ReadFormat},
+    reader::{Builder, FileReader, Reader},
+};
+use tokio::io::AsyncRead;
+
+use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile};
+
+pub struct ZipVersionBackend {
+    path: PathBuf,
+}
+impl ZipVersionBackend {
+    pub fn new(path: PathBuf) -> anyhow::Result<Self> {
+        Ok(Self { path })
+    }
+
+    fn open_archive(&self) -> Result<FileReader, anyhow::Error> {
+        let mut archive = Builder::new();
+        archive.support_format(ReadFormat::All)?;
+        archive.support_compression(ReadCompression::All)?;
+        let archive = archive.open_file(&self.path)?;
+
+        Ok(archive)
+    }
+}
+
+struct ArchiveReader {
+    archive: FileReader,
+}
+
+impl AsyncRead for ArchiveReader {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        _cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        let block = match self.archive.read_block() {
+            Ok(v) => v,
+            Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))),
+        };
+
+        let block = match block {
+            Some(v) => v,
+            None => return Poll::Ready(Ok(())),
+        };
+
+        buf.put_slice(block);
+        return Poll::Ready(Ok(()));
+    }
+}
+
+#[async_trait]
+impl VersionBackend for ZipVersionBackend {
+    async fn list_files(&self) -> anyhow::Result<Vec<VersionFile>> {
+        let mut archive = self.open_archive()?;
+        let mut results = Vec::new();
+
+        while let Some(header) = archive.next_header() {
+            match header.filetype() {
+                FileType::RegularFile => (),
+                _ => {
+                    continue;
+                }
+            }
+            results.push(VersionFile {
+                relative_filename: header.pathname().to_string(),
+                permission: 0o744,
+                size: header.size().try_into()?,
+            });
+            println!("storing file {}", header.pathname());
+        }
+
+        Ok(results)
+    }
+
+    async fn reader(
+        &self,
+        file: &VersionFile,
+        _start: u64,
+        _end: u64,
+    ) -> anyhow::Result<Box<dyn MinimumFileObject>> {
+        let mut archive = self.open_archive()?;
+
+        // Find entry in archive
+        loop {
+            let entry = match archive.next_header() {
+                Some(v) => v,
+                None => return Err(anyhow!("entry not found:{}", file.relative_filename)),
+            };
+            if entry.pathname() == file.relative_filename {
+                break;
+            }
+        }
+
+        Ok(Box::new(ArchiveReader { archive }))
+    }
+
+    async fn peek_file(&self, sub_path: String) -> anyhow::Result<VersionFile> {
+        let files = self.list_files().await?;
+        let file = files
+            .iter()
+            .find(|v| v.relative_filename == sub_path)
+            .expect("file not found");
+
+        Ok(file.clone())
+    }
+
+    fn require_whole_files(&self) -> bool {
+        true
+    }
+}

libraries/droplet/src/versions/backends.rs

@@ -1,230 +0,0 @@
-#[cfg(unix)]
-use std::os::unix::fs::PermissionsExt;
-use std::{
-    fs::{metadata, read_dir},
-    io::SeekFrom,
-    path::{Path, PathBuf},
-    process::Stdio,
-    sync::LazyLock,
-};
-
-use anyhow::{anyhow, Result};
-use async_trait::async_trait;
-use tokio::{
-    fs::File,
-    io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader},
-    process::Command,
-};
-
-use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile};
-
-pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) -> Result<()> {
-    if metadata(path)?.is_dir() {
-        let paths = read_dir(path)?;
-        for path_result in paths {
-            let full_path = path_result?.path();
-            if metadata(&full_path)?.is_dir() {
-                _list_files(vec, &full_path)?;
-            } else {
-                vec.push(full_path);
-            }
-        }
-    };
-
-    Ok(())
-}
-
-#[derive(Clone)]
-pub struct PathVersionBackend {
-    pub base_dir: PathBuf,
-}
-
-#[async_trait]
-impl VersionBackend for PathVersionBackend {
-    async fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
-        let mut vec = Vec::new();
-        _list_files(&mut vec, &self.base_dir)?;
-
-        let mut results = Vec::new();
-
-        for pathbuf in vec.iter() {
-            let relative = pathbuf.strip_prefix(self.base_dir.clone())?;
-
-            results.push(
-                self.peek_file(
-                    relative
-                        .to_str()
-                        .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))?
-                        .to_owned(),
-                )
-                .await?,
-            );
-        }
-
-        Ok(results)
-    }
-
-    async fn reader(
-        &mut self,
-        file: &VersionFile,
-        start: u64,
-        end: u64,
-    ) -> anyhow::Result<Box<dyn MinimumFileObject>> {
-        let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?;
-
-        if start != 0 {
-            file.seek(SeekFrom::Start(start)).await?;
-        }
-
-        if end != 0 {
-            return Ok(Box::new(file.take(end - start)));
-        }
-
-        Ok(Box::new(file))
-    }
-
-    async fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
-        let pathbuf = self.base_dir.join(sub_path.clone());
-        if !pathbuf.exists() {
-            return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy()));
-        };
-
-        let file = File::open(pathbuf.clone()).await?;
-        let metadata = file.try_clone().await?.metadata().await?;
-        let permission_object = metadata.permissions();
-        let permissions = {
-            let perm: u32;
-            #[cfg(target_family = "unix")]
-            {
-                perm = permission_object.mode();
-            }
-            #[cfg(not(target_family = "unix"))]
-            {
-                perm = 0
-            }
-            perm
-        };
-
-        Ok(VersionFile {
-            relative_filename: sub_path,
-            permission: permissions,
-            size: metadata.len(),
-        })
-    }
-
-    fn require_whole_files(&self) -> bool {
-        false
-    }
-}
-
-pub static SEVEN_ZIP_INSTALLED: LazyLock<bool> =
-    LazyLock::new(|| std::process::Command::new("7z").output().is_ok());
-// https://7-zip.opensource.jp/chm/general/formats.htm
-// Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above
-pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [
-    "7z", "bz2", "bzip2", "tbz2", "tbz", "gz", "gzip", "tgz", "tar", "wim", "swm", "esd", "xz",
-    "txz", "zip", "zipx", "jar", "xpi", "odt", "ods", "docx", "xlsx", "epub", "apm", "ar", "a",
-    "deb", "lib", "arj", "cab", "chm", "chw", "chi", "chq", "msi", "msp", "doc", "xls", "ppt",
-    "cpio", "cramfs", "dmg", "ext", "ext2", "ext3", "ext4", "img", "fat", "img", "hfs", "hfsx",
-    "hxs", "hxr", "hxq", "hxw", "lit", "ihex", "iso", "img", "lzh", "lha", "lzma", "mbr", "mslz",
-    "mub", "nsis", "ntfs", "img", "mbr", "rar", "r00", "rpm", "ppmd", "qcow", "qcow2", "qcow2c",
-    "squashfs", "udf", "iso", "img", "scap", "uefif", "vdi", "vhd", "vmdk", "xar", "pkg", "z",
-    "taz",
-];
-
-#[derive(Clone)]
-pub struct ZipVersionBackend {
-    path: String,
-}
-impl ZipVersionBackend {
-    pub fn new(path: PathBuf) -> anyhow::Result<Self> {
-        Ok(Self {
-            path: path.to_str().expect("invalid utf path").to_owned(),
-        })
-    }
-}
-
-#[async_trait]
-impl VersionBackend for ZipVersionBackend {
-    async fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
-        let mut list_command = Command::new("7z");
-        list_command.args(vec!["l", &self.path]);
-        let result = list_command.output().await?;
-        if !result.status.success() {
-            return Err(anyhow!(
-                "failed to list files: code {:?}",
-                result.status.code()
-            ));
-        }
-        let raw_result = String::from_utf8(result.stdout)?;
-        let mut lines = raw_result.split("\n").skip(11);
-
-        let mut column_lines = lines
-            .find(|v| v.starts_with('-'))
-            .ok_or(anyhow!("invalid 7z output"))?
-            .chars();
-        let file_lines = lines.take_while(|v| !v.starts_with("-"));
-
-        /*
-           Date      Time    Attr         Size   Compressed  Name
-        ------------------- ----- ------------ ------------  ------------------------
-                 */
-        let datetime_pos = 0usize;
-        let attrs_pos = datetime_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
-        let size_pos = attrs_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
-        let compressed_size_pos = size_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
-        let name_pos = compressed_size_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
-
-        let mut results = Vec::new();
-        for file in file_lines {
-            let name = file[name_pos..].trim();
-            let size = file[size_pos..compressed_size_pos].trim();
-
-            let size = str::parse::<u64>(size)?;
-
-            let version_file = VersionFile {
-                relative_filename: name.to_string(),
-                permission: 0o744,
-                size,
-            };
-
-            results.push(version_file);
-        }
-
-        Ok(results)
-    }
-
-    async fn reader(
-        &mut self,
-        file: &VersionFile,
-        _start: u64,
-        _end: u64,
-    ) -> anyhow::Result<Box<dyn MinimumFileObject>> {
-        let mut read_command = Command::new("7z");
-        read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]);
-        let mut output = read_command
-            .stdout(Stdio::piped())
-            .spawn()
-            .expect("failed to spawn 7z");
-        let stdout = output
-            .stdout
-            .take()
-            .ok_or(anyhow!("no stdout on 7zip process"))?;
-        let reader = BufReader::new(stdout);
-        Ok(Box::new(reader))
-    }
-
-    async fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
-        let files = self.list_files().await?;
-        let file = files
-            .iter()
-            .find(|v| v.relative_filename == sub_path)
-            .expect("file not found");
-
-        Ok(file.clone())
-    }
-
-    fn require_whole_files(&self) -> bool {
-        true
-    }
-}

libraries/droplet/src/versions/mod.rs

@@ -1,15 +1,37 @@
-use std::path::Path;
+use std::{
+    fs::{metadata, read_dir},
+    path::{Path, PathBuf},
+};
 
 use anyhow::Result;
 
 use crate::versions::{
-    backends::{
+    archive_backend::ZipVersionBackend, path_backend::PathVersionBackend, types::VersionBackend,
-        PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED, SUPPORTED_FILE_EXTENSIONS,
-    },
-    types::VersionBackend,
 };
 
-pub mod backends;
+pub mod archive_backend;
+pub mod path_backend;
+
+pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) -> Result<()> {
+    if metadata(path)?.is_dir() {
+        let paths = read_dir(path)?;
+        for path_result in paths {
+            let full_path = path_result?.path();
+            if metadata(&full_path)?.is_dir() {
+                _list_files(vec, &full_path)?;
+            } else {
+                vec.push(full_path);
+            }
+        }
+    };
+
+    Ok(())
+}
+
+const SUPPORTED_FILE_EXTENSIONS: [&'static str; 11] = [
+    "tar", "pax", "cpio", "zip", "jar", "ar", "xar", "rar", "rpm", "7z", "iso",
+];
+
 pub mod types;
 pub fn create_backend_constructor<'a>(
     path: &Path,
@@ -26,27 +48,17 @@ pub fn create_backend_constructor<'a>(
         }));
     };
 
-    if *SEVEN_ZIP_INSTALLED {
+    let file_extension = path
-        /*
+        .extension()
-        Slow 7zip integrity test
+        .map(|v| v.to_str())
-        let mut test = Command::new("7z");
+        .flatten()?;
-        test.args(vec!["t", path.to_str().expect("invalid utf path")]);
+
-        let status = test.status().ok()?;
+    if SUPPORTED_FILE_EXTENSIONS
-        if status.code().unwrap_or(1) == 0 {
+        .iter()
-          let buf = path.to_path_buf();
+        .any(|v| *v == file_extension)
-          return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
+    {
-        }
+        let buf = path.to_path_buf();
-         */
+        return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
-        // Fast filename-based test
-        if let Some(extension) = path.extension().and_then(|v| v.to_str()) {
-            let supported = SUPPORTED_FILE_EXTENSIONS
-                .iter()
-                .any(|v| **v == *extension);
-            if supported {
-                let buf = path.to_path_buf();
-                return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
-            }
-        }
     }
 
     None

libraries/droplet/src/versions/path_backend.rs

@@ -0,0 +1,98 @@
+#[cfg(unix)]
+use std::os::unix::fs::PermissionsExt;
+use std::{
+    io::SeekFrom,
+    path::PathBuf,
+};
+
+use anyhow::anyhow;
+use async_trait::async_trait;
+use tokio::{
+    fs::File,
+    io::{AsyncReadExt as _, AsyncSeekExt as _},
+};
+
+#[derive(Clone)]
+pub struct PathVersionBackend {
+    pub base_dir: PathBuf,
+}
+
+use crate::versions::{_list_files, types::{MinimumFileObject, VersionBackend, VersionFile}};
+
+#[async_trait]
+impl VersionBackend for PathVersionBackend {
+    async fn list_files(&self) -> anyhow::Result<Vec<VersionFile>> {
+        let mut vec = Vec::new();
+        _list_files(&mut vec, &self.base_dir)?;
+
+        let mut results = Vec::new();
+
+        for pathbuf in vec.iter() {
+            let relative = pathbuf.strip_prefix(self.base_dir.clone())?;
+
+            results.push(
+                self.peek_file(
+                    relative
+                        .to_str()
+                        .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))?
+                        .to_owned(),
+                )
+                .await?,
+            );
+        }
+
+        Ok(results)
+    }
+
+    async fn reader(
+        &self,
+        file: &VersionFile,
+        start: u64,
+        end: u64,
+    ) -> anyhow::Result<Box<dyn MinimumFileObject>> {
+        let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?;
+
+        if start != 0 {
+            file.seek(SeekFrom::Start(start)).await?;
+        }
+
+        if end != 0 {
+            return Ok(Box::new(file.take(end - start)));
+        }
+
+        Ok(Box::new(file))
+    }
+
+    async fn peek_file(&self, sub_path: String) -> anyhow::Result<VersionFile> {
+        let pathbuf = self.base_dir.join(sub_path.clone());
+        if !pathbuf.exists() {
+            return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy()));
+        };
+
+        let file = File::open(pathbuf.clone()).await?;
+        let metadata = file.try_clone().await?.metadata().await?;
+        let permission_object = metadata.permissions();
+        let permissions = {
+            let perm: u32;
+            #[cfg(target_family = "unix")]
+            {
+                perm = permission_object.mode();
+            }
+            #[cfg(not(target_family = "unix"))]
+            {
+                perm = 0
+            }
+            perm
+        };
+
+        Ok(VersionFile {
+            relative_filename: sub_path,
+            permission: permissions,
+            size: metadata.len(),
+        })
+    }
+
+    fn require_whole_files(&self) -> bool {
+        false
+    }
+}

libraries/droplet/src/versions/types.rs

@@ -1,7 +1,6 @@
 use std::fmt::Debug;
 
 use async_trait::async_trait;
-use dyn_clone::DynClone;
 use tokio::io::AsyncRead;
 
 #[derive(Debug, Clone)]
@@ -16,16 +15,14 @@ impl<T: AsyncRead + Send + Unpin> MinimumFileObject for T {}
 
 
 #[async_trait]
-pub trait VersionBackend: DynClone {
+pub trait VersionBackend {
     fn require_whole_files(&self) -> bool;
-    async fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>>;
+    async fn list_files(&self) -> anyhow::Result<Vec<VersionFile>>;
-    async fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile>;
+    async fn peek_file(&self, sub_path: String) -> anyhow::Result<VersionFile>;
     async fn reader(
-        &mut self,
+        &self,
         file: &VersionFile,
         start: u64,
         end: u64,
     ) -> anyhow::Result<Box<dyn MinimumFileObject>>;
 }
-
-dyn_clone::clone_trait_object!(VersionBackend);