From 2d67febdb1aa96f79036301554a8466d464a338f Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 11:11:46 +1100 Subject: [PATCH] feat: move version backend --- libraries/droplet/Cargo.lock | 32 ++- libraries/droplet/Cargo.toml | 5 +- libraries/droplet/src/lib.rs | 1 + libraries/droplet/src/versions/backends.rs | 246 +++++++++++++++++++++ libraries/droplet/src/versions/mod.rs | 2 + libraries/droplet/src/versions/types.rs | 55 +++++ 6 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 libraries/droplet/src/versions/backends.rs create mode 100644 libraries/droplet/src/versions/mod.rs create mode 100644 libraries/droplet/src/versions/types.rs diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 6d36edbb..22be2c74 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "asn1-rs" version = "0.6.2" @@ -152,17 +158,26 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.7.3" +version = "0.8.0" dependencies = [ + "anyhow", + "dyn-clone", "hex", "rcgen", "ring", "time", "time-macros", + "tokio", "webpki", "x509-parser 0.17.0", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "getrandom" version = "0.2.15" @@ -282,6 +297,12 @@ dependencies = [ "serde", ] +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + [[package]] name = "powerfmt" version = "0.2.0" @@ -468,6 +489,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "pin-project-lite", +] + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index bb08d453..b0d0c7da 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.7.3" +version = "0.8.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -12,6 +12,9 @@ time-macros = "0.2.22" time = "0.3.41" webpki = "0.22.4" ring = "0.17.14" +dyn-clone = "1.0.20" +tokio = "1.48.0" +anyhow = "1.0.100" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index fc5931d1..ec1ad139 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -2,3 +2,4 @@ pub mod file_utils; pub mod ssl; +pub mod versions; diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs new file mode 100644 index 00000000..96fd6ad8 --- /dev/null +++ b/libraries/droplet/src/versions/backends.rs @@ -0,0 +1,246 @@ +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{ + cell::LazyCell, + fs::{self, metadata, File}, + io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink}, + path::{Path, PathBuf}, + process::{Child, ChildStdout, Command, Stdio}, + sync::{Arc, LazyLock}, +}; + +use anyhow::{anyhow, Result}; + +use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; + +pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { + if metadata(path)?.is_dir() { + let paths = fs::read_dir(path)?; + for path_result in paths { + let full_path = path_result?.path(); + if metadata(&full_path)?.is_dir() { + _list_files(vec, &full_path)?; + } else { + vec.push(full_path); + } + } + }; + + Ok(()) +} + +#[derive(Clone)] +pub struct PathVersionBackend { + pub base_dir: PathBuf, +} +impl VersionBackend for PathVersionBackend { + fn list_files(&mut self) -> anyhow::Result> { + let mut vec = Vec::new(); + _list_files(&mut vec, &self.base_dir)?; + + let mut results = Vec::new(); + + for pathbuf in vec.iter() { + let relative = pathbuf.strip_prefix(self.base_dir.clone())?; + + results.push( + self.peek_file( + relative + .to_str() + .ok_or(anyhow!("Could not parse path"))? + .to_owned(), + )?, + ); + } + + Ok(results) + } + + fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result> { + let mut file = File::open(self.base_dir.join(file.relative_filename.clone()))?; + + if start != 0 { + file.seek(SeekFrom::Start(start))?; + } + + if end != 0 { + return Ok(Box::new(file.take(end - start))); + } + + Ok(Box::new(file)) + } + + fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + let pathbuf = self.base_dir.join(sub_path.clone()); + if !pathbuf.exists() { + return Err(anyhow!("Path doesn't exist.")); + }; + + let file = File::open(pathbuf.clone())?; + let metadata = file.try_clone()?.metadata()?; + let permission_object = metadata.permissions(); + let permissions = { + let perm: u32; + #[cfg(target_family = "unix")] + { + perm = permission_object.mode(); + } + #[cfg(not(target_family = "unix"))] + { + perm = 0 + } + perm + }; + + Ok(VersionFile { + relative_filename: sub_path, + permission: permissions, + size: metadata.len(), + }) + } + + fn require_whole_files(&self) -> bool { + false + } +} + +pub static SEVEN_ZIP_INSTALLED: LazyLock = + LazyLock::new(|| Command::new("7z").output().is_ok()); +// https://7-zip.opensource.jp/chm/general/formats.htm +// Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above +pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [ + "7z", "bz2", "bzip2", "tbz2", "tbz", "gz", "gzip", "tgz", "tar", "wim", "swm", "esd", "xz", + "txz", "zip", "zipx", "jar", "xpi", "odt", "ods", "docx", "xlsx", "epub", "apm", "ar", "a", + "deb", "lib", "arj", "cab", "chm", "chw", "chi", "chq", "msi", "msp", "doc", "xls", "ppt", + "cpio", "cramfs", "dmg", "ext", "ext2", "ext3", "ext4", "img", "fat", "img", "hfs", "hfsx", + "hxs", "hxr", "hxq", "hxw", "lit", "ihex", "iso", "img", "lzh", "lha", "lzma", "mbr", "mslz", + "mub", "nsis", "ntfs", "img", "mbr", "rar", "r00", "rpm", "ppmd", "qcow", "qcow2", "qcow2c", + "squashfs", "udf", "iso", "img", "scap", "uefif", "vdi", "vhd", "vmdk", "xar", "pkg", "z", + "taz", +]; + +#[derive(Clone)] +pub struct ZipVersionBackend { + path: String, +} +impl ZipVersionBackend { + pub fn new(path: PathBuf) -> anyhow::Result { + Ok(Self { + path: path.to_str().expect("invalid utf path").to_owned(), + }) + } +} + +pub struct ZipFileWrapper { + command: Child, + reader: BufReader, +} + +impl ZipFileWrapper { + pub fn new(mut command: Child) -> Self { + let stdout = command + .stdout + .take() + .expect("failed to access stdout of 7z"); + let reader = BufReader::new(stdout); + ZipFileWrapper { command, reader } + } +} + +/** + * This read implemention is a result of debugging hell + * It should probably be replaced with a .take() call. + */ +impl Read for ZipFileWrapper { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.reader.read(buf) + } +} + +impl Drop for ZipFileWrapper { + fn drop(&mut self) { + self.command.wait().expect("failed to wait for 7z exit"); + } +} + +impl VersionBackend for ZipVersionBackend { + fn list_files(&mut self) -> anyhow::Result> { + let mut list_command = Command::new("7z"); + list_command.args(vec!["l", "-ba", &self.path]); + let result = list_command.output()?; + if !result.status.success() { + return Err(anyhow!( + "failed to list files: code {:?}", + result.status.code() + )); + } + let raw_result = String::from_utf8(result.stdout)?; + let files = raw_result + .split("\n") + .filter(|v| v.len() > 0) + .map(|v| v.split(" ").filter(|v| v.len() > 0)); + let mut results = Vec::new(); + + for file in files { + let values = file.collect::>(); + let mut iter = values.iter(); + let (date, time, attrs, size, compress, name) = ( + iter.next().expect("failed to read date"), + iter.next().expect("failed to read time"), + iter.next().expect("failed to read attrs"), + iter.next().expect("failed to read size"), + iter.next().expect("failed to read compress"), + iter.collect::>(), + ); + if attrs.starts_with("D") { + continue; + } + results.push(VersionFile { + relative_filename: name + .into_iter() + .map(|v| *v) + .fold(String::new(), |a, b| a + b + " ") + .trim_end() + .to_owned(), + permission: 0o744, // owner r/w/x, everyone else, read + size: size.parse().unwrap(), + }); + } + + Ok(results) + } + + fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result> { + let mut read_command = Command::new("7z"); + read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); + let output = read_command + .stdout(Stdio::piped()) + .spawn() + .expect("failed to spawn 7z"); + Ok(Box::new(ZipFileWrapper::new(output))) + } + + fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + let files = self.list_files()?; + let file = files + .iter() + .find(|v| v.relative_filename == sub_path) + .expect("file not found"); + + Ok(file.clone()) + } + + fn require_whole_files(&self) -> bool { + true + } +} diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs new file mode 100644 index 00000000..21b07e7c --- /dev/null +++ b/libraries/droplet/src/versions/mod.rs @@ -0,0 +1,2 @@ +pub mod types; +pub mod backends; \ No newline at end of file diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs new file mode 100644 index 00000000..91c234d5 --- /dev/null +++ b/libraries/droplet/src/versions/types.rs @@ -0,0 +1,55 @@ +use std::{fmt::Debug, io::Read}; + +use dyn_clone::DynClone; +use tokio::io::{self, AsyncRead}; + +#[derive(Debug, Clone)] +pub struct VersionFile { + pub relative_filename: String, + pub permission: u32, + pub size: u64, +} + +pub trait MinimumFileObject: Read + Send {} +impl MinimumFileObject for T {} + +// Intentionally not a generic, because of types in read_file +pub struct ReadToAsyncRead<'a> { + pub inner: Box, +} + +const ASYNC_READ_BUFFER_SIZE: usize = 8128; + +impl<'a> AsyncRead for ReadToAsyncRead<'a> { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; + let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); + match self.inner.read(&mut read_buf[0..read_size]) { + Ok(read) => { + buf.put_slice(&read_buf[0..read]); + std::task::Poll::Ready(Ok(())) + } + Err(err) => { + std::task::Poll::Ready(Err(err)) + }, + } + } +} + +pub trait VersionBackend: DynClone { + fn require_whole_files(&self) -> bool; + fn list_files(&mut self) -> anyhow::Result>; + fn peek_file(&mut self, sub_path: String) -> anyhow::Result; + fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result>; +} + +dyn_clone::clone_trait_object!(VersionBackend);