From bd6d7060fd5fe6f3b364cc375c759cc25813fc3f Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 2 Oct 2025 17:06:58 +1000 Subject: [PATCH] feat: the 7z update --- index.d.ts | 3 +- package.json | 2 +- src/lib.rs | 1 + src/manifest.rs | 4 +- src/version/backends.rs | 172 +++++++++++++++++----------------------- src/version/types.rs | 1 + src/version/utils.rs | 22 +++-- 7 files changed, 96 insertions(+), 109 deletions(-) diff --git a/index.d.ts b/index.d.ts index 426b0e9..eced819 100644 --- a/index.d.ts +++ b/index.d.ts @@ -21,8 +21,7 @@ export declare class Script { export declare class ScriptEngine { constructor() - buildRahiScript(content: string): Script - buildLuaScript(content: string): Script + buildRhaiScript(content: string): Script buildJsScript(content: string): Script execute(script: Script): void fetchStrings(script: Script): Array diff --git a/package.json b/package.json index 6af8255..fdc6763 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@drop-oss/droplet", - "version": "3.0.1", + "version": "3.1.0", "main": "index.js", "types": "index.d.ts", "napi": { diff --git a/src/lib.rs b/src/lib.rs index bfe8518..f149e46 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ #![deny(clippy::expect_used)] #![deny(clippy::panic)] #![feature(trait_alias)] +#![feature(iterator_try_collect)] pub mod manifest; diff --git a/src/manifest.rs b/src/manifest.rs index 1b83c99..35d0409 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -48,6 +48,8 @@ pub fn generate_manifest<'a>( let backend: &'static mut Box = unsafe { std::mem::transmute(backend) }; + let required_single_file = backend.require_whole_files(); + thread::spawn(move || { let callback_borrow = &callback_sfn; @@ -91,7 +93,7 @@ pub fn generate_manifest<'a>( buffer.extend_from_slice(&buf[0..read]); - if length >= CHUNK_SIZE { + if length >= CHUNK_SIZE && !required_single_file { break; } } diff --git a/src/version/backends.rs b/src/version/backends.rs index 6c5f8c0..959ee4d 100644 --- a/src/version/backends.rs +++ b/src/version/backends.rs @@ -1,10 +1,12 @@ #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::{ + cell::LazyCell, fs::{self, metadata, File}, - io::{self, Read, Seek, SeekFrom, Sink}, + io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink}, path::{Path, PathBuf}, - sync::Arc, + process::{Child, ChildStdout, Command, Stdio}, + sync::{Arc, LazyLock}, }; use anyhow::anyhow; @@ -106,118 +108,88 @@ impl VersionBackend for PathVersionBackend { size: metadata.len(), }) } + + fn require_whole_files(&self) -> bool { + false + } } +pub static SEVEN_ZIP_INSTALLED: LazyLock = + LazyLock::new(|| Command::new("7z").output().is_ok()); + #[derive(Clone)] pub struct ZipVersionBackend { - archive: Arc>, + path: String, } impl ZipVersionBackend { - pub fn new(archive: File) -> anyhow::Result { - let archive = ZipArchive::from_file(archive, &mut [0u8; RECOMMENDED_BUFFER_SIZE])?; + pub fn new(path: PathBuf) -> anyhow::Result { Ok(Self { - archive: Arc::new(archive), - }) - } - - pub fn new_entry<'archive>( - &self, - entry: ZipEntry<'archive, FileReader>, - compression_method: CompressionMethod, - start: u64, - end: u64, - ) -> anyhow::Result> { - let deflater: Box = match compression_method { - CompressionMethod::Store => Box::new(entry.reader()), - CompressionMethod::Deflate => Box::new(DeflateDecoder::new(entry.reader())), - CompressionMethod::Deflate64 => Box::new(DeflateDecoder::new(entry.reader())), - _ => Err(anyhow!( - "unsupported decompression algorithm: {compression_method:?}" - ))?, - }; - - let mut verifier = entry.verifying_reader(deflater); - if start != 0 { - io::copy(&mut (&mut verifier).take(start), &mut Sink::default())?; - } - - Ok(ZipFileWrapper { - reader: verifier, - limit: (end - start) as usize, - current: 0, + path: path.to_str().expect("invalid utf path").to_owned(), }) } } -pub struct ZipFileWrapper<'archive> { - reader: ZipVerifier<'archive, Box, FileReader>, - limit: usize, - current: usize, +pub struct ZipFileWrapper { + command: Child, + reader: BufReader +} + +impl ZipFileWrapper { + pub fn new(mut command: Child) -> Self { + let stdout = command.stdout.take().expect("failed to access stdout of 7z"); + let reader = BufReader::new(stdout); + ZipFileWrapper { command, reader } + } } /** * This read implemention is a result of debugging hell * It should probably be replaced with a .take() call. */ -impl<'a> Read for ZipFileWrapper<'a> { +impl Read for ZipFileWrapper { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let has_limit = self.limit != 0; - - // End this stream if the read is the right size - if has_limit && self.current >= self.limit { - return Ok(0); - } - - let read = self.reader.read(buf)?; - if self.limit != 0 { - self.current += read; - if self.current > self.limit { - let over = self.current - self.limit; - return Ok(read - over); - } - } - Ok(read) + self.reader.read(buf) } } -//impl<'a> MinimumFileObject for ZipFileWrapper<'a> {} -impl ZipVersionBackend { - fn find_wayfinder( - &mut self, - filename: &str, - ) -> anyhow::Result<(ZipArchiveEntryWayfinder, CompressionMethod)> { - let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE]; - let mut entries = self.archive.entries(read_buffer); - let entry = loop { - if let Some(v) = entries.next_entry()? { - if v.file_path().try_normalize()?.as_ref() == filename { - break Ok(v); - } - } else { - break Err(anyhow!("failed to fetch zip file header.")); - } - }?; - - let wayfinder = entry.wayfinder(); - - Ok((wayfinder, entry.compression_method())) - } +impl Drop for ZipFileWrapper { + fn drop(&mut self) { + self.command.wait().expect("failed to wait for 7z exit"); + } } + impl VersionBackend for ZipVersionBackend { fn list_files(&mut self) -> anyhow::Result> { + let mut list_command = Command::new("7z"); + list_command.args(vec!["l", "-ba", &self.path]); + let result = list_command.output()?; + if !result.status.success() { + return Err(anyhow!( + "failed to list files: code {:?}", + result.status.code() + )); + } + let raw_result = String::from_utf8(result.stdout)?; + let files = raw_result.split("\n").filter(|v| v.len() > 0).map(|v| v.split(" ").filter(|v| v.len() > 0)); let mut results = Vec::new(); - let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE]; - let mut budget_iterator = self.archive.entries(read_buffer); - while let Some(entry) = budget_iterator.next_entry()? { - if entry.is_dir() { - continue; - } + + for mut file in files { + let (date, time, attrs, size, compress, name) = ( + file.next().unwrap(), + file.next().unwrap(), + file.next().unwrap(), + file.next().unwrap(), + file.next().unwrap(), + file.next().unwrap(), + ); + println!("got line: {} {} {} {} {} {}", date, time, attrs, size, compress, name); results.push(VersionFile { - relative_filename: String::from(entry.file_path().try_normalize()?), - permission: entry.mode().permissions(), - size: entry.uncompressed_size_hint(), + relative_filename: name.to_owned(), + permission: 0, + size: size.parse().unwrap(), }); } + Ok(results) } @@ -227,23 +199,23 @@ impl VersionBackend for ZipVersionBackend { start: u64, end: u64, ) -> anyhow::Result> { - let (wayfinder, compression_method) = self.find_wayfinder(&file.relative_filename)?; - let local_entry = self - .archive - .get_entry(wayfinder)?; - - let wrapper = self.new_entry(local_entry, compression_method, start, end)?; - - Ok(Box::new(wrapper) as Box) + let mut read_command = Command::new("7z"); + read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); + let output = read_command.stdout(Stdio::piped()).spawn().expect("failed to spawn 7z"); + Ok(Box::new(ZipFileWrapper::new(output))) } fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let (entry, _) = self.find_wayfinder(&sub_path)?; + let files = self.list_files()?; + let file = files + .iter() + .find(|v| v.relative_filename == sub_path) + .expect("file not found"); - Ok(VersionFile { - relative_filename: sub_path, - permission: 0, - size: entry.uncompressed_size_hint(), - }) + Ok(file.clone()) + } + + fn require_whole_files(&self) -> bool { + true } } diff --git a/src/version/types.rs b/src/version/types.rs index d6d5cf3..91c234d 100644 --- a/src/version/types.rs +++ b/src/version/types.rs @@ -41,6 +41,7 @@ impl<'a> AsyncRead for ReadToAsyncRead<'a> { } pub trait VersionBackend: DynClone { + fn require_whole_files(&self) -> bool; fn list_files(&mut self) -> anyhow::Result>; fn peek_file(&mut self, sub_path: String) -> anyhow::Result; fn reader( diff --git a/src/version/utils.rs b/src/version/utils.rs index 3bd8cf5..b375c34 100644 --- a/src/version/utils.rs +++ b/src/version/utils.rs @@ -1,11 +1,16 @@ -use std::{collections::HashMap, fs::File, path::Path}; +use std::{ + collections::HashMap, + fs::File, + path::Path, + process::{Command, ExitStatus}, +}; use anyhow::anyhow; use napi::{bindgen_prelude::*, sys::napi_value__, tokio_stream::StreamExt}; use tokio_util::codec::{BytesCodec, FramedRead}; use crate::version::{ - backends::{PathVersionBackend, ZipVersionBackend}, + backends::{PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED}, types::{ReadToAsyncRead, VersionBackend, VersionFile}, }; @@ -27,9 +32,16 @@ pub fn create_backend_constructor<'a>( })); }; - if path.to_string_lossy().ends_with(".zip") { - let f = File::open(path.to_path_buf()).ok()?; - return Some(Box::new(|| Ok(Box::new(ZipVersionBackend::new(f)?)))); + if *SEVEN_ZIP_INSTALLED { + let mut test = Command::new("7z"); + test.args(vec!["t", path.to_str().expect("invalid utf path")]); + let status = test.status().ok()?; + if status.code().unwrap_or(1) == 0 { + let buf = path.to_path_buf(); + return Some(Box::new(move || { + Ok(Box::new(ZipVersionBackend::new(buf)?)) + })); + } } None