feat: the 7z update

index.d.ts

@@ -21,8 +21,7 @@ export declare class Script {
 
 export declare class ScriptEngine {
   constructor()
-  buildRahiScript(content: string): Script
-  buildLuaScript(content: string): Script
+  buildRhaiScript(content: string): Script
   buildJsScript(content: string): Script
   execute(script: Script): void
   fetchStrings(script: Script): Array<string>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {

src/lib.rs

@@ -2,6 +2,7 @@
 #![deny(clippy::expect_used)]
 #![deny(clippy::panic)]
 #![feature(trait_alias)]
+#![feature(iterator_try_collect)]
 
 
 pub mod manifest;

src/manifest.rs

@@ -48,6 +48,8 @@ pub fn generate_manifest<'a>(
   let backend: &'static mut Box<dyn VersionBackend + Send> =
     unsafe { std::mem::transmute(backend) };
 
+    let required_single_file = backend.require_whole_files();
+
   thread::spawn(move || {
     let callback_borrow = &callback_sfn;
 
@@ -91,7 +93,7 @@ pub fn generate_manifest<'a>(
 
             buffer.extend_from_slice(&buf[0..read]);
 
-            if length >= CHUNK_SIZE {
+            if length >= CHUNK_SIZE && !required_single_file {
               break;
             }
           }

src/version/backends.rs

@@ -1,10 +1,12 @@
 #[cfg(unix)]
 use std::os::unix::fs::PermissionsExt;
 use std::{
+  cell::LazyCell,
   fs::{self, metadata, File},
-  io::{self, Read, Seek, SeekFrom, Sink},
+  io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink},
   path::{Path, PathBuf},
-  sync::Arc,
+  process::{Child, ChildStdout, Command, Stdio},
+  sync::{Arc, LazyLock},
 };
 
 use anyhow::anyhow;
@@ -106,118 +108,88 @@ impl VersionBackend for PathVersionBackend {
       size: metadata.len(),
     })
   }
+
+  fn require_whole_files(&self) -> bool {
+    false
   }
+}
+
+pub static SEVEN_ZIP_INSTALLED: LazyLock<bool> =
+  LazyLock::new(|| Command::new("7z").output().is_ok());
 
 #[derive(Clone)]
 pub struct ZipVersionBackend {
-  archive: Arc<ZipArchive<FileReader>>,
+  path: String,
 }
 impl ZipVersionBackend {
-  pub fn new(archive: File) -> anyhow::Result<Self> {
-    let archive = ZipArchive::from_file(archive, &mut [0u8; RECOMMENDED_BUFFER_SIZE])?;
+  pub fn new(path: PathBuf) -> anyhow::Result<Self> {
     Ok(Self {
-      archive: Arc::new(archive),
-    })
-  }
-
-  pub fn new_entry<'archive>(
-    &self,
-    entry: ZipEntry<'archive, FileReader>,
-    compression_method: CompressionMethod,
-    start: u64,
-    end: u64,
-  ) -> anyhow::Result<ZipFileWrapper<'archive>> {
-    let deflater: Box<dyn Read + Send + 'archive> = match compression_method {
-      CompressionMethod::Store => Box::new(entry.reader()),
-      CompressionMethod::Deflate => Box::new(DeflateDecoder::new(entry.reader())),
-      CompressionMethod::Deflate64 => Box::new(DeflateDecoder::new(entry.reader())),
-      _ => Err(anyhow!(
-        "unsupported decompression algorithm: {compression_method:?}"
-      ))?,
-    };
-
-    let mut verifier = entry.verifying_reader(deflater);
-    if start != 0 {
-      io::copy(&mut (&mut verifier).take(start), &mut Sink::default())?;
-    }
-
-    Ok(ZipFileWrapper {
-      reader: verifier,
-      limit: (end - start) as usize,
-      current: 0,
+      path: path.to_str().expect("invalid utf path").to_owned(),
     })
   }
 }
 
-pub struct ZipFileWrapper<'archive> {
-  reader: ZipVerifier<'archive, Box<dyn Read + Send + 'archive>, FileReader>,
-  limit: usize,
-  current: usize,
+pub struct ZipFileWrapper {
+  command: Child,
+  reader: BufReader<ChildStdout>
+}
+
+impl ZipFileWrapper {
+  pub fn new(mut command: Child) -> Self {
+    let stdout = command.stdout.take().expect("failed to access stdout of 7z");
+    let reader = BufReader::new(stdout);
+    ZipFileWrapper { command, reader }
+  }
 }
 
 /**
  * This read implemention is a result of debugging hell
  * It should probably be replaced with a .take() call.
  */
-impl<'a> Read for ZipFileWrapper<'a> {
+impl Read for ZipFileWrapper {
   fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-    let has_limit = self.limit != 0;
-
-    // End this stream if the read is the right size
-    if has_limit && self.current >= self.limit {
-      return Ok(0);
+    self.reader.read(buf)
+  }
 }
 
-    let read = self.reader.read(buf)?;
-    if self.limit != 0 {
-      self.current += read;
-      if self.current > self.limit {
-        let over = self.current - self.limit;
-        return Ok(read - over);
+impl Drop for ZipFileWrapper {
+    fn drop(&mut self) {
+        self.command.wait().expect("failed to wait for 7z exit");
     }
 }
-    Ok(read)
-  }
-}
-//impl<'a> MinimumFileObject for ZipFileWrapper<'a> {}
 
-impl ZipVersionBackend {
-  fn find_wayfinder(
-    &mut self,
-    filename: &str,
-  ) -> anyhow::Result<(ZipArchiveEntryWayfinder, CompressionMethod)> {
-    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-    let mut entries = self.archive.entries(read_buffer);
-    let entry = loop {
-      if let Some(v) = entries.next_entry()? {
-        if v.file_path().try_normalize()?.as_ref() == filename {
-          break Ok(v);
-        }
-      } else {
-        break Err(anyhow!("failed to fetch zip file header."));
-      }
-    }?;
-
-    let wayfinder = entry.wayfinder();
-
-    Ok((wayfinder, entry.compression_method()))
-  }
-}
 impl VersionBackend for ZipVersionBackend {
   fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
-    let mut results = Vec::new();
-    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-    let mut budget_iterator = self.archive.entries(read_buffer);
-    while let Some(entry) = budget_iterator.next_entry()? {
-      if entry.is_dir() {
-        continue;
+    let mut list_command = Command::new("7z");
+    list_command.args(vec!["l", "-ba", &self.path]);
+    let result = list_command.output()?;
+    if !result.status.success() {
+      return Err(anyhow!(
+        "failed to list files: code {:?}",
+        result.status.code()
+      ));
     }
+    let raw_result = String::from_utf8(result.stdout)?;
+    let files = raw_result.split("\n").filter(|v| v.len() > 0).map(|v| v.split(" ").filter(|v| v.len() > 0));
+    let mut results = Vec::new();
+
+    for mut file in files {
+      let (date, time, attrs, size, compress, name) = (
+        file.next().unwrap(),
+        file.next().unwrap(),
+        file.next().unwrap(),
+        file.next().unwrap(),
+        file.next().unwrap(),
+        file.next().unwrap(),
+      );
+      println!("got line: {} {} {} {} {} {}", date, time, attrs, size, compress, name);
       results.push(VersionFile {
-        relative_filename: String::from(entry.file_path().try_normalize()?),
-        permission: entry.mode().permissions(),
-        size: entry.uncompressed_size_hint(),
+        relative_filename: name.to_owned(),
+        permission: 0,
+        size: size.parse().unwrap(),
       });
     }
+
     Ok(results)
   }
 
@@ -227,23 +199,23 @@ impl VersionBackend for ZipVersionBackend {
     start: u64,
     end: u64,
   ) -> anyhow::Result<Box<dyn MinimumFileObject + '_>> {
-    let (wayfinder, compression_method) = self.find_wayfinder(&file.relative_filename)?;
-    let local_entry = self
-      .archive
-      .get_entry(wayfinder)?;
-
-    let wrapper = self.new_entry(local_entry, compression_method, start, end)?;
-
-    Ok(Box::new(wrapper) as Box<dyn MinimumFileObject>)
+    let mut read_command = Command::new("7z");
+    read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]);
+    let output = read_command.stdout(Stdio::piped()).spawn().expect("failed to spawn 7z");
+    Ok(Box::new(ZipFileWrapper::new(output)))
   }
 
   fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
-    let (entry, _) = self.find_wayfinder(&sub_path)?;
+    let files = self.list_files()?;
+    let file = files
+      .iter()
+      .find(|v| v.relative_filename == sub_path)
+      .expect("file not found");
 
-    Ok(VersionFile {
-      relative_filename: sub_path,
-      permission: 0,
-      size: entry.uncompressed_size_hint(),
-    })
+    Ok(file.clone())
+  }
+
+  fn require_whole_files(&self) -> bool {
+    true
   }
 }

src/version/types.rs

@@ -41,6 +41,7 @@ impl<'a> AsyncRead for ReadToAsyncRead<'a> {
 }
 
 pub trait VersionBackend: DynClone {
+  fn require_whole_files(&self) -> bool;
   fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>>;
   fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile>;
   fn reader(

src/version/utils.rs

@@ -1,11 +1,16 @@
-use std::{collections::HashMap, fs::File, path::Path};
+use std::{
+  collections::HashMap,
+  fs::File,
+  path::Path,
+  process::{Command, ExitStatus},
+};
 
 use anyhow::anyhow;
 use napi::{bindgen_prelude::*, sys::napi_value__, tokio_stream::StreamExt};
 use tokio_util::codec::{BytesCodec, FramedRead};
 
 use crate::version::{
-  backends::{PathVersionBackend, ZipVersionBackend},
+  backends::{PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED},
   types::{ReadToAsyncRead, VersionBackend, VersionFile},
 };
 
@@ -27,9 +32,16 @@ pub fn create_backend_constructor<'a>(
     }));
   };
 
-  if path.to_string_lossy().ends_with(".zip") {
-    let f = File::open(path.to_path_buf()).ok()?;
-    return Some(Box::new(|| Ok(Box::new(ZipVersionBackend::new(f)?))));
+  if *SEVEN_ZIP_INSTALLED {
+    let mut test = Command::new("7z");
+    test.args(vec!["t", path.to_str().expect("invalid utf path")]);
+    let status = test.status().ok()?;
+    if status.code().unwrap_or(1) == 0 {
+      let buf = path.to_path_buf();
+      return Some(Box::new(move || {
+        Ok(Box::new(ZipVersionBackend::new(buf)?))
+      }));
+    }
   }
 
   None