feat: zip file reading

.gitignore

@@ -202,3 +202,6 @@ manifest.json
 
 # JetBrains
 .idea
+
+assets/*
+!assets/generate.sh

Cargo.toml

@@ -24,7 +24,13 @@ webpki = "0.22.4"
 ring = "0.17.14"
 tokio = { version = "1.45.1", features = ["fs", "io-util"] }
 tokio-util = { version = "0.7.15", features = ["codec"] }
-zip = "4.2.0"
+rawzip = "0.2.0"
+
+[package.metadata.patch]
+crates = ["rawzip"]
+
+[patch.crates-io]
+rawzip = { path="./target/patch/rawzip-0.2.0" }
 
 [dependencies.x509-parser]
 version = "0.17.0"

__test__/utils.spec.mjs

@@ -2,7 +2,7 @@ import test from "ava";
 import fs from "node:fs";
 import path from "path";
 
-import droplet from "../index.js";
+import droplet, { generateManifest } from "../index.js";
 
 test("check alt thread util", async (t) => {
   let endtime1, endtime2;
@@ -45,7 +45,6 @@ test("read file", async (t) => {
   fs.rmSync(dirName, { recursive: true });
 });
 
-
 test("read file offset", async (t) => {
   const dirName = "./.test3";
   if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true });
@@ -65,6 +64,36 @@ test("read file offset", async (t) => {
 
   const expectedString = testString.slice(1, 4);
 
-  t.assert(finalString == expectedString, "file strings don't match");
+  t.assert(
+    finalString == expectedString,
+    `file strings don't match: ${finalString} vs ${expectedString}`
+  );
   fs.rmSync(dirName, { recursive: true });
 });
+
+test("zip file reader", async (t) => {
+  const manifest = JSON.parse(
+    await new Promise((r, e) =>
+      generateManifest(
+        "./assets/TheGame.zip",
+        (_, __) => {},
+        (_, __) => {},
+        (err, manifest) => (err ? e(err) : r(manifest))
+      )
+    )
+  );
+
+  console.log(manifest);
+
+  return t.pass();
+  const stream = droplet.readFile("./assets/TheGame.zip", "TheGame/setup.exe");
+
+  let finalString;
+  for await (const chunk of stream) {
+    console.log(`read chunk ${chunk}`);
+    // Do something with each 'chunk'
+    finalString += String.fromCharCode.apply(null, chunk);
+  }
+
+  console.log(finalString);
+});

assets/generate.sh

@@ -0,0 +1,3 @@
+dd if=/dev/random of=./setup.exe bs=1024 count=1000000
+zip TheGame.zip setup.exe
+rm setup.exe

patches/rawzip+0.2.0.patch

@@ -0,0 +1,26 @@
+diff --git a/src/archive.rs b/src/archive.rs
+index 1203015..837c405 100644
+--- a/src/archive.rs
++++ b/src/archive.rs
+@@ -275,7 +275,7 @@ impl<'data> Iterator for ZipSliceEntries<'data> {
+ /// ```
+ #[derive(Debug, Clone)]
+ pub struct ZipArchive<R> {
+-    pub(crate) reader: R,
++    pub reader: R,
+     pub(crate) comment: ZipString,
+     pub(crate) eocd: EndOfCentralDirectory,
+ }
+@@ -431,9 +431,9 @@ where
+ #[derive(Debug, Clone)]
+ pub struct ZipEntry<'archive, R> {
+     archive: &'archive ZipArchive<R>,
+-    body_offset: u64,
+-    body_end_offset: u64,
+-    entry: ZipArchiveEntryWayfinder,
++    pub body_offset: u64,
++    pub body_end_offset: u64,
++    pub entry: ZipArchiveEntryWayfinder,
+ }
+ 
+ impl<'archive, R> ZipEntry<'archive, R>

src/version/backends.rs

@@ -1,11 +1,18 @@
+use core::arch;
 #[cfg(unix)]
 use std::os::unix::fs::PermissionsExt;
 use std::{
   fs::File,
   io::{self, Read},
   path::PathBuf,
+  pin::Pin,
+  rc::Rc,
+  sync::Arc,
+};
+
+use rawzip::{
+  FileReader, ReaderAt, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, RECOMMENDED_BUFFER_SIZE,
 };
-use zip::{read::ZipFile, ZipArchive};
 
 use crate::version::{
   types::{MinimumFileObject, Skippable, VersionBackend, VersionFile},
@@ -57,51 +64,96 @@ impl VersionBackend for PathVersionBackend {
 }
 
 pub struct ZipVersionBackend {
-  archive: ZipArchive<File>,
+  archive: Arc<ZipArchive<FileReader>>,
 }
 impl ZipVersionBackend {
-  pub fn new(archive: PathBuf) -> Self {
+  pub fn new(archive: File) -> Self {
-    let handle = File::open(archive).unwrap();
+    let archive = ZipArchive::from_file(archive, &mut [0u8; RECOMMENDED_BUFFER_SIZE]).unwrap();
     Self {
-      archive: ZipArchive::new(handle).unwrap(),
+      archive: Arc::new(archive),
+    }
+  }
+
+  pub fn new_entry(&self, entry: ZipEntry<'_, FileReader>) -> ZipFileWrapper {
+    ZipFileWrapper {
+      archive: self.archive.clone(),
+      wayfinder: entry.entry,
+      offset: entry.body_offset,
+      end_offset: entry.body_end_offset,
     }
   }
 }
-
+impl Drop for ZipVersionBackend {
-struct ZipFileWrapper<'a> {
+  fn drop(&mut self) {
-  inner: ZipFile<'a, File>,
+    println!("dropping archive");
+  }
 }
 
-impl Read for ZipFileWrapper<'_> {
+struct ZipFileWrapper {
+  pub archive: Arc<ZipArchive<FileReader>>,
+  wayfinder: ZipArchiveEntryWayfinder,
+  offset: u64,
+  end_offset: u64,
+}
+
+impl Read for ZipFileWrapper {
   fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-    self.inner.read(buf)
+    let read_size = buf.len().min((self.end_offset - self.offset) as usize);
+    let read = self
+      .archive
+      .reader
+      .read_at(&mut buf[..read_size], self.offset)?;
+    self.offset += read as u64;
+    Ok(read)
   }
 }
-impl Skippable for ZipFileWrapper<'_> {
+impl Skippable for ZipFileWrapper {
   fn skip(&mut self, amount: u64) {
-    io::copy(&mut self.inner.by_ref().take(amount), &mut io::sink()).unwrap();
+    /*io::copy(
+        &mut self.inner.reader().by_ref().take(amount),
+        &mut io::sink(),
+      )
+      .unwrap();
+    */
   }
 }
-impl MinimumFileObject for ZipFileWrapper<'_> {}
+impl MinimumFileObject for ZipFileWrapper {}
 
 impl VersionBackend for ZipVersionBackend {
   fn list_files(&mut self) -> Vec<VersionFile> {
     let mut results = Vec::new();
-    for i in 0..self.archive.len() {
+    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-      let entry = self.archive.by_index(i).unwrap();
+    let mut budget_iterator = self.archive.entries(read_buffer);
+    while let Some(entry) = budget_iterator.next_entry().unwrap() {
+      if entry.is_dir() {
+        continue;
+      }
       results.push(VersionFile {
-        relative_filename: entry.name().to_owned(),
+        relative_filename: entry.file_safe_path().unwrap().to_string(),
-        permission: entry.unix_mode().or(Some(0)).unwrap(),
+        permission: 744, // apparently ZIPs with permissions are not supported by this library, so we let the owner do anything
       });
     }
     results
   }
 
   fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject)>> {
-    let file = self.archive.by_name(&file.relative_filename).ok()?;
+    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-    let zip_file_wrapper = ZipFileWrapper { inner: file };
+    let mut entries = self.archive.entries(read_buffer);
+    let entry = loop {
+      if let Some(v) = entries.next_entry().unwrap() {
+        if v.file_safe_path().unwrap().to_string() == file.relative_filename {
+          break Some(v);
+        }
+      } else {
+        break None;
+      }
+    }?;
 
-    //Some(Box::new(zip_file_wrapper))
+    let wayfinder = entry.wayfinder();
-    None
+    let local_entry = self.archive.get_entry(wayfinder).unwrap();
+
+    let wrapper = self.new_entry(local_entry);
+
+    Some(Box::new(wrapper))
   }
 }

src/version/types.rs

@@ -1,4 +1,6 @@
-use std::io::{Read, Seek, SeekFrom};
+use std::{
+  fmt::Debug, io::{Read, Seek, SeekFrom}
+};
 
 use tokio::io::{self, AsyncRead};
 
@@ -26,6 +28,7 @@ impl<T: Read + Send + Seek> MinimumFileObject for T {}
 // Intentionally not a generic, because of types in read_file
 pub struct ReadToAsyncRead {
   pub inner: Box<(dyn Read + Send)>,
+  pub backend: Box<(dyn VersionBackend + Send)>,
 }
 
 impl AsyncRead for ReadToAsyncRead {
@@ -35,7 +38,8 @@ impl AsyncRead for ReadToAsyncRead {
     buf: &mut tokio::io::ReadBuf<'_>,
   ) -> std::task::Poll<io::Result<()>> {
     let mut read_buf = [0u8; 8192];
-    let amount = self.inner.read(&mut read_buf).unwrap();
+    let var_name = self.inner.read(&mut read_buf).unwrap();
+    let amount = var_name;
     buf.put_slice(&read_buf[0..amount]);
     std::task::Poll::Ready(Ok(()))
   }

src/version/utils.rs

@@ -6,11 +6,10 @@ use std::{
 
 use napi::{bindgen_prelude::*, tokio_stream::StreamExt};
 use tokio_util::codec::{BytesCodec, FramedRead};
-use zip::ZipArchive;
 
 use crate::version::{
   backends::{PathVersionBackend, ZipVersionBackend},
-  types::{MinimumFileObject, ReadToAsyncRead, VersionBackend, VersionFile},
+  types::{ReadToAsyncRead, VersionBackend, VersionFile},
 };
 
 pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
@@ -27,7 +26,7 @@ pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
   }
 }
 
-pub fn create_backend_for_path(path: &Path) -> Option<Box<(dyn VersionBackend + Send)>> {
+pub fn create_backend_for_path<'a>(path: &Path) -> Option<Box<(dyn VersionBackend + Send + 'a)>> {
   let is_directory = path.is_dir();
   if is_directory {
     return Some(Box::new(PathVersionBackend {
@@ -35,12 +34,9 @@ pub fn create_backend_for_path(path: &Path) -> Option<Box<(dyn VersionBackend +
     }));
   };
 
-  /*
+  if path.to_string_lossy().ends_with(".zip") {
-    Insert checks for whatever backend you like
+    let f = File::open(path.to_path_buf()).unwrap();
-  */
+    return Some(Box::new(ZipVersionBackend::new(f)));
-
-  if path.ends_with(".zip") {
-    return Some(Box::new(ZipVersionBackend::new(path.to_path_buf())));
   }
 
   None
@@ -56,11 +52,12 @@ pub fn has_backend_for_path(path: String) -> bool {
 }
 
 #[napi]
-pub fn list_files(path: String) -> Vec<String> {
+pub fn list_files(path: String) -> Result<Vec<String>> {
   let path = Path::new(&path);
-  let mut backend = create_backend_for_path(path).unwrap();
+  let mut backend =
+    create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
   let files = backend.list_files();
-  files.into_iter().map(|e| e.relative_filename).collect()
+  Ok(files.into_iter().map(|e| e.relative_filename).collect())
 }
 
 #[napi]
@@ -70,7 +67,7 @@ pub fn read_file(
   env: &Env,
   start: Option<u32>,
   end: Option<u32>,
-) -> Option<ReadableStream<'static, BufferSlice<'static>>> {
+) -> Option<ReadableStream<'_, BufferSlice<'_>>> {
   let path = Path::new(&path);
   let mut backend = create_backend_for_path(path).unwrap();
   let version_file = VersionFile {
@@ -90,9 +87,10 @@ pub fn read_file(
     let amount = limit - start.or(Some(0)).unwrap();
     ReadToAsyncRead {
       inner: Box::new(reader.take(amount.into())),
+      backend
     }
   } else {
-    ReadToAsyncRead { inner: reader }
+    ReadToAsyncRead { inner: reader, backend }
   };
 
   // Create a FramedRead stream with BytesCodec for chunking