diff --git a/.gitignore b/.gitignore index 8ea66da..9982b06 100644 --- a/.gitignore +++ b/.gitignore @@ -201,4 +201,7 @@ test.mjs manifest.json # JetBrains -.idea \ No newline at end of file +.idea + +assets/* +!assets/generate.sh \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 871fdf9..7862d72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,13 @@ webpki = "0.22.4" ring = "0.17.14" tokio = { version = "1.45.1", features = ["fs", "io-util"] } tokio-util = { version = "0.7.15", features = ["codec"] } -zip = "4.2.0" +rawzip = "0.2.0" + +[package.metadata.patch] +crates = ["rawzip"] + +[patch.crates-io] +rawzip = { path="./target/patch/rawzip-0.2.0" } [dependencies.x509-parser] version = "0.17.0" diff --git a/__test__/utils.spec.mjs b/__test__/utils.spec.mjs index 502b4f5..2bbcfe9 100644 --- a/__test__/utils.spec.mjs +++ b/__test__/utils.spec.mjs @@ -2,7 +2,7 @@ import test from "ava"; import fs from "node:fs"; import path from "path"; -import droplet from "../index.js"; +import droplet, { generateManifest } from "../index.js"; test("check alt thread util", async (t) => { let endtime1, endtime2; @@ -45,7 +45,6 @@ test("read file", async (t) => { fs.rmSync(dirName, { recursive: true }); }); - test("read file offset", async (t) => { const dirName = "./.test3"; if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true }); @@ -65,6 +64,36 @@ test("read file offset", async (t) => { const expectedString = testString.slice(1, 4); - t.assert(finalString == expectedString, "file strings don't match"); + t.assert( + finalString == expectedString, + `file strings don't match: ${finalString} vs ${expectedString}` + ); fs.rmSync(dirName, { recursive: true }); }); + +test("zip file reader", async (t) => { + const manifest = JSON.parse( + await new Promise((r, e) => + generateManifest( + "./assets/TheGame.zip", + (_, __) => {}, + (_, __) => {}, + (err, manifest) => (err ? e(err) : r(manifest)) + ) + ) + ); + + console.log(manifest); + + return t.pass(); + const stream = droplet.readFile("./assets/TheGame.zip", "TheGame/setup.exe"); + + let finalString; + for await (const chunk of stream) { + console.log(`read chunk ${chunk}`); + // Do something with each 'chunk' + finalString += String.fromCharCode.apply(null, chunk); + } + + console.log(finalString); +}); diff --git a/assets/generate.sh b/assets/generate.sh new file mode 100755 index 0000000..5ba1f19 --- /dev/null +++ b/assets/generate.sh @@ -0,0 +1,3 @@ +dd if=/dev/random of=./setup.exe bs=1024 count=1000000 +zip TheGame.zip setup.exe +rm setup.exe \ No newline at end of file diff --git a/patches/rawzip+0.2.0.patch b/patches/rawzip+0.2.0.patch new file mode 100644 index 0000000..e0cb06a --- /dev/null +++ b/patches/rawzip+0.2.0.patch @@ -0,0 +1,26 @@ +diff --git a/src/archive.rs b/src/archive.rs +index 1203015..837c405 100644 +--- a/src/archive.rs ++++ b/src/archive.rs +@@ -275,7 +275,7 @@ impl<'data> Iterator for ZipSliceEntries<'data> { + /// ``` + #[derive(Debug, Clone)] + pub struct ZipArchive { +- pub(crate) reader: R, ++ pub reader: R, + pub(crate) comment: ZipString, + pub(crate) eocd: EndOfCentralDirectory, + } +@@ -431,9 +431,9 @@ where + #[derive(Debug, Clone)] + pub struct ZipEntry<'archive, R> { + archive: &'archive ZipArchive, +- body_offset: u64, +- body_end_offset: u64, +- entry: ZipArchiveEntryWayfinder, ++ pub body_offset: u64, ++ pub body_end_offset: u64, ++ pub entry: ZipArchiveEntryWayfinder, + } + + impl<'archive, R> ZipEntry<'archive, R> diff --git a/src/version/backends.rs b/src/version/backends.rs index 3147385..de01d81 100644 --- a/src/version/backends.rs +++ b/src/version/backends.rs @@ -1,11 +1,18 @@ +use core::arch; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::{ fs::File, io::{self, Read}, path::PathBuf, + pin::Pin, + rc::Rc, + sync::Arc, +}; + +use rawzip::{ + FileReader, ReaderAt, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, RECOMMENDED_BUFFER_SIZE, }; -use zip::{read::ZipFile, ZipArchive}; use crate::version::{ types::{MinimumFileObject, Skippable, VersionBackend, VersionFile}, @@ -57,51 +64,96 @@ impl VersionBackend for PathVersionBackend { } pub struct ZipVersionBackend { - archive: ZipArchive, + archive: Arc>, } impl ZipVersionBackend { - pub fn new(archive: PathBuf) -> Self { - let handle = File::open(archive).unwrap(); + pub fn new(archive: File) -> Self { + let archive = ZipArchive::from_file(archive, &mut [0u8; RECOMMENDED_BUFFER_SIZE]).unwrap(); Self { - archive: ZipArchive::new(handle).unwrap(), + archive: Arc::new(archive), + } + } + + pub fn new_entry(&self, entry: ZipEntry<'_, FileReader>) -> ZipFileWrapper { + ZipFileWrapper { + archive: self.archive.clone(), + wayfinder: entry.entry, + offset: entry.body_offset, + end_offset: entry.body_end_offset, } } } - -struct ZipFileWrapper<'a> { - inner: ZipFile<'a, File>, +impl Drop for ZipVersionBackend { + fn drop(&mut self) { + println!("dropping archive"); + } } -impl Read for ZipFileWrapper<'_> { +struct ZipFileWrapper { + pub archive: Arc>, + wayfinder: ZipArchiveEntryWayfinder, + offset: u64, + end_offset: u64, +} + +impl Read for ZipFileWrapper { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.inner.read(buf) + let read_size = buf.len().min((self.end_offset - self.offset) as usize); + let read = self + .archive + .reader + .read_at(&mut buf[..read_size], self.offset)?; + self.offset += read as u64; + Ok(read) } } -impl Skippable for ZipFileWrapper<'_> { +impl Skippable for ZipFileWrapper { fn skip(&mut self, amount: u64) { - io::copy(&mut self.inner.by_ref().take(amount), &mut io::sink()).unwrap(); + /*io::copy( + &mut self.inner.reader().by_ref().take(amount), + &mut io::sink(), + ) + .unwrap(); + */ } } -impl MinimumFileObject for ZipFileWrapper<'_> {} +impl MinimumFileObject for ZipFileWrapper {} impl VersionBackend for ZipVersionBackend { fn list_files(&mut self) -> Vec { let mut results = Vec::new(); - for i in 0..self.archive.len() { - let entry = self.archive.by_index(i).unwrap(); + let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE]; + let mut budget_iterator = self.archive.entries(read_buffer); + while let Some(entry) = budget_iterator.next_entry().unwrap() { + if entry.is_dir() { + continue; + } results.push(VersionFile { - relative_filename: entry.name().to_owned(), - permission: entry.unix_mode().or(Some(0)).unwrap(), + relative_filename: entry.file_safe_path().unwrap().to_string(), + permission: 744, // apparently ZIPs with permissions are not supported by this library, so we let the owner do anything }); } results } fn reader(&mut self, file: &VersionFile) -> Option> { - let file = self.archive.by_name(&file.relative_filename).ok()?; - let zip_file_wrapper = ZipFileWrapper { inner: file }; + let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE]; + let mut entries = self.archive.entries(read_buffer); + let entry = loop { + if let Some(v) = entries.next_entry().unwrap() { + if v.file_safe_path().unwrap().to_string() == file.relative_filename { + break Some(v); + } + } else { + break None; + } + }?; - //Some(Box::new(zip_file_wrapper)) - None + let wayfinder = entry.wayfinder(); + let local_entry = self.archive.get_entry(wayfinder).unwrap(); + + let wrapper = self.new_entry(local_entry); + + Some(Box::new(wrapper)) } } diff --git a/src/version/types.rs b/src/version/types.rs index 18c779f..5889d78 100644 --- a/src/version/types.rs +++ b/src/version/types.rs @@ -1,4 +1,6 @@ -use std::io::{Read, Seek, SeekFrom}; +use std::{ + fmt::Debug, io::{Read, Seek, SeekFrom} +}; use tokio::io::{self, AsyncRead}; @@ -26,6 +28,7 @@ impl MinimumFileObject for T {} // Intentionally not a generic, because of types in read_file pub struct ReadToAsyncRead { pub inner: Box<(dyn Read + Send)>, + pub backend: Box<(dyn VersionBackend + Send)>, } impl AsyncRead for ReadToAsyncRead { @@ -35,7 +38,8 @@ impl AsyncRead for ReadToAsyncRead { buf: &mut tokio::io::ReadBuf<'_>, ) -> std::task::Poll> { let mut read_buf = [0u8; 8192]; - let amount = self.inner.read(&mut read_buf).unwrap(); + let var_name = self.inner.read(&mut read_buf).unwrap(); + let amount = var_name; buf.put_slice(&read_buf[0..amount]); std::task::Poll::Ready(Ok(())) } diff --git a/src/version/utils.rs b/src/version/utils.rs index 6dc30da..fa4bd59 100644 --- a/src/version/utils.rs +++ b/src/version/utils.rs @@ -6,11 +6,10 @@ use std::{ use napi::{bindgen_prelude::*, tokio_stream::StreamExt}; use tokio_util::codec::{BytesCodec, FramedRead}; -use zip::ZipArchive; use crate::version::{ backends::{PathVersionBackend, ZipVersionBackend}, - types::{MinimumFileObject, ReadToAsyncRead, VersionBackend, VersionFile}, + types::{ReadToAsyncRead, VersionBackend, VersionFile}, }; pub fn _list_files(vec: &mut Vec, path: &Path) { @@ -27,7 +26,7 @@ pub fn _list_files(vec: &mut Vec, path: &Path) { } } -pub fn create_backend_for_path(path: &Path) -> Option> { +pub fn create_backend_for_path<'a>(path: &Path) -> Option> { let is_directory = path.is_dir(); if is_directory { return Some(Box::new(PathVersionBackend { @@ -35,12 +34,9 @@ pub fn create_backend_for_path(path: &Path) -> Option bool { } #[napi] -pub fn list_files(path: String) -> Vec { +pub fn list_files(path: String) -> Result> { let path = Path::new(&path); - let mut backend = create_backend_for_path(path).unwrap(); + let mut backend = + create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?; let files = backend.list_files(); - files.into_iter().map(|e| e.relative_filename).collect() + Ok(files.into_iter().map(|e| e.relative_filename).collect()) } #[napi] @@ -70,7 +67,7 @@ pub fn read_file( env: &Env, start: Option, end: Option, -) -> Option>> { +) -> Option>> { let path = Path::new(&path); let mut backend = create_backend_for_path(path).unwrap(); let version_file = VersionFile { @@ -90,9 +87,10 @@ pub fn read_file( let amount = limit - start.or(Some(0)).unwrap(); ReadToAsyncRead { inner: Box::new(reader.take(amount.into())), + backend } } else { - ReadToAsyncRead { inner: reader } + ReadToAsyncRead { inner: reader, backend } }; // Create a FramedRead stream with BytesCodec for chunking