feat: add support for partially deflated zips

__test__/utils.spec.mjs

@@ -1,6 +1,7 @@
 import test from "ava";
 import fs from "node:fs";
 import path from "path";
+import { createHash } from "node:crypto";
 import prettyBytes from "pretty-bytes";
 
 import droplet, { DropletHandler, generateManifest } from "../index.js";
@@ -57,7 +58,12 @@ test("read file", async (t) => {
 
   const dropletHandler = new DropletHandler();
 
-  const stream = dropletHandler.readFile(dirName, "TESTFILE", BigInt(0), BigInt(testString.length));
+  const stream = dropletHandler.readFile(
+    dirName,
+    "TESTFILE",
+    BigInt(0),
+    BigInt(testString.length)
+  );
 
   let finalString = "";
 
@@ -157,6 +163,7 @@ test.skip("zip manifest test", async (t) => {
   for (const [filename, data] of Object.entries(manifest)) {
     let start = 0;
     for (const [chunkIndex, length] of data.lengths.entries()) {
+      const hash = createHash("md5");
       const stream = (
         await dropletHandler.readFile(
           "./assets/TheGame.zip",
@@ -171,6 +178,7 @@ test.skip("zip manifest test", async (t) => {
         new WritableStream({
           write(chunk) {
             streamLength += chunk.length;
+            hash.update(chunk);
           },
         })
       );
@@ -180,9 +188,33 @@ test.skip("zip manifest test", async (t) => {
           `stream length for chunk index ${chunkIndex} was not expected: real: ${streamLength} vs expected: ${length}`
         );
 
+      const digest = hash.digest("hex");
+      if (data.checksums[chunkIndex] != digest)
+        return t.fail(
+          `checksums did not match for chunk index ${chunkIndex}: real: ${digest} vs expected: ${data.checksums[chunkIndex]}`
+        );
+
       start += length;
     }
   }
 
   t.pass();
 });
+
+test.skip("partially compress zip test", async (t) => {
+  const dropletHandler = new DropletHandler();
+
+  const manifest = JSON.parse(
+    await new Promise((r, e) =>
+      generateManifest(
+        dropletHandler,
+        "./assets/my horror game.zip",
+        (_, __) => {},
+        (_, __) => {},
+        (err, manifest) => (err ? e(err) : r(manifest))
+      )
+    )
+  );
+
+  return t.pass();
+});

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "2.2.1",
+  "version": "2.3.1",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {

src/manifest.rs

@@ -80,17 +80,17 @@ pub fn generate_manifest<'a>(
 
           length += read;
 
-          if length >= CHUNK_SIZE {
-            break;
-          }
-
           // If we're out of data, add this chunk and then move onto the next file
           if read == 0 {
             file_empty = true;
             break;
           }
 
-          buffer.extend_from_slice(&buf[..read]);
+          buffer.extend_from_slice(&buf[0..read]);
+
+          if length >= CHUNK_SIZE {
+            break;
+          }
         }
 
         let chunk_id = Uuid::new_v4();

src/version/backends.rs

@@ -9,7 +9,8 @@ use std::{
 
 use flate2::read::DeflateDecoder;
 use rawzip::{
-  FileReader, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, ZipReader, RECOMMENDED_BUFFER_SIZE,
+  CompressionMethod, FileReader, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry,
+  ZipVerifier, RECOMMENDED_BUFFER_SIZE,
 };
 
 use crate::version::types::{MinimumFileObject, VersionBackend, VersionFile};
@@ -116,15 +117,27 @@ impl ZipVersionBackend {
   pub fn new_entry<'archive>(
     &self,
     entry: ZipEntry<'archive, FileReader>,
+    compression_method: CompressionMethod,
     start: u64,
     end: u64,
   ) -> ZipFileWrapper<'archive> {
-    let mut deflater = DeflateDecoder::new(entry.reader());
+    let deflater: Box<dyn Read + Send + 'archive> = match compression_method {
+      CompressionMethod::Store => Box::new(entry.reader()),
+      CompressionMethod::Deflate => Box::new(DeflateDecoder::new(entry.reader())),
+      CompressionMethod::Deflate64 => Box::new(DeflateDecoder::new(entry.reader())),
+      _ => panic!(
+        "unsupported decompression algorithm: {:?}",
+        compression_method
+      ),
+    };
+
+    let mut verifier = entry.verifying_reader(deflater);
     if start != 0 {
-      io::copy(&mut (&mut deflater).take(start), &mut Sink::default()).unwrap();
+      io::copy(&mut (&mut verifier).take(start), &mut Sink::default()).unwrap();
     }
+
     ZipFileWrapper {
-      reader: deflater,
+      reader: verifier,
       limit: (end - start) as usize,
       current: 0,
     }
@@ -132,7 +145,7 @@ impl ZipVersionBackend {
 }
 
 pub struct ZipFileWrapper<'archive> {
-  reader: DeflateDecoder<ZipReader<'archive, FileReader>>,
+  reader: ZipVerifier<'archive, Box<dyn Read + Send + 'archive>, FileReader>,
   limit: usize,
   current: usize,
 }
@@ -163,9 +176,13 @@ impl<'a> Read for ZipFileWrapper<'a> {
     return Ok(read);
   }
 }
+//impl<'a> MinimumFileObject for ZipFileWrapper<'a> {}
 
 impl ZipVersionBackend {
-  fn find_wayfinder(&mut self, filename: &str) -> Option<ZipArchiveEntryWayfinder> {
+  fn find_wayfinder(
+    &mut self,
+    filename: &str,
+  ) -> Option<(ZipArchiveEntryWayfinder, CompressionMethod)> {
     let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
     let mut entries = self.archive.entries(read_buffer);
     let entry = loop {
@@ -180,7 +197,7 @@ impl ZipVersionBackend {
 
     let wayfinder = entry.wayfinder();
 
-    Some(wayfinder)
+    Some((wayfinder, entry.compression_method()))
   }
 }
 impl VersionBackend for ZipVersionBackend {
@@ -207,16 +224,16 @@ impl VersionBackend for ZipVersionBackend {
     start: u64,
     end: u64,
   ) -> Option<Box<dyn MinimumFileObject + '_>> {
-    let wayfinder = self.find_wayfinder(&file.relative_filename)?;
+    let (wayfinder, compression_method) = self.find_wayfinder(&file.relative_filename)?;
     let local_entry = self.archive.get_entry(wayfinder).unwrap();
 
-    let wrapper = self.new_entry(local_entry, start, end);
+    let wrapper = self.new_entry(local_entry, compression_method, start, end);
 
-    Some(Box::new(wrapper))
+    Some(Box::new(wrapper) as Box<dyn MinimumFileObject>)
   }
 
   fn peek_file(&mut self, sub_path: String) -> Option<VersionFile> {
-    let entry = self.find_wayfinder(&sub_path)?;
+    let (entry, _) = self.find_wayfinder(&sub_path)?;
 
     Some(VersionFile {
       relative_filename: sub_path,