fix: skip zip test

__test__/debug.spec.mjs

@@ -0,0 +1,22 @@
+import test from "ava";
+import { DropletHandler, generateManifest } from "../index.js";
+
+test.skip("debug", async (t) => {
+  const handler = new DropletHandler();
+
+  console.log("created handler");
+
+  const manifest = JSON.parse(
+    await new Promise((r, e) =>
+      generateManifest(
+        handler,
+        "./assets/TheGame.zip",
+        (_, __) => {},
+        (_, __) => {},
+        (err, manifest) => (err ? e(err) : r(manifest))
+      )
+    )
+  );
+
+  return t.pass();
+});

__test__/utils.spec.mjs

@@ -1,6 +1,7 @@
 import test from "ava";
 import fs from "node:fs";
 import path from "path";
+import prettyBytes from "pretty-bytes";
 
 import droplet, { DropletHandler, generateManifest } from "../index.js";
 
@@ -78,7 +79,12 @@ test("read file offset", async (t) => {
   fs.writeFileSync(dirName + "/TESTFILE", testString);
 
   const dropletHandler = new DropletHandler();
-  const stream = dropletHandler.readFile(dirName, "TESTFILE", BigInt(1), BigInt(4));
+  const stream = dropletHandler.readFile(
+    dirName,
+    "TESTFILE",
+    BigInt(1),
+    BigInt(4)
+  );
 
   let finalString = "";
 
@@ -96,10 +102,45 @@ test("read file offset", async (t) => {
   fs.rmSync(dirName, { recursive: true });
 });
 
-test("zip file reader", async (t) => {
-  return t.pass();
+test.skip("zip speed test", async (t) => {
+  t.timeout(100_000_000);
+  const dropletHandler = new DropletHandler();
 
-  t.timeout(10_000);
+  const stream = dropletHandler.readFile("./assets/TheGame.zip", "setup.exe");
+
+  let totalRead = 0;
+  let totalSeconds = 0;
+
+  let lastTime = process.hrtime.bigint();
+  const timeThreshold = BigInt(1_000_000_000);
+  let runningTotal = 0;
+  let runningTime = BigInt(0);
+  for await (const chunk of stream.getStream()) {
+    // Do something with each 'chunk'
+    const currentTime = process.hrtime.bigint();
+    const timeDiff = currentTime - lastTime;
+    lastTime = currentTime;
+    runningTime += timeDiff;
+
+    runningTotal += chunk.length;
+
+    if (runningTime >= timeThreshold) {
+      console.log(`${prettyBytes(runningTotal)}/s`);
+      totalRead += runningTotal;
+      totalSeconds += 1;
+      runningTime = BigInt(0);
+      runningTotal = 0;
+    }
+  }
+
+  const roughAverage = totalRead / totalSeconds;
+
+  console.log(`total rough average: ${prettyBytes(roughAverage)}/s`);
+
+  t.pass();
+});
+
+test.skip("zip manifest test", async (t) => {
   const dropletHandler = new DropletHandler();
   const manifest = JSON.parse(
     await new Promise((r, e) =>
@@ -113,19 +154,11 @@ test("zip file reader", async (t) => {
     )
   );
 
-  const stream = dropletHandler.readFile(
-    "./assets/TheGame.zip",
-    "setup.exe",
-    BigInt(10),
-    BigInt(20)
-  );
+  const file = manifest[Object.keys(manifest).at(0)];
+  const amount = file.ids.length;
 
-
-  let finalString = "";
-  for await (const chunk of stream.getStream()) {
-    // Do something with each 'chunk'
-    finalString = String.fromCharCode.apply(null, chunk);
-    if(finalString.length > 100) break;
+  if(amount > 20) {
+    return t.fail(`Zip manifest has ${amount} chunks, more than 20`);
   }
 
   t.pass();

assets/generate.sh

@@ -1,3 +1,4 @@
-yes "droplet is awesome" | dd of=./setup.exe bs=1024 count=1000000
+# yes "droplet is awesome" | dd of=./setup.exe bs=1024 count=1000000
+dd if=/dev/random of=./setup.exe bs=1024 count=1000000
 zip TheGame.zip setup.exe
 rm setup.exe

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "2.0.1",
+  "version": "2.1.1",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -24,7 +24,9 @@
   "devDependencies": {
     "@napi-rs/cli": "3.0.0-alpha.91",
     "@types/node": "^22.13.10",
-    "ava": "^6.2.0"
+    "ava": "^6.2.0",
+    "pretty-bytes": "^7.0.1",
+    "tsimp": "^2.0.12"
   },
   "ava": {
     "timeout": "3m",
@@ -51,8 +53,5 @@
   "packageManager": "yarn@4.7.0",
   "repository": {
     "url": "git+https://github.com/Drop-OSS/droplet.git"
-  },
-  "dependencies": {
-    "tsimp": "^2.0.12"
   }
 }

src/manifest.rs

@@ -1,7 +1,6 @@
 use std::{
   collections::HashMap,
   io::{BufRead, BufReader},
-  path::Path,
   sync::Arc,
   thread,
 };
@@ -42,11 +41,11 @@ pub fn generate_manifest<'a>(
   log_sfn: ThreadsafeFunction<String>,
   callback_sfn: ThreadsafeFunction<String>,
 ) -> Result<()> {
-  let backend: &mut Box<dyn VersionBackend + Send> =
-    droplet_handler.create_backend_for_path(dir).ok_or(napi::Error::from_reason("Could not create backend for path."))?;
-  let backend: &'static mut Box<dyn VersionBackend + Send> =
-    unsafe { std::mem::transmute(backend) };
-  thread::spawn(move || {
+  let backend: &mut Box<dyn VersionBackend + Send> = droplet_handler
+    .create_backend_for_path(dir)
+    .ok_or(napi::Error::from_reason(
+    "Could not create backend for path.",
+  ))?;
     let files = backend.list_files();
 
     // Filepath to chunk data
@@ -56,8 +55,8 @@ pub fn generate_manifest<'a>(
     let mut i: i32 = 0;
 
     for version_file in files {
-      let raw_reader = backend.reader(&version_file).unwrap();
-      let mut reader = BufReader::with_capacity(CHUNK_SIZE, raw_reader);
+      let reader = backend.reader(&version_file).unwrap();
+      let mut reader = BufReader::with_capacity(8128, reader);
 
       let mut chunk_data = ChunkData {
         permissions: version_file.permission,
@@ -68,14 +67,30 @@ pub fn generate_manifest<'a>(
 
       let mut chunk_index = 0;
       loop {
+        let mut length = 0;
         let mut buffer: Vec<u8> = Vec::new();
-        reader.fill_buf().unwrap().clone_into(&mut buffer);
-        let length = buffer.len();
+        let mut file_empty = false;
 
-        if length == 0 {
+        loop {
+          let read_buf = reader.fill_buf().unwrap();
+          let buf_length = read_buf.len();
+
+          length += buf_length;
+
+          if length >= CHUNK_SIZE {
             break;
           }
 
+          // If we're out of data, add this chunk and then move onto the next file
+          if buf_length == 0 {
+            file_empty = true;
+            break;
+          }
+
+          buffer.extend_from_slice(read_buf);
+          reader.consume(length);
+        }
+
         let chunk_id = Uuid::new_v4();
         let checksum = md5::compute(buffer).0;
         let checksum_string = hex::encode(checksum);
@@ -88,10 +103,14 @@ pub fn generate_manifest<'a>(
           "Processed chunk {} for {}",
           chunk_index, &version_file.relative_filename
         );
+
         log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
 
-        reader.consume(length);
         chunk_index += 1;
+
+        if file_empty {
+          break;
+        }
       }
 
       chunks.insert(version_file.relative_filename, chunk_data);
@@ -105,7 +124,6 @@ pub fn generate_manifest<'a>(
       Ok(json!(chunks).to_string()),
       ThreadsafeFunctionCallMode::Blocking,
     );
-  });
 
   Ok(())
 }

src/version/backends.rs

@@ -121,7 +121,7 @@ impl<'a> Read for ZipFileWrapper<'a> {
 }
 impl<'a> Skippable for ZipFileWrapper<'a> {
   fn skip(&mut self, amount: u64) {
-    io::copy(&mut self.reader.by_ref().take(amount), &mut Sink::default()).unwrap();
+    io::copy(&mut self.take(amount), &mut Sink::default()).unwrap();
   }
 }
 impl<'a> MinimumFileObject for ZipFileWrapper<'a> {}

yarn.lock

@@ -12,6 +12,7 @@ __metadata:
     "@napi-rs/cli": "npm:3.0.0-alpha.91"
     "@types/node": "npm:^22.13.10"
     ava: "npm:^6.2.0"
+    pretty-bytes: "npm:^7.0.1"
     tsimp: "npm:^2.0.12"
   languageName: unknown
   linkType: soft
@@ -2432,6 +2433,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pretty-bytes@npm:^7.0.1":
+  version: 7.0.1
+  resolution: "pretty-bytes@npm:7.0.1"
+  checksum: 10c0/14ffb503d2de3588042c722848062a4897e6faece1694e0c83ba5669ec003d73311d946d50d2b3c6099a6a306760011b8446ee3cf9cf86eca13a454a8f1c47cb
+  languageName: node
+  linkType: hard
+
 "pretty-ms@npm:^9.1.0":
   version: 9.2.0
   resolution: "pretty-ms@npm:9.2.0"