fix: 7z and streams

__test__/utils.spec.mjs

@@ -67,7 +67,7 @@ test("read file", async (t) => {
 
   let finalString = "";
 
-  for await (const chunk of stream.getStream()) {
+  for await (const chunk of stream) {
     // Do something with each 'chunk'
     finalString += String.fromCharCode.apply(null, chunk);
   }
@@ -94,7 +94,7 @@ test("read file offset", async (t) => {
 
   let finalString = "";
 
-  for await (const chunk of stream.getStream()) {
+  for await (const chunk of stream) {
     // Do something with each 'chunk'
     finalString += String.fromCharCode.apply(null, chunk);
   }
@@ -121,7 +121,7 @@ test.skip("zip speed test", async (t) => {
   const timeThreshold = BigInt(1_000_000_000);
   let runningTotal = 0;
   let runningTime = BigInt(0);
-  for await (const chunk of stream.getStream()) {
+  for await (const chunk of stream) {
     // Do something with each 'chunk'
     const currentTime = process.hrtime.bigint();
     const timeDiff = currentTime - lastTime;
@@ -146,55 +146,61 @@ test.skip("zip speed test", async (t) => {
   t.pass();
 });
 
-test.skip("zip manifest test", async (t) => {
+test("zip manifest test", async (t) => {
+  const zipFiles = fs.readdirSync("./assets").filter((v) => v.endsWith(".zip"));
   const dropletHandler = new DropletHandler();
-  const manifest = JSON.parse(
-    await new Promise((r, e) =>
-      generateManifest(
-        dropletHandler,
-        "./assets/TheGame.zip",
-        (_, __) => {},
-        (_, __) => {},
-        (err, manifest) => (err ? e(err) : r(manifest))
-      )
-    )
-  );
 
-  for (const [filename, data] of Object.entries(manifest)) {
+  for (const zipFile of zipFiles) {
-    let start = 0;
+    console.log("generating manifest for " + zipFile);
-    for (const [chunkIndex, length] of data.lengths.entries()) {
+    const manifest = JSON.parse(
-      const hash = createHash("md5");
+      await new Promise((r, e) =>
-      const stream = (
+        generateManifest(
-        await dropletHandler.readFile(
+          dropletHandler,
-          "./assets/TheGame.zip",
+          "./assets/" + zipFile,
-          filename,
+          (_, __) => {},
-          BigInt(start),
+          (_, __) => {},
-          BigInt(start + length)
+          (err, manifest) => (err ? e(err) : r(manifest))
         )
-      ).getStream();
+      )
+    );
 
-      let streamLength = 0;
+    for (const [filename, data] of Object.entries(manifest)) {
-      await stream.pipeTo(
+      let start = 0;
-        new WritableStream({
+      for (const [chunkIndex, length] of data.lengths.entries()) {
-          write(chunk) {
+        const hash = createHash("md5");
-            streamLength += chunk.length;
+        const stream = (
-            hash.update(chunk);
+          await dropletHandler.readFile(
-          },
+            "./assets/" + zipFile,
-        })
+            filename,
-      );
+            BigInt(start),
+            BigInt(start + length)
+          )
+        );
+        console.log(stream);
 
-      if (streamLength != length)
+        let streamLength = 0;
-        return t.fail(
+        await stream.pipeTo(
-          `stream length for chunk index ${chunkIndex} was not expected: real: ${streamLength} vs expected: ${length}`
+          new WritableStream({
+            write(chunk) {
+              streamLength += chunk.length;
+              hash.update(chunk);
+            },
+          })
         );
 
-      const digest = hash.digest("hex");
+        if (streamLength != length)
-      if (data.checksums[chunkIndex] != digest)
+          return t.fail(
-        return t.fail(
+            `stream length for chunk index ${chunkIndex} was not expected: real: ${streamLength} vs expected: ${length}`
-          `checksums did not match for chunk index ${chunkIndex}: real: ${digest} vs expected: ${data.checksums[chunkIndex]}`
+          );
-        );
 
-      start += length;
+        const digest = hash.digest("hex");
+        if (data.checksums[chunkIndex] != digest)
+          return t.fail(
+            `checksums did not match for chunk index ${chunkIndex}: real: ${digest} vs expected: ${data.checksums[chunkIndex]}`
+          );
+
+        start += length;
+      }
     }
   }
 

assets/generate.sh

@@ -1,4 +1,4 @@
 # yes "droplet is awesome" | dd of=./setup.exe bs=1024 count=1000000
 dd if=/dev/random of=./setup.exe bs=1024 count=1000000
-zip TheGame.zip setup.exe
+zip TheGame.zip setup.exe "test file.txt"
 rm setup.exe

index.d.ts

@@ -8,11 +8,7 @@ export declare class DropletHandler {
   hasBackendForPath(path: string): boolean
   listFiles(path: string): Array<string>
   peekFile(path: string, subPath: string): bigint
-  readFile(path: string, subPath: string, start?: bigint | undefined | null, end?: bigint | undefined | null): JsDropStreamable
+  readFile(path: string, subPath: string, start?: bigint | undefined | null, end?: bigint | undefined | null): ReadableStream
-}
-
-export declare class JsDropStreamable {
-  getStream(): any
 }
 
 export declare class Script {

index.js

@@ -377,7 +377,6 @@ if (!nativeBinding) {
 
 module.exports = nativeBinding
 module.exports.DropletHandler = nativeBinding.DropletHandler
-module.exports.JsDropStreamable = nativeBinding.JsDropStreamable
 module.exports.Script = nativeBinding.Script
 module.exports.ScriptEngine = nativeBinding.ScriptEngine
 module.exports.callAltThreadFunc = nativeBinding.callAltThreadFunc

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "3.1.0",
+  "version": "3.4.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -20,7 +20,7 @@
       ]
     }
   },
-  "license": "MIT",
+  "license": "AGPL-3.0-only",
   "devDependencies": {
     "@napi-rs/cli": "3.0.0-alpha.91",
     "@types/node": "^22.13.10",

src/version/backends.rs

@@ -126,12 +126,15 @@ impl ZipVersionBackend {
 
 pub struct ZipFileWrapper {
   command: Child,
-  reader: BufReader<ChildStdout>
+  reader: BufReader<ChildStdout>,
 }
 
 impl ZipFileWrapper {
   pub fn new(mut command: Child) -> Self {
-    let stdout = command.stdout.take().expect("failed to access stdout of 7z");
+    let stdout = command
+      .stdout
+      .take()
+      .expect("failed to access stdout of 7z");
     let reader = BufReader::new(stdout);
     ZipFileWrapper { command, reader }
   }
@@ -148,9 +151,9 @@ impl Read for ZipFileWrapper {
 }
 
 impl Drop for ZipFileWrapper {
-    fn drop(&mut self) {
+  fn drop(&mut self) {
-        self.command.wait().expect("failed to wait for 7z exit");
+    self.command.wait().expect("failed to wait for 7z exit");
-    }
+  }
 }
 
 impl VersionBackend for ZipVersionBackend {
@@ -165,22 +168,29 @@ impl VersionBackend for ZipVersionBackend {
       ));
     }
     let raw_result = String::from_utf8(result.stdout)?;
-    let files = raw_result.split("\n").filter(|v| v.len() > 0).map(|v| v.split(" ").filter(|v| v.len() > 0));
+    let files = raw_result
+      .split("\n")
+      .filter(|v| v.len() > 0)
+      .map(|v| v.split(" ").filter(|v| v.len() > 0));
     let mut results = Vec::new();
 
-    for mut file in files {
+    for file in files {
+      let values = file.collect::<Vec<&str>>();
+      let mut iter = values.iter();
       let (date, time, attrs, size, compress, name) = (
-        file.next().unwrap(),
+        iter.next().expect("failed to read date"),
-        file.next().unwrap(),
+        iter.next().expect("failed to read time"),
-        file.next().unwrap(),
+        iter.next().expect("failed to read attrs"),
-        file.next().unwrap(),
+        iter.next().expect("failed to read size"),
-        file.next().unwrap(),
+        iter.next().expect("failed to read compress"),
-        file.next().unwrap(),
+        iter.collect::<Vec<&&str>>(),
       );
-      println!("got line: {} {} {} {} {} {}", date, time, attrs, size, compress, name);
+      if attrs.starts_with("D") {
+        continue;
+      }
       results.push(VersionFile {
-        relative_filename: name.to_owned(),
+        relative_filename: name.into_iter().map(|v| *v).fold(String::new(), |a, b| a + b + " ").trim_end().to_owned(),
-        permission: 0,
+        permission: 0o744, // owner r/w/x, everyone else, read
         size: size.parse().unwrap(),
       });
     }
@@ -196,7 +206,10 @@ impl VersionBackend for ZipVersionBackend {
   ) -> anyhow::Result<Box<dyn MinimumFileObject + '_>> {
     let mut read_command = Command::new("7z");
     read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]);
-    let output = read_command.stdout(Stdio::piped()).spawn().expect("failed to spawn 7z");
+    let output = read_command
+      .stdout(Stdio::piped())
+      .spawn()
+      .expect("failed to spawn 7z");
     Ok(Box::new(ZipFileWrapper::new(output)))
   }
 

src/version/utils.rs

@@ -38,9 +38,7 @@ pub fn create_backend_constructor<'a>(
     let status = test.status().ok()?;
     if status.code().unwrap_or(1) == 0 {
       let buf = path.to_path_buf();
-      return Some(Box::new(move || {
+      return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
-        Ok(Box::new(ZipVersionBackend::new(buf)?))
-      }));
     }
   }
 
@@ -111,7 +109,7 @@ impl<'a> DropletHandler<'a> {
     Ok(file.size)
   }
 
-  #[napi]
+  #[napi(ts_return_type = "ReadableStream")]
   pub fn read_file(
     &mut self,
     reference: Reference<DropletHandler<'static>>,
@@ -120,7 +118,7 @@ impl<'a> DropletHandler<'a> {
     env: Env,
     start: Option<BigInt>,
     end: Option<BigInt>,
-  ) -> anyhow::Result<JsDropStreamable> {
+  ) -> anyhow::Result<*mut napi_value__> {
     let stream = reference.share_with(env, |handler| {
       let backend = handler
         .create_backend_for_path(path)
@@ -149,25 +147,9 @@ impl<'a> DropletHandler<'a> {
             // Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
             .map_err(napi::Error::from) // napi::Error implements From<tokio::io::Error>
         });
-      // Create the napi-rs ReadableStream from the tokio_stream::Stream
-      // The unwrap() here means if stream creation fails, it will panic.
-      // For a production system, consider returning Result<Option<...>> and handling this.
       ReadableStream::create_with_stream_bytes(&env, stream)
     })?;
 
-    Ok(JsDropStreamable { inner: stream })
+    Ok(stream.raw())
-  }
-}
-
-#[napi]
-pub struct JsDropStreamable {
-  inner: SharedReference<DropletHandler<'static>, ReadableStream<'static, BufferSlice<'static>>>,
-}
-
-#[napi]
-impl JsDropStreamable {
-  #[napi]
-  pub fn get_stream(&self) -> *mut napi_value__ {
-    self.inner.raw()
   }
 }