fix: unix permissions properly fixed with 7z

Bump rawzip to 0.3

Cargo.toml

@@ -9,14 +9,9 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "3.0.0-beta.11", default-features = false, features = [
-    "napi4",
-    "async",
-    "web_stream",
-] }
+napi = { version = "3.0.0-beta.11", default-features = false, features = ["napi6", "async", "web_stream", "error_anyhow"] }
 napi-derive = "3.0.0-beta.11"
 hex = "0.4.3"
-serde_json = "1.0.128"
 md5 = "0.7.0"
 time-macros = "0.2.22"
 time = "0.3.41"
@@ -24,13 +19,12 @@ webpki = "0.22.4"
 ring = "0.17.14"
 tokio = { version = "1.45.1", features = ["fs", "io-util"] }
 tokio-util = { version = "0.7.15", features = ["codec"] }
-rawzip = "0.2.0"
-
-[package.metadata.patch]
-crates = ["rawzip"]
-
-[patch.crates-io]
-rawzip = { path="./target/patch/rawzip-0.2.0" }
+dyn-clone = "1.0.20"
+rhai = "1.22.2"
+# mlua = { version = "0.11.2", features = ["luajit"] }
+boa_engine = "0.20.0"
+serde_json = "1.0.143"
+anyhow = "1.0.99"
 
 [dependencies.x509-parser]
 version = "0.17.0"

__test__/debug.spec.mjs

@@ -0,0 +1,22 @@
+import test from "ava";
+import { DropletHandler, generateManifest } from "../index.js";
+
+test.skip("debug", async (t) => {
+  const handler = new DropletHandler();
+
+  console.log("created handler");
+
+  const manifest = JSON.parse(
+    await new Promise((r, e) =>
+      generateManifest(
+        handler,
+        "./assets/TheGame.zip",
+        (_, __) => {},
+        (_, __) => {},
+        (err, manifest) => (err ? e(err) : r(manifest))
+      )
+    )
+  );
+
+  return t.pass();
+});

__test__/manifest.spec.mjs

@@ -2,7 +2,7 @@ import test from "ava";
 import fs from "node:fs";
 import path from "path";
 
-import { generateManifest, listFiles } from "../index.js";
+import { DropletHandler, generateManifest } from "../index.js";
 
 test("numerous small file", async (t) => {
   // Setup test dir
@@ -18,9 +18,12 @@ test("numerous small file", async (t) => {
     fs.writeFileSync(fileName, i.toString());
   }
 
+  const dropletHandler = new DropletHandler();
+
   const manifest = JSON.parse(
     await new Promise((r, e) =>
       generateManifest(
+        dropletHandler,
         dirName,
         (_, __) => {},
         (_, __) => {},
@@ -56,7 +59,6 @@ test("numerous small file", async (t) => {
 
 test.skip("performance test", async (t) => {
   t.timeout(5 * 60 * 1000);
-  return t.pass();
   const dirName = "./.test/pt";
   if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true });
   fs.mkdirSync(dirName, { recursive: true });
@@ -73,9 +75,12 @@ test.skip("performance test", async (t) => {
     randomStream.on("end", r);
   });
 
+  const dropletHandler = new DropletHandler();
+
   const start = Date.now();
   await new Promise((r, e) =>
     generateManifest(
+      dropletHandler,
       dirName,
       (_, __) => {},
       (_, __) => {},
@@ -88,3 +93,47 @@ test.skip("performance test", async (t) => {
 
   fs.rmSync(dirName, { recursive: true });
 });
+
+test("special characters", async (t) => {
+  // Setup test dir
+  const dirName = "./.test/sc";
+  if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true });
+  fs.mkdirSync(dirName, { recursive: true });
+
+  // Config
+  const fileNames = ["Technická podpora.rtf", "Servicio técnico.rtf"];
+
+  for (let i = 0; i < fileNames.length; i++) {
+    const fileName = path.join(dirName, fileNames[i]);
+    fs.writeFileSync(fileName, i.toString());
+  }
+
+  const dropletHandler = new DropletHandler();
+
+  const manifest = JSON.parse(
+    await new Promise((r, e) =>
+      generateManifest(
+        dropletHandler,
+        dirName,
+        (_, __) => {},
+        (_, __) => {},
+        (err, manifest) => (err ? e(err) : r(manifest))
+      )
+    )
+  );
+
+  // Check the first few checksums
+  const checksums = [
+    "cfcd208495d565ef66e7dff9f98764da",
+    "c4ca4238a0b923820dcc509a6f75849b",
+  ];
+  for (let index in checksums) {
+    const entry = manifest[fileNames[index]];
+    if (!entry) return t.fail(`manifest missing file ${index}`);
+
+    const checksum = entry.checksums[0];
+    t.is(checksum, checksums[index], `checksums do not match for ${index}`);
+  }
+
+  fs.rmSync(dirName, { recursive: true });
+});

__test__/script.spec.mjs

@@ -0,0 +1,62 @@
+import test from "ava";
+import { ScriptEngine } from "../index.js";
+
+test.skip("lua syntax fail", (t) => {
+  const scriptEngine = new ScriptEngine();
+
+  const luaIshCode = `
+    print("hello world);
+    `;
+
+  try {
+    const script = scriptEngine.buildLuaScript(luaIshCode);
+  } catch {
+    return t.pass();
+  }
+  t.fail();
+});
+
+test("js syntax fail", (t) => {
+  const scriptEngine = new ScriptEngine();
+
+  const jsIshCode = `
+    const v = "hello world;
+    `;
+
+  try {
+    const script = scriptEngine.buildJsScript(jsIshCode);
+  } catch {
+    return t.pass();
+  }
+  t.fail();
+});
+
+test("js", (t) => {
+  const scriptEngine = new ScriptEngine();
+
+  const jsModule = `
+    const v = "1" + "2";
+    ["1", "2", "3", v]
+       `;
+
+  const script = scriptEngine.buildJsScript(jsModule);
+
+  scriptEngine.fetchStrings(script);
+
+  t.pass();
+});
+
+test.skip("lua", (t) => {
+  const scriptEngine = new ScriptEngine();
+
+  const luaModule = `
+    local arr = {"1", "2"};
+    return arr;
+    `;
+
+  const script = scriptEngine.buildLuaScript(luaModule);
+
+  scriptEngine.fetchStrings(script);
+
+  t.pass();
+});

__test__/utils.spec.mjs

@@ -1,8 +1,10 @@
 import test from "ava";
 import fs from "node:fs";
 import path from "path";
+import { createHash } from "node:crypto";
+import prettyBytes from "pretty-bytes";
 
-import droplet, { generateManifest } from "../index.js";
+import droplet, { DropletHandler, generateManifest } from "../index.js";
 
 test("check alt thread util", async (t) => {
   let endtime1, endtime2;
@@ -23,6 +25,28 @@ test("check alt thread util", async (t) => {
   t.pass();
 });
 
+test("list files", async (t) => {
+  const dirName = "./.listfiles";
+  if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true });
+  fs.mkdirSync(dirName, { recursive: true });
+  fs.mkdirSync(dirName + "/subdir", { recursive: true });
+  fs.mkdirSync(dirName + "/subddir", { recursive: true });
+
+  fs.writeFileSync(dirName + "/root.txt", "root");
+  fs.writeFileSync(dirName + "/subdir/one.txt", "the first subdir");
+  fs.writeFileSync(dirName + "/subddir/two.txt", "the second");
+
+  const dropletHandler = new DropletHandler();
+  const files = dropletHandler.listFiles(dirName);
+
+  t.assert(
+    files.sort().join("\n"),
+    ["root.txt", "subddir/two.txt", "subdir/one.txt"].join("\n")
+  );
+
+  fs.rmSync(dirName, { recursive: true });
+});
+
 test("read file", async (t) => {
   const dirName = "./.test2";
   if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true });
@@ -32,11 +56,18 @@ test("read file", async (t) => {
 
   fs.writeFileSync(dirName + "/TESTFILE", testString);
 
-  const stream = droplet.readFile(dirName, "TESTFILE");
+  const dropletHandler = new DropletHandler();
+
+  const stream = dropletHandler.readFile(
+    dirName,
+    "TESTFILE",
+    BigInt(0),
+    BigInt(testString.length)
+  );
 
   let finalString = "";
 
-  for await (const chunk of stream) {
+  for await (const chunk of stream.getStream()) {
     // Do something with each 'chunk'
     finalString += String.fromCharCode.apply(null, chunk);
   }
@@ -53,11 +84,17 @@ test("read file offset", async (t) => {
   const testString = "0123456789";
   fs.writeFileSync(dirName + "/TESTFILE", testString);
 
-  const stream = droplet.readFile(dirName, "TESTFILE", 1, 4);
+  const dropletHandler = new DropletHandler();
+  const stream = dropletHandler.readFile(
+    dirName,
+    "TESTFILE",
+    BigInt(1),
+    BigInt(4)
+  );
 
   let finalString = "";
 
-  for await (const chunk of stream) {
+  for await (const chunk of stream.getStream()) {
     // Do something with each 'chunk'
     finalString += String.fromCharCode.apply(null, chunk);
   }
@@ -71,11 +108,50 @@ test("read file offset", async (t) => {
   fs.rmSync(dirName, { recursive: true });
 });
 
-test("zip file reader", async (t) => {
-  return t.pass();
+test.skip("zip speed test", async (t) => {
+  t.timeout(100_000_000);
+  const dropletHandler = new DropletHandler();
+
+  const stream = dropletHandler.readFile("./assets/TheGame.zip", "setup.exe");
+
+  let totalRead = 0;
+  let totalSeconds = 0;
+
+  let lastTime = process.hrtime.bigint();
+  const timeThreshold = BigInt(1_000_000_000);
+  let runningTotal = 0;
+  let runningTime = BigInt(0);
+  for await (const chunk of stream.getStream()) {
+    // Do something with each 'chunk'
+    const currentTime = process.hrtime.bigint();
+    const timeDiff = currentTime - lastTime;
+    lastTime = currentTime;
+    runningTime += timeDiff;
+
+    runningTotal += chunk.length;
+
+    if (runningTime >= timeThreshold) {
+      console.log(`${prettyBytes(runningTotal)}/s`);
+      totalRead += runningTotal;
+      totalSeconds += 1;
+      runningTime = BigInt(0);
+      runningTotal = 0;
+    }
+  }
+
+  const roughAverage = totalRead / totalSeconds;
+
+  console.log(`total rough average: ${prettyBytes(roughAverage)}/s`);
+
+  t.pass();
+});
+
+test.skip("zip manifest test", async (t) => {
+  const dropletHandler = new DropletHandler();
   const manifest = JSON.parse(
     await new Promise((r, e) =>
       generateManifest(
+        dropletHandler,
         "./assets/TheGame.zip",
         (_, __) => {},
         (_, __) => {},
@@ -84,17 +160,61 @@ test("zip file reader", async (t) => {
     )
   );
 
-  console.log(manifest);
+  for (const [filename, data] of Object.entries(manifest)) {
+    let start = 0;
+    for (const [chunkIndex, length] of data.lengths.entries()) {
+      const hash = createHash("md5");
+      const stream = (
+        await dropletHandler.readFile(
+          "./assets/TheGame.zip",
+          filename,
+          BigInt(start),
+          BigInt(start + length)
+        )
+      ).getStream();
 
-  return t.pass();
-  const stream = droplet.readFile("./assets/TheGame.zip", "TheGame/setup.exe");
+      let streamLength = 0;
+      await stream.pipeTo(
+        new WritableStream({
+          write(chunk) {
+            streamLength += chunk.length;
+            hash.update(chunk);
+          },
+        })
+      );
 
-  let finalString;
-  for await (const chunk of stream) {
-    console.log(`read chunk ${chunk}`);
-    // Do something with each 'chunk'
-    finalString += String.fromCharCode.apply(null, chunk);
+      if (streamLength != length)
+        return t.fail(
+          `stream length for chunk index ${chunkIndex} was not expected: real: ${streamLength} vs expected: ${length}`
+        );
+
+      const digest = hash.digest("hex");
+      if (data.checksums[chunkIndex] != digest)
+        return t.fail(
+          `checksums did not match for chunk index ${chunkIndex}: real: ${digest} vs expected: ${data.checksums[chunkIndex]}`
+        );
+
+      start += length;
+    }
   }
 
-  console.log(finalString);
+  t.pass();
+});
+
+test.skip("partially compress zip test", async (t) => {
+  const dropletHandler = new DropletHandler();
+
+  const manifest = JSON.parse(
+    await new Promise((r, e) =>
+      generateManifest(
+        dropletHandler,
+        "./assets/my horror game.zip",
+        (_, __) => {},
+        (_, __) => {},
+        (err, manifest) => (err ? e(err) : r(manifest))
+      )
+    )
+  );
+
+  return t.pass();
 });

assets/generate.sh

@@ -1,3 +1,4 @@
+# yes "droplet is awesome" | dd of=./setup.exe bs=1024 count=1000000
 dd if=/dev/random of=./setup.exe bs=1024 count=1000000
 zip TheGame.zip setup.exe
 rm setup.exe

index.d.ts

@@ -1,24 +1,40 @@
 /* auto-generated by NAPI-RS */
 /* eslint-disable */
+/**
+ * Persistent object so we can cache things between commands
+ */
+export declare class DropletHandler {
+  constructor()
+  hasBackendForPath(path: string): boolean
+  listFiles(path: string): Array<string>
+  peekFile(path: string, subPath: string): bigint
+  readFile(path: string, subPath: string, start?: bigint | undefined | null, end?: bigint | undefined | null): JsDropStreamable
+}
+
+export declare class JsDropStreamable {
+  getStream(): any
+}
+
+export declare class Script {
+
+}
+
+export declare class ScriptEngine {
+  constructor()
+  buildRhaiScript(content: string): Script
+  buildJsScript(content: string): Script
+  execute(script: Script): void
+  fetchStrings(script: Script): Array<string>
+}
+
 export declare function callAltThreadFunc(tsfn: ((err: Error | null, ) => any)): void
 
 export declare function generateClientCertificate(clientId: string, clientName: string, rootCa: string, rootCaPrivate: string): Array<string>
 
-export declare function generateManifest(dir: string, progressSfn: ((err: Error | null, arg: number) => any), logSfn: ((err: Error | null, arg: string) => any), callbackSfn: ((err: Error | null, arg: string) => any)): void
+export declare function generateManifest(dropletHandler: DropletHandler, dir: string, progressSfn: ((err: Error | null, arg: number) => any), logSfn: ((err: Error | null, arg: string) => any), callbackSfn: ((err: Error | null, arg: string) => any)): void
 
 export declare function generateRootCa(): Array<string>
 
-export declare function hasBackendForPath(path: string): boolean
-
-export declare function listFiles(path: string): Array<string>
-
-/**
- * This is inefficient, but is used in attempt to keep the interface simple
- */
-export declare function peekFile(path: string, subPath: string): number
-
-export declare function readFile(path: string, subPath: string, start?: number | undefined | null, end?: number | undefined | null): ReadableStream<Buffer> | null
-
 export declare function signNonce(privateKey: string, nonce: string): string
 
 export declare function verifyClientCertificate(clientCert: string, rootCa: string): boolean

index.js

@@ -376,14 +376,14 @@ if (!nativeBinding) {
 }
 
 module.exports = nativeBinding
+module.exports.DropletHandler = nativeBinding.DropletHandler
+module.exports.JsDropStreamable = nativeBinding.JsDropStreamable
+module.exports.Script = nativeBinding.Script
+module.exports.ScriptEngine = nativeBinding.ScriptEngine
 module.exports.callAltThreadFunc = nativeBinding.callAltThreadFunc
 module.exports.generateClientCertificate = nativeBinding.generateClientCertificate
 module.exports.generateManifest = nativeBinding.generateManifest
 module.exports.generateRootCa = nativeBinding.generateRootCa
-module.exports.hasBackendForPath = nativeBinding.hasBackendForPath
-module.exports.listFiles = nativeBinding.listFiles
-module.exports.peekFile = nativeBinding.peekFile
-module.exports.readFile = nativeBinding.readFile
 module.exports.signNonce = nativeBinding.signNonce
 module.exports.verifyClientCertificate = nativeBinding.verifyClientCertificate
 module.exports.verifyNonce = nativeBinding.verifyNonce

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "1.5.3",
+  "version": "3.2.2",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -20,14 +20,23 @@
       ]
     }
   },
-  "license": "MIT",
+  "license": "AGPL-3.0-only",
   "devDependencies": {
     "@napi-rs/cli": "3.0.0-alpha.91",
     "@types/node": "^22.13.10",
-    "ava": "^6.2.0"
+    "ava": "^6.2.0",
+    "pretty-bytes": "^7.0.1",
+    "tsimp": "^2.0.12"
   },
   "ava": {
-    "timeout": "3m"
+    "timeout": "3m",
+    "extensions": [
+      "cjs",
+      "mjs",
+      "js",
+      "ts",
+      "mts"
+    ]
   },
   "engines": {
     "node": ">= 10"
@@ -37,7 +46,7 @@
     "build": "napi build --platform --release",
     "build:debug": "napi build --platform",
     "prepublishOnly": "napi prepublish -t npm",
-    "test": "ava",
+    "test": "ava ",
     "universal": "napi universalize",
     "version": "napi version"
   },

patches/rawzip+0.2.0.patch

@@ -1,26 +0,0 @@
-diff --git a/src/archive.rs b/src/archive.rs
-index 1203015..837c405 100644
---- a/src/archive.rs
-+++ b/src/archive.rs
-@@ -275,7 +275,7 @@ impl<'data> Iterator for ZipSliceEntries<'data> {
- /// ```
- #[derive(Debug, Clone)]
- pub struct ZipArchive<R> {
--    pub(crate) reader: R,
-+    pub reader: R,
-     pub(crate) comment: ZipString,
-     pub(crate) eocd: EndOfCentralDirectory,
- }
-@@ -431,9 +431,9 @@ where
- #[derive(Debug, Clone)]
- pub struct ZipEntry<'archive, R> {
-     archive: &'archive ZipArchive<R>,
--    body_offset: u64,
--    body_end_offset: u64,
--    entry: ZipArchiveEntryWayfinder,
-+    pub body_offset: u64,
-+    pub body_end_offset: u64,
-+    pub entry: ZipArchiveEntryWayfinder,
- }
- 
- impl<'archive, R> ZipEntry<'archive, R>

src/lib.rs

@@ -1,9 +1,14 @@
-#![deny(clippy::all)]
+#![deny(clippy::unwrap_used)]
+#![deny(clippy::expect_used)]
+#![deny(clippy::panic)]
 #![feature(trait_alias)]
+#![feature(iterator_try_collect)]
+
 
 pub mod manifest;
+pub mod script;
 pub mod ssl;
 pub mod version;
 
 #[macro_use]
-extern crate napi_derive;
+extern crate napi_derive;

src/manifest.rs

@@ -1,10 +1,4 @@
-use std::{
-  collections::HashMap,
-  io::{BufRead, BufReader},
-  path::Path,
-  sync::Arc,
-  thread,
-};
+use std::{collections::HashMap, sync::Arc, thread};
 
 use napi::{
   threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
@@ -13,8 +7,7 @@ use napi::{
 use serde_json::json;
 use uuid::Uuid;
 
-use crate::version::utils::create_backend_for_path;
-
+use crate::version::{types::VersionBackend, utils::DropletHandler};
 
 const CHUNK_SIZE: usize = 1024 * 1024 * 64;
 
@@ -36,74 +29,117 @@ pub fn call_alt_thread_func(tsfn: Arc<ThreadsafeFunction<()>>) -> Result<(), Str
 }
 
 #[napi]
-pub fn generate_manifest(
+pub fn generate_manifest<'a>(
+  droplet_handler: &mut DropletHandler,
   dir: String,
   progress_sfn: ThreadsafeFunction<i32>,
   log_sfn: ThreadsafeFunction<String>,
   callback_sfn: ThreadsafeFunction<String>,
-) -> Result<(), String> {
+) -> anyhow::Result<()> {
+  let backend: &mut Box<dyn VersionBackend + Send> = droplet_handler
+    .create_backend_for_path(dir)
+    .ok_or(napi::Error::from_reason(
+    "Could not create backend for path.",
+  ))?;
+
+  // This is unsafe (obviously)
+  // But it's allg as long the DropletHandler doesn't get
+  // dropped while we're generating the manifest.
+  let backend: &'static mut Box<dyn VersionBackend + Send> =
+    unsafe { std::mem::transmute(backend) };
+
+    let required_single_file = backend.require_whole_files();
+
   thread::spawn(move || {
-    let base_dir = Path::new(&dir);
-    let mut backend = create_backend_for_path(base_dir).unwrap();
-    let files = backend.list_files();
+    let callback_borrow = &callback_sfn;
 
-    // Filepath to chunk data
-    let mut chunks: HashMap<String, ChunkData> = HashMap::new();
+    let mut inner = move || -> Result<()> {
+      let files = backend.list_files()?;
 
-    let total: i32 = files.len() as i32;
-    let mut i: i32 = 0;
+      // Filepath to chunk data
+      let mut chunks: HashMap<String, ChunkData> = HashMap::new();
 
-    for version_file in files {
-      let raw_reader= backend.reader(&version_file).unwrap();
-      let mut reader = BufReader::with_capacity(CHUNK_SIZE, raw_reader);
+      let total: i32 = files.len() as i32;
+      let mut i: i32 = 0;
 
-      let mut chunk_data = ChunkData {
-        permissions: version_file.permission,
-        ids: Vec::new(),
-        checksums: Vec::new(),
-        lengths: Vec::new(),
-      };
+      let mut buf = [0u8; 1024 * 16];
 
-      let mut chunk_index = 0;
-      loop {
-        let mut buffer: Vec<u8> = Vec::new();
-        reader.fill_buf().unwrap().clone_into(&mut buffer);
-        let length = buffer.len();
+      for version_file in files {
+        let mut reader = backend.reader(&version_file, 0, 0)?;
 
-        if length == 0 {
-          break;
+        let mut chunk_data = ChunkData {
+          permissions: version_file.permission,
+          ids: Vec::new(),
+          checksums: Vec::new(),
+          lengths: Vec::new(),
+        };
+
+        let mut chunk_index = 0;
+        loop {
+          let mut length = 0;
+          let mut buffer: Vec<u8> = Vec::new();
+          let mut file_empty = false;
+
+          loop {
+            let read = reader.read(&mut buf)?;
+
+            length += read;
+
+            // If we're out of data, add this chunk and then move onto the next file
+            if read == 0 {
+              file_empty = true;
+              break;
+            }
+
+            buffer.extend_from_slice(&buf[0..read]);
+
+            if length >= CHUNK_SIZE && !required_single_file {
+              break;
+            }
+          }
+
+          let chunk_id = Uuid::new_v4();
+          let checksum = md5::compute(buffer).0;
+          let checksum_string = hex::encode(checksum);
+
+          chunk_data.ids.push(chunk_id.to_string());
+          chunk_data.checksums.push(checksum_string);
+          chunk_data.lengths.push(length);
+
+          let log_str = format!(
+            "Processed chunk {} for {}",
+            chunk_index, &version_file.relative_filename
+          );
+
+          log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
+
+          chunk_index += 1;
+
+          if file_empty {
+            break;
+          }
         }
 
-        let chunk_id = Uuid::new_v4();
-        let checksum = md5::compute(buffer).0;
-        let checksum_string = hex::encode(checksum);
+        chunks.insert(version_file.relative_filename, chunk_data);
 
-        chunk_data.ids.push(chunk_id.to_string());
-        chunk_data.checksums.push(checksum_string);
-        chunk_data.lengths.push(length);
-
-        let log_str = format!(
-          "Processed chunk {} for {}",
-          chunk_index, &version_file.relative_filename
-        );
-        log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
-
-        reader.consume(length);
-        chunk_index += 1;
+        i += 1;
+        let progress = i * 100 / total;
+        progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
       }
 
-      chunks.insert(version_file.relative_filename, chunk_data);
+      callback_borrow.call(
+        Ok(json!(chunks).to_string()),
+        ThreadsafeFunctionCallMode::Blocking,
+      );
 
-      i += 1;
-      let progress = i * 100 / total;
-      progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
+      Ok(())
+    };
+
+    let result = inner();
+    if let Err(generate_err) = result {
+      callback_borrow.call(Err(generate_err), ThreadsafeFunctionCallMode::Blocking);
     }
-
-    callback_sfn.call(
-      Ok(json!(chunks).to_string()),
-      ThreadsafeFunctionCallMode::Blocking,
-    );
   });
 
   Ok(())
-}
+}

src/script/mod.rs

@@ -0,0 +1,133 @@
+use boa_engine::{Context, JsValue, Source};
+// use mlua::{FromLuaMulti, Function, Lua};
+use napi::Result;
+use rhai::AST;
+
+pub enum ScriptType {
+  Rhai,
+  Lua,
+  Javascript,
+}
+
+#[napi]
+pub struct Script(ScriptInner);
+
+pub enum ScriptInner {
+  Rhai { script: AST },
+  // Lua { script: Function },
+  Javascript { script: boa_engine::Script },
+}
+
+#[napi]
+pub struct ScriptEngine {
+  rhai_engine: rhai::Engine,
+  // lua_engine: Lua,
+  js_engine: Context,
+}
+
+#[napi]
+impl ScriptEngine {
+  #[napi(constructor)]
+  pub fn new() -> Self {
+    ScriptEngine {
+      rhai_engine: rhai::Engine::new(),
+      // lua_engine: Lua::new(),
+      js_engine: Context::default(),
+    }
+  }
+
+  #[napi]
+  pub fn build_rhai_script(&self, content: String) -> Result<Script> {
+    let script = self
+      .rhai_engine
+      .compile(content.clone())
+      .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+    Ok(Script(ScriptInner::Rhai { script }))
+  }
+
+  /*
+  #[napi]
+  pub fn build_lua_script(&self, content: String) -> Result<Script> {
+    let func = self
+      .lua_engine
+      .load(content.clone())
+      .into_function()
+      .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+    Ok(Script(ScriptInner::Lua { script: func }))
+  }
+  */
+
+  #[napi]
+  pub fn build_js_script(&mut self, content: String) -> Result<Script> {
+    let source = Source::from_bytes(content.as_bytes());
+    let script = boa_engine::Script::parse(source, None, &mut self.js_engine)
+      .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+
+    Ok(Script(ScriptInner::Javascript { script }))
+  }
+
+  fn execute_rhai_script<T>(&self, ast: &AST) -> Result<T>
+  where
+    T: Clone + 'static,
+  {
+    let v = self
+      .rhai_engine
+      .eval_ast::<T>(ast)
+      .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+
+    Ok(v)
+  }
+
+  /*
+  fn execute_lua_script<T>(&self, function: &Function) -> Result<T>
+  where
+    T: FromLuaMulti,
+  {
+    let v = function
+      .call::<T>(())
+      .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+    Ok(v)
+  }
+   */
+
+  fn execute_js_script(&mut self, func: &boa_engine::Script) -> Result<JsValue> {
+    let v = func
+      .evaluate(&mut self.js_engine)
+      .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+
+    Ok(v)
+  }
+
+  #[napi]
+  pub fn execute(&mut self, script: &mut Script) -> Result<()> {
+    match &script.0 {
+      ScriptInner::Rhai { script } => {
+        self.execute_rhai_script::<()>(script)?;
+      }
+      /*ScriptInner::Lua { script } => {
+        self.execute_lua_script::<()>(script)?;
+      }*/
+      ScriptInner::Javascript { script } => {
+        self.execute_js_script(script)?;
+      }
+    };
+    Ok(())
+  }
+
+  #[napi]
+  pub fn fetch_strings(&mut self, script: &mut Script) -> Result<Vec<String>> {
+    Ok(match &script.0 {
+      ScriptInner::Rhai { script } => self.execute_rhai_script(script)?,
+      //ScriptInner::Lua { script } => self.execute_lua_script(script)?,
+      ScriptInner::Javascript { script } => {
+        let v = self.execute_js_script(script)?;
+
+        serde_json::from_value(
+          v.to_json(&mut self.js_engine)
+            .map_err(|e| napi::Error::from_reason(e.to_string()))?,
+        )
+        .map_err(|e| napi::Error::from_reason(e.to_string()))?
+      }
+    })
+  }
+}

src/ssl.rs

@@ -1,4 +1,4 @@
-use napi::Error;
+use anyhow::anyhow;
 use rcgen::{
   CertificateParams, DistinguishedName, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData,
   SubjectPublicKeyInfo,
@@ -10,7 +10,7 @@ use x509_parser::parse_x509_certificate;
 use x509_parser::pem::Pem;
 
 #[napi]
-pub fn generate_root_ca() -> Result<Vec<String>, Error> {
+pub fn generate_root_ca() -> anyhow::Result<Vec<String>> {
   let mut params = CertificateParams::default();
 
   let mut name = DistinguishedName::new();
@@ -22,7 +22,7 @@ pub fn generate_root_ca() -> Result<Vec<String>, Error> {
   params.not_before = OffsetDateTime::now_utc();
   params.not_after = OffsetDateTime::now_utc()
     .checked_add(Duration::days(365 * 1000))
-    .unwrap();
+    .ok_or(anyhow!("failed to calculate end date"))?;
 
   params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
 
@@ -32,9 +32,8 @@ pub fn generate_root_ca() -> Result<Vec<String>, Error> {
     KeyUsagePurpose::DigitalSignature,
   ];
 
-  let key_pair = KeyPair::generate().map_err(|e| napi::Error::from_reason(e.to_string()))?;
-  let certificate = CertificateParams::self_signed(params, &key_pair)
-    .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+  let key_pair = KeyPair::generate()?;
+  let certificate = CertificateParams::self_signed(params, &key_pair)?;
 
   // Returns certificate, then private key
   Ok(vec![certificate.pem(), key_pair.serialize_pem()])
@@ -46,13 +45,10 @@ pub fn generate_client_certificate(
   _client_name: String,
   root_ca: String,
   root_ca_private: String,
-) -> Result<Vec<String>, Error> {
-  let root_key_pair =
-    KeyPair::from_pem(&root_ca_private).map_err(|e| napi::Error::from_reason(e.to_string()))?;
-  let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca)
-    .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-  let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair)
-    .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+) -> anyhow::Result<Vec<String>> {
+  let root_key_pair = KeyPair::from_pem(&root_ca_private)?;
+  let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca)?;
+  let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair)?;
 
   let mut params = CertificateParams::default();
 
@@ -66,28 +62,24 @@ pub fn generate_client_certificate(
     KeyUsagePurpose::DataEncipherment,
   ];
 
-  let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384)
-    .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-  let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair)
-    .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+  let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384)?;
+  let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair)?;
 
   // Returns certificate, then private key
   Ok(vec![certificate.pem(), key_pair.serialize_pem()])
 }
 
 #[napi]
-pub fn verify_client_certificate(client_cert: String, root_ca: String) -> Result<bool, Error> {
+pub fn verify_client_certificate(client_cert: String, root_ca: String) -> anyhow::Result<bool> {
   let root_ca = Pem::iter_from_buffer(root_ca.as_bytes())
     .next()
-    .unwrap()
-    .unwrap();
-  let root_ca = root_ca.parse_x509().unwrap();
+    .ok_or(anyhow!("no certificates in root ca"))??;
+  let root_ca = root_ca.parse_x509()?;
 
   let client_cert = Pem::iter_from_buffer(client_cert.as_bytes())
     .next()
-    .unwrap()
-    .unwrap();
-  let client_cert = client_cert.parse_x509().unwrap();
+    .ok_or(anyhow!("No client certs in chain."))??;
+  let client_cert = client_cert.parse_x509()?;
 
   let valid = root_ca
     .verify_signature(Some(client_cert.public_key()))
@@ -97,31 +89,33 @@ pub fn verify_client_certificate(client_cert: String, root_ca: String) -> Result
 }
 
 #[napi]
-pub fn sign_nonce(private_key: String, nonce: String) -> Result<String, Error> {
+pub fn sign_nonce(private_key: String, nonce: String) -> anyhow::Result<String> {
   let rng = SystemRandom::new();
 
-  let key_pair = KeyPair::from_pem(&private_key).unwrap();
+  let key_pair = KeyPair::from_pem(&private_key)?;
 
   let key_pair = EcdsaKeyPair::from_pkcs8(
     &ring::signature::ECDSA_P384_SHA384_FIXED_SIGNING,
     &key_pair.serialize_der(),
     &rng,
   )
-  .unwrap();
+  .map_err(|e| napi::Error::from_reason(e.to_string()))?;
 
-  let signature = key_pair.sign(&rng, nonce.as_bytes()).unwrap();
+  let signature = key_pair
+    .sign(&rng, nonce.as_bytes())
+    .map_err(|e| napi::Error::from_reason(e.to_string()))?;
   let hex_signature = hex::encode(signature);
 
   Ok(hex_signature)
 }
 
 #[napi]
-pub fn verify_nonce(public_cert: String, nonce: String, signature: String) -> Result<bool, Error> {
-  let (_, pem) = x509_parser::pem::parse_x509_pem(public_cert.as_bytes()).unwrap();
-  let (_, spki) = parse_x509_certificate(&pem.contents).unwrap();
-  let public_key = SubjectPublicKeyInfo::from_der(spki.public_key().raw).unwrap();
+pub fn verify_nonce(public_cert: String, nonce: String, signature: String) -> anyhow::Result<bool> {
+  let (_, pem) = x509_parser::pem::parse_x509_pem(public_cert.as_bytes())?;
+  let (_, spki) = parse_x509_certificate(&pem.contents)?;
+  let public_key = SubjectPublicKeyInfo::from_der(spki.public_key().raw)?;
 
-  let raw_signature = hex::decode(signature).unwrap();
+  let raw_signature = hex::decode(signature)?;
 
   let valid = ring::signature::ECDSA_P384_SHA384_FIXED
     .verify(

src/version/backends.rs

@@ -1,151 +1,219 @@
-use core::arch;
 #[cfg(unix)]
 use std::os::unix::fs::PermissionsExt;
 use std::{
-  fs::File,
-  io::{self, Read, Seek},
-  path::PathBuf,
-  pin::Pin,
-  rc::Rc,
-  sync::Arc,
+  cell::LazyCell,
+  fs::{self, metadata, File},
+  io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink},
+  path::{Path, PathBuf},
+  process::{Child, ChildStdout, Command, Stdio},
+  sync::{Arc, LazyLock},
 };
 
-use rawzip::{
-  FileReader, ReaderAt, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, RECOMMENDED_BUFFER_SIZE,
-};
+use anyhow::anyhow;
 
-use crate::version::{
-  types::{MinimumFileObject, Skippable, VersionBackend, VersionFile},
-  utils::_list_files,
-};
+use crate::version::types::{MinimumFileObject, VersionBackend, VersionFile};
 
+pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) -> napi::Result<()> {
+  if metadata(path)?.is_dir() {
+    let paths = fs::read_dir(path)?;
+    for path_result in paths {
+      let full_path = path_result?.path();
+      if metadata(&full_path)?.is_dir() {
+        _list_files(vec, &full_path)?;
+      } else {
+        vec.push(full_path);
+      }
+    }
+  };
+
+  Ok(())
+}
+
+#[derive(Clone)]
 pub struct PathVersionBackend {
   pub base_dir: PathBuf,
 }
 impl VersionBackend for PathVersionBackend {
-  fn list_files(&mut self) -> Vec<VersionFile> {
+  fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
     let mut vec = Vec::new();
-    _list_files(&mut vec, &self.base_dir);
+    _list_files(&mut vec, &self.base_dir)?;
 
     let mut results = Vec::new();
 
     for pathbuf in vec.iter() {
-      let file = File::open(pathbuf.clone()).unwrap();
-      let relative = pathbuf.strip_prefix(self.base_dir.clone()).unwrap();
-      let metadata = file.try_clone().unwrap().metadata().unwrap();
-      let permission_object = metadata.permissions();
-      let permissions = {
-        let perm: u32;
-        #[cfg(target_family = "unix")]
-        {
-          perm = permission_object.mode();
-        }
-        #[cfg(not(target_family = "unix"))]
-        {
-          perm = 0
-        }
-        perm
-      };
+      let relative = pathbuf.strip_prefix(self.base_dir.clone())?;
 
-      results.push(VersionFile {
-        relative_filename: relative.to_string_lossy().to_string(),
-        permission: permissions,
-        size: metadata.len(),
-      });
+      results.push(
+        self.peek_file(
+          relative
+            .to_str()
+            .ok_or(napi::Error::from_reason("Could not parse path"))?
+            .to_owned(),
+        )?,
+      );
     }
 
-    results
+    Ok(results)
   }
 
-  fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject + 'static)>> {
-    let file = File::open(self.base_dir.join(file.relative_filename.clone())).ok()?;
+  fn reader(
+    &mut self,
+    file: &VersionFile,
+    start: u64,
+    end: u64,
+  ) -> anyhow::Result<Box<dyn MinimumFileObject + 'static>> {
+    let mut file = File::open(self.base_dir.join(file.relative_filename.clone()))?;
 
-    return Some(Box::new(file));
+    if start != 0 {
+      file.seek(SeekFrom::Start(start))?;
+    }
+
+    if end != 0 {
+      return Ok(Box::new(file.take(end - start)));
+    }
+
+    Ok(Box::new(file))
+  }
+
+  fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
+    let pathbuf = self.base_dir.join(sub_path.clone());
+    if !pathbuf.exists() {
+      return Err(anyhow!("Path doesn't exist."));
+    };
+
+    let file = File::open(pathbuf.clone())?;
+    let metadata = file.try_clone()?.metadata()?;
+    let permission_object = metadata.permissions();
+    let permissions = {
+      let perm: u32;
+      #[cfg(target_family = "unix")]
+      {
+        perm = permission_object.mode();
+      }
+      #[cfg(not(target_family = "unix"))]
+      {
+        perm = 0
+      }
+      perm
+    };
+
+    Ok(VersionFile {
+      relative_filename: sub_path,
+      permission: permissions,
+      size: metadata.len(),
+    })
+  }
+
+  fn require_whole_files(&self) -> bool {
+    false
   }
 }
 
+pub static SEVEN_ZIP_INSTALLED: LazyLock<bool> =
+  LazyLock::new(|| Command::new("7z").output().is_ok());
+
+#[derive(Clone)]
 pub struct ZipVersionBackend {
-  archive: Arc<ZipArchive<FileReader>>,
+  path: String,
 }
 impl ZipVersionBackend {
-  pub fn new(archive: File) -> Self {
-    let archive = ZipArchive::from_file(archive, &mut [0u8; RECOMMENDED_BUFFER_SIZE]).unwrap();
-    Self {
-      archive: Arc::new(archive),
-    }
-  }
-
-  pub fn new_entry(&self, entry: ZipEntry<'_, FileReader>) -> ZipFileWrapper {
-    ZipFileWrapper {
-      archive: self.archive.clone(),
-      wayfinder: entry.entry,
-      offset: entry.body_offset,
-      end_offset: entry.body_end_offset,
-    }
+  pub fn new(path: PathBuf) -> anyhow::Result<Self> {
+    Ok(Self {
+      path: path.to_str().expect("invalid utf path").to_owned(),
+    })
   }
 }
 
 pub struct ZipFileWrapper {
-  pub archive: Arc<ZipArchive<FileReader>>,
-  wayfinder: ZipArchiveEntryWayfinder,
-  offset: u64,
-  end_offset: u64,
+  command: Child,
+  reader: BufReader<ChildStdout>
 }
 
+impl ZipFileWrapper {
+  pub fn new(mut command: Child) -> Self {
+    let stdout = command.stdout.take().expect("failed to access stdout of 7z");
+    let reader = BufReader::new(stdout);
+    ZipFileWrapper { command, reader }
+  }
+}
+
+/**
+ * This read implemention is a result of debugging hell
+ * It should probably be replaced with a .take() call.
+ */
 impl Read for ZipFileWrapper {
   fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-    let read_size = buf.len().min((self.end_offset - self.offset) as usize);
-    let read = self
-      .archive
-      .reader
-      .read_at(&mut buf[..read_size], self.offset)?;
-    self.offset += read as u64;
-    Ok(read)
+    self.reader.read(buf)
   }
 }
-impl Skippable for ZipFileWrapper {
-  fn skip(&mut self, amount: u64) {
-    self.offset += amount;
-  }
+
+impl Drop for ZipFileWrapper {
+    fn drop(&mut self) {
+        self.command.wait().expect("failed to wait for 7z exit");
+    }
 }
-impl MinimumFileObject for ZipFileWrapper {}
 
 impl VersionBackend for ZipVersionBackend {
-  fn list_files(&mut self) -> Vec<VersionFile> {
+  fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
+    let mut list_command = Command::new("7z");
+    list_command.args(vec!["l", "-ba", &self.path]);
+    let result = list_command.output()?;
+    if !result.status.success() {
+      return Err(anyhow!(
+        "failed to list files: code {:?}",
+        result.status.code()
+      ));
+    }
+    let raw_result = String::from_utf8(result.stdout)?;
+    let files = raw_result.split("\n").filter(|v| v.len() > 0).map(|v| v.split(" ").filter(|v| v.len() > 0));
     let mut results = Vec::new();
-    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-    let mut budget_iterator = self.archive.entries(read_buffer);
-    while let Some(entry) = budget_iterator.next_entry().unwrap() {
-      if entry.is_dir() {
+
+    for file in files {
+      let mut values = file.collect::<Vec<&str>>();
+      values.reverse();
+      let mut iter = values.iter();
+      let (name, compress, size, attrs) = (
+        iter.next().expect("failed to fetch name"),
+        iter.next().expect("failed to read compressed size"),
+        iter.next().expect("failed to read file size"),
+        iter.next().expect("failed to fetch attrs")
+      );
+      if attrs.starts_with("D") {
         continue;
       }
       results.push(VersionFile {
-        relative_filename: entry.file_safe_path().unwrap().to_string(),
-        permission: 744, // apparently ZIPs with permissions are not supported by this library, so we let the owner do anything
-        size: entry.uncompressed_size_hint(),
+        relative_filename: name.to_owned().to_owned(),
+        permission: 0o744, // owner r/w/x, everyone else, read
+        size: size.parse().unwrap(),
       });
     }
-    results
+
+    Ok(results)
   }
 
-  fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject)>> {
-    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-    let mut entries = self.archive.entries(read_buffer);
-    let entry = loop {
-      if let Some(v) = entries.next_entry().unwrap() {
-        if v.file_safe_path().unwrap().to_string() == file.relative_filename {
-          break Some(v);
-        }
-      } else {
-        break None;
-      }
-    }?;
+  fn reader(
+    &mut self,
+    file: &VersionFile,
+    start: u64,
+    end: u64,
+  ) -> anyhow::Result<Box<dyn MinimumFileObject + '_>> {
+    let mut read_command = Command::new("7z");
+    read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]);
+    let output = read_command.stdout(Stdio::piped()).spawn().expect("failed to spawn 7z");
+    Ok(Box::new(ZipFileWrapper::new(output)))
+  }
 
-    let wayfinder = entry.wayfinder();
-    let local_entry = self.archive.get_entry(wayfinder).unwrap();
+  fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
+    let files = self.list_files()?;
+    let file = files
+      .iter()
+      .find(|v| v.relative_filename == sub_path)
+      .expect("file not found");
 
-    let wrapper = self.new_entry(local_entry);
+    Ok(file.clone())
+  }
 
-    Some(Box::new(wrapper))
+  fn require_whole_files(&self) -> bool {
+    true
   }
 }

src/version/types.rs

@@ -1,7 +1,6 @@
-use std::{
-  fmt::Debug, io::{Read, Seek, SeekFrom}
-};
+use std::{fmt::Debug, io::Read};
 
+use dyn_clone::DynClone;
 use tokio::io::{self, AsyncRead};
 
 #[derive(Debug, Clone)]
@@ -11,42 +10,46 @@ pub struct VersionFile {
   pub size: u64,
 }
 
-pub trait Skippable {
-  fn skip(&mut self, amount: u64);
-}
-impl<T> Skippable for T
-where
-  T: Seek,
-{
-  fn skip(&mut self, amount: u64) {
-    self.seek(SeekFrom::Start(amount)).unwrap();
-  }
-}
-
-pub trait MinimumFileObject: Read + Send + Skippable {}
-impl<T: Read + Send + Seek> MinimumFileObject for T {}
+pub trait MinimumFileObject: Read + Send {}
+impl<T: Read + Send> MinimumFileObject for T {}
 
 // Intentionally not a generic, because of types in read_file
-pub struct ReadToAsyncRead {
-  pub inner: Box<(dyn Read + Send)>,
-  pub backend: Box<(dyn VersionBackend + Send)>,
+pub struct ReadToAsyncRead<'a> {
+  pub inner: Box<dyn Read + Send + 'a>,
 }
 
-impl AsyncRead for ReadToAsyncRead {
+const ASYNC_READ_BUFFER_SIZE: usize = 8128;
+
+impl<'a> AsyncRead for ReadToAsyncRead<'a> {
   fn poll_read(
     mut self: std::pin::Pin<&mut Self>,
     _cx: &mut std::task::Context<'_>,
     buf: &mut tokio::io::ReadBuf<'_>,
   ) -> std::task::Poll<io::Result<()>> {
-    let mut read_buf = [0u8; 8192];
-    let var_name = self.inner.read(&mut read_buf).unwrap();
-    let amount = var_name;
-    buf.put_slice(&read_buf[0..amount]);
-    std::task::Poll::Ready(Ok(()))
+    let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE];
+    let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining());
+    match self.inner.read(&mut read_buf[0..read_size]) {
+      Ok(read) => {
+        buf.put_slice(&read_buf[0..read]);
+        std::task::Poll::Ready(Ok(()))
+      }
+      Err(err) => {
+        std::task::Poll::Ready(Err(err))
+      },
+    }
   }
 }
 
-pub trait VersionBackend {
-  fn list_files(&mut self) -> Vec<VersionFile>;
-  fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject)>>;
+pub trait VersionBackend: DynClone {
+  fn require_whole_files(&self) -> bool;
+  fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>>;
+  fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile>;
+  fn reader(
+    &mut self,
+    file: &VersionFile,
+    start: u64,
+    end: u64,
+  ) -> anyhow::Result<Box<dyn MinimumFileObject + '_>>;
 }
+
+dyn_clone::clone_trait_object!(VersionBackend);

src/version/utils.rs

@@ -1,131 +1,173 @@
 use std::{
-  fs::{self, metadata, File},
-  io::Read,
-  path::{Path, PathBuf},
+  collections::HashMap,
+  fs::File,
+  path::Path,
+  process::{Command, ExitStatus},
 };
 
-use napi::{bindgen_prelude::*, tokio_stream::StreamExt};
+use anyhow::anyhow;
+use napi::{bindgen_prelude::*, sys::napi_value__, tokio_stream::StreamExt};
 use tokio_util::codec::{BytesCodec, FramedRead};
 
 use crate::version::{
-  backends::{PathVersionBackend, ZipVersionBackend},
+  backends::{PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED},
   types::{ReadToAsyncRead, VersionBackend, VersionFile},
 };
 
-pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
-  if metadata(path).unwrap().is_dir() {
-    let paths = fs::read_dir(path).unwrap();
-    for path_result in paths {
-      let full_path = path_result.unwrap().path();
-      if metadata(&full_path).unwrap().is_dir() {
-        _list_files(vec, &full_path);
-      } else {
-        vec.push(full_path);
-      }
-    }
+/**
+ * Append new backends here
+ */
+pub fn create_backend_constructor<'a>(
+  path: &Path,
+) -> Option<Box<dyn FnOnce() -> Result<Box<dyn VersionBackend + Send + 'a>>>> {
+  if !path.exists() {
+    return None;
   }
-}
 
-pub fn create_backend_for_path<'a>(path: &Path) -> Option<Box<(dyn VersionBackend + Send + 'a)>> {
   let is_directory = path.is_dir();
   if is_directory {
-    return Some(Box::new(PathVersionBackend {
-      base_dir: path.to_path_buf(),
+    let base_dir = path.to_path_buf();
+    return Some(Box::new(move || {
+      Ok(Box::new(PathVersionBackend { base_dir }))
     }));
   };
 
-  if path.to_string_lossy().ends_with(".zip") {
-    let f = File::open(path.to_path_buf()).unwrap();
-    return Some(Box::new(ZipVersionBackend::new(f)));
+  if *SEVEN_ZIP_INSTALLED {
+    let mut test = Command::new("7z");
+    test.args(vec!["t", path.to_str().expect("invalid utf path")]);
+    let status = test.status().ok()?;
+    if status.code().unwrap_or(1) == 0 {
+      let buf = path.to_path_buf();
+      return Some(Box::new(move || {
+        Ok(Box::new(ZipVersionBackend::new(buf)?))
+      }));
+    }
   }
 
   None
 }
 
-#[napi]
-pub fn has_backend_for_path(path: String) -> bool {
-  let path = Path::new(&path);
-
-  let has_backend = create_backend_for_path(path).is_some();
-
-  has_backend
-}
-
-#[napi]
-pub fn list_files(path: String) -> Result<Vec<String>> {
-  let path = Path::new(&path);
-  let mut backend =
-    create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
-  let files = backend.list_files();
-  Ok(files.into_iter().map(|e| e.relative_filename).collect())
-}
-
 /**
- * This is inefficient, but is used in attempt to keep the interface simple
+ * Persistent object so we can cache things between commands
  */
-#[napi]
-pub fn peek_file(path: String, sub_path: String) -> Result<u32> {
-  let path = Path::new(&path);
-  let mut backend =
-    create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
-  let files = backend.list_files();
-
-  let file = files
-    .iter()
-    .find(|e| e.relative_filename == sub_path)
-    .ok_or(napi::Error::from_reason("Can't find file to peek"))?;
-  return Ok(file.size.try_into().unwrap());
+#[napi(js_name = "DropletHandler")]
+pub struct DropletHandler<'a> {
+  backend_cache: HashMap<String, Box<dyn VersionBackend + Send + 'a>>,
 }
 
 #[napi]
-pub fn read_file(
-  path: String,
-  sub_path: String,
-  env: &Env,
-  start: Option<u32>,
-  end: Option<u32>,
-) -> Option<ReadableStream<'_, BufferSlice<'_>>> {
-  let path = Path::new(&path);
-  let mut backend = create_backend_for_path(path).unwrap();
-  let version_file = VersionFile {
-    relative_filename: sub_path,
-    permission: 0, // Shouldn't matter
-    size: 0,       // Shouldn't matter
-  };
-  // Use `?` operator for cleaner error propagation from `Option`
-  let mut reader = backend.reader(&version_file)?;
-
-  // Skip the 'start' amount of bytes without seek
-  if let Some(skip) = start {
-    reader.skip(skip.into());
-    // io::copy(&mut reader.by_ref().take(skip.into()), &mut io::sink()).unwrap();
+impl<'a> DropletHandler<'a> {
+  #[napi(constructor)]
+  pub fn new() -> Self {
+    DropletHandler {
+      backend_cache: HashMap::new(),
+    }
   }
 
-  let async_reader = if let Some(limit) = end {
-    let amount = limit - start.or(Some(0)).unwrap();
-    ReadToAsyncRead {
-      inner: Box::new(reader.take(amount.into())),
-      backend,
-    }
-  } else {
-    ReadToAsyncRead {
-      inner: reader,
-      backend,
-    }
-  };
+  pub fn create_backend_for_path(
+    &mut self,
+    path: String,
+  ) -> Option<&mut Box<dyn VersionBackend + Send + 'a>> {
+    let fs_path = Path::new(&path);
+    let constructor = create_backend_constructor(fs_path)?;
 
-  // Create a FramedRead stream with BytesCodec for chunking
-  let stream = FramedRead::new(async_reader, BytesCodec::new())
-    // Use StreamExt::map to transform each Result item
-    .map(|result_item| {
-      result_item
-        // Apply Result::map to transform Ok(BytesMut) to Ok(Vec<u8>)
-        .map(|bytes| bytes.to_vec())
-        // Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
-        .map_err(|e| napi::Error::from(e)) // napi::Error implements From<tokio::io::Error>
-    });
-  // Create the napi-rs ReadableStream from the tokio_stream::Stream
-  // The unwrap() here means if stream creation fails, it will panic.
-  // For a production system, consider returning Result<Option<...>> and handling this.
-  Some(ReadableStream::create_with_stream_bytes(env, stream).unwrap())
+    let existing_backend = match self.backend_cache.entry(path) {
+      std::collections::hash_map::Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
+      std::collections::hash_map::Entry::Vacant(vacant_entry) => {
+        let backend = constructor().ok()?;
+        vacant_entry.insert(backend)
+      }
+    };
+
+    Some(existing_backend)
+  }
+
+  #[napi]
+  pub fn has_backend_for_path(&self, path: String) -> bool {
+    let path = Path::new(&path);
+
+    let has_backend = create_backend_constructor(path).is_some();
+
+    has_backend
+  }
+
+  #[napi]
+  pub fn list_files(&mut self, path: String) -> Result<Vec<String>> {
+    let backend = self
+      .create_backend_for_path(path)
+      .ok_or(napi::Error::from_reason("No backend for path"))?;
+    let files = backend.list_files()?;
+    Ok(files.into_iter().map(|e| e.relative_filename).collect())
+  }
+
+  #[napi]
+  pub fn peek_file(&mut self, path: String, sub_path: String) -> Result<u64> {
+    let backend = self
+      .create_backend_for_path(path)
+      .ok_or(napi::Error::from_reason("No backend for path"))?;
+
+    let file = backend.peek_file(sub_path)?;
+
+    Ok(file.size)
+  }
+
+  #[napi]
+  pub fn read_file(
+    &mut self,
+    reference: Reference<DropletHandler<'static>>,
+    path: String,
+    sub_path: String,
+    env: Env,
+    start: Option<BigInt>,
+    end: Option<BigInt>,
+  ) -> anyhow::Result<JsDropStreamable> {
+    let stream = reference.share_with(env, |handler| {
+      let backend = handler
+        .create_backend_for_path(path)
+        .ok_or(anyhow!("Failed to create backend."))?;
+      let version_file = VersionFile {
+        relative_filename: sub_path,
+        permission: 0, // Shouldn't matter
+        size: 0,       // Shouldn't matter
+      };
+      // Use `?` operator for cleaner error propagation from `Option`
+      let reader = backend.reader(
+        &version_file,
+        start.map(|e| e.get_u64().1).unwrap_or(0),
+        end.map(|e| e.get_u64().1).unwrap_or(0),
+      )?;
+
+      let async_reader = ReadToAsyncRead { inner: reader };
+
+      // Create a FramedRead stream with BytesCodec for chunking
+      let stream = FramedRead::new(async_reader, BytesCodec::new())
+        // Use StreamExt::map to transform each Result item
+        .map(|result_item| {
+          result_item
+            // Apply Result::map to transform Ok(BytesMut) to Ok(Vec<u8>)
+            .map(|bytes| bytes.to_vec())
+            // Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
+            .map_err(napi::Error::from) // napi::Error implements From<tokio::io::Error>
+        });
+      // Create the napi-rs ReadableStream from the tokio_stream::Stream
+      // The unwrap() here means if stream creation fails, it will panic.
+      // For a production system, consider returning Result<Option<...>> and handling this.
+      ReadableStream::create_with_stream_bytes(&env, stream)
+    })?;
+
+    Ok(JsDropStreamable { inner: stream })
+  }
+}
+
+#[napi]
+pub struct JsDropStreamable {
+  inner: SharedReference<DropletHandler<'static>, ReadableStream<'static, BufferSlice<'static>>>,
+}
+
+#[napi]
+impl JsDropStreamable {
+  #[napi]
+  pub fn get_stream(&self) -> *mut napi_value__ {
+    self.inner.raw()
+  }
 }

yarn.lock

@@ -12,6 +12,8 @@ __metadata:
     "@napi-rs/cli": "npm:3.0.0-alpha.91"
     "@types/node": "npm:^22.13.10"
     ava: "npm:^6.2.0"
+    pretty-bytes: "npm:^7.0.1"
+    tsimp: "npm:^2.0.12"
   languageName: unknown
   linkType: soft
 
@@ -268,6 +270,38 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@isaacs/balanced-match@npm:^4.0.1":
+  version: 4.0.1
+  resolution: "@isaacs/balanced-match@npm:4.0.1"
+  checksum: 10c0/7da011805b259ec5c955f01cee903da72ad97c5e6f01ca96197267d3f33103d5b2f8a1af192140f3aa64526c593c8d098ae366c2b11f7f17645d12387c2fd420
+  languageName: node
+  linkType: hard
+
+"@isaacs/brace-expansion@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "@isaacs/brace-expansion@npm:5.0.0"
+  dependencies:
+    "@isaacs/balanced-match": "npm:^4.0.1"
+  checksum: 10c0/b4d4812f4be53afc2c5b6c545001ff7a4659af68d4484804e9d514e183d20269bb81def8682c01a22b17c4d6aed14292c8494f7d2ac664e547101c1a905aa977
+  languageName: node
+  linkType: hard
+
+"@isaacs/cached@npm:^1.0.1":
+  version: 1.0.1
+  resolution: "@isaacs/cached@npm:1.0.1"
+  dependencies:
+    "@isaacs/catcher": "npm:^1.0.0"
+  checksum: 10c0/1c15dc2a60873f2c73f4b04ed59ecfc8d9679976ff09af1b5b45e7273a590a4f86a339cc4c785c2d22309277ca47293611af20dd7d41550cdcfb53e06a04ac65
+  languageName: node
+  linkType: hard
+
+"@isaacs/catcher@npm:^1.0.0, @isaacs/catcher@npm:^1.0.4":
+  version: 1.0.4
+  resolution: "@isaacs/catcher@npm:1.0.4"
+  checksum: 10c0/d8b77e2c6b84a6301d390d0b2badea1b4a321f2e8ba662645b045efc42f20a54a6c760f3181fab4ed0d90da58f2cb084a93490a892c53b4da21ec05278b8ba4f
+  languageName: node
+  linkType: hard
+
 "@isaacs/cliui@npm:^8.0.2":
   version: 8.0.2
   resolution: "@isaacs/cliui@npm:8.0.2"
@@ -1756,7 +1790,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"foreground-child@npm:^3.1.0":
+"foreground-child@npm:^3.1.0, foreground-child@npm:^3.1.1, foreground-child@npm:^3.3.1":
   version: 3.3.1
   resolution: "foreground-child@npm:3.3.1"
   dependencies:
@@ -1812,6 +1846,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"glob@npm:^11.0.0":
+  version: 11.0.3
+  resolution: "glob@npm:11.0.3"
+  dependencies:
+    foreground-child: "npm:^3.3.1"
+    jackspeak: "npm:^4.1.1"
+    minimatch: "npm:^10.0.3"
+    minipass: "npm:^7.1.2"
+    package-json-from-dist: "npm:^1.0.0"
+    path-scurry: "npm:^2.0.0"
+  bin:
+    glob: dist/esm/bin.mjs
+  checksum: 10c0/7d24457549ec2903920dfa3d8e76850e7c02aa709122f0164b240c712f5455c0b457e6f2a1eee39344c6148e39895be8094ae8cfef7ccc3296ed30bce250c661
+  languageName: node
+  linkType: hard
+
 "glob@npm:^7.1.3":
   version: 7.2.3
   resolution: "glob@npm:7.2.3"
@@ -1996,6 +2046,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"jackspeak@npm:^4.1.1":
+  version: 4.1.1
+  resolution: "jackspeak@npm:4.1.1"
+  dependencies:
+    "@isaacs/cliui": "npm:^8.0.2"
+  checksum: 10c0/84ec4f8e21d6514db24737d9caf65361511f75e5e424980eebca4199f400874f45e562ac20fa8aeb1dd20ca2f3f81f0788b6e9c3e64d216a5794fd6f30e0e042
+  languageName: node
+  linkType: hard
+
 "js-string-escape@npm:^1.0.1":
   version: 1.0.1
   resolution: "js-string-escape@npm:1.0.1"
@@ -2063,6 +2122,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lru-cache@npm:^11.0.0":
+  version: 11.1.0
+  resolution: "lru-cache@npm:11.1.0"
+  checksum: 10c0/85c312f7113f65fae6a62de7985348649937eb34fb3d212811acbf6704dc322a421788aca253b62838f1f07049a84cc513d88f494e373d3756514ad263670a64
+  languageName: node
+  linkType: hard
+
 "matcher@npm:^5.0.0":
   version: 5.0.0
   resolution: "matcher@npm:5.0.0"
@@ -2114,6 +2180,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"minimatch@npm:^10.0.3":
+  version: 10.0.3
+  resolution: "minimatch@npm:10.0.3"
+  dependencies:
+    "@isaacs/brace-expansion": "npm:^5.0.0"
+  checksum: 10c0/e43e4a905c5d70ac4cec8530ceaeccb9c544b1ba8ac45238e2a78121a01c17ff0c373346472d221872563204eabe929ad02669bb575cb1f0cc30facab369f70f
+  languageName: node
+  linkType: hard
+
 "minimatch@npm:^3.1.1":
   version: 3.1.2
   resolution: "minimatch@npm:3.1.2"
@@ -2311,6 +2386,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"path-scurry@npm:^2.0.0":
+  version: 2.0.0
+  resolution: "path-scurry@npm:2.0.0"
+  dependencies:
+    lru-cache: "npm:^11.0.0"
+    minipass: "npm:^7.1.2"
+  checksum: 10c0/3da4adedaa8e7ef8d6dc4f35a0ff8f05a9b4d8365f2b28047752b62d4c1ad73eec21e37b1579ef2d075920157856a3b52ae8309c480a6f1a8bbe06ff8e52b33c
+  languageName: node
+  linkType: hard
+
 "path-type@npm:^6.0.0":
   version: 6.0.0
   resolution: "path-type@npm:6.0.0"
@@ -2332,6 +2417,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pirates@npm:^4.0.6":
+  version: 4.0.7
+  resolution: "pirates@npm:4.0.7"
+  checksum: 10c0/a51f108dd811beb779d58a76864bbd49e239fa40c7984cd11596c75a121a8cc789f1c8971d8bb15f0dbf9d48b76c05bb62fcbce840f89b688c0fa64b37e8478a
+  languageName: node
+  linkType: hard
+
 "plur@npm:^5.1.0":
   version: 5.1.0
   resolution: "plur@npm:5.1.0"
@@ -2341,6 +2433,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pretty-bytes@npm:^7.0.1":
+  version: 7.0.1
+  resolution: "pretty-bytes@npm:7.0.1"
+  checksum: 10c0/14ffb503d2de3588042c722848062a4897e6faece1694e0c83ba5669ec003d73311d946d50d2b3c6099a6a306760011b8446ee3cf9cf86eca13a454a8f1c47cb
+  languageName: node
+  linkType: hard
+
 "pretty-ms@npm:^9.1.0":
   version: 9.2.0
   resolution: "pretty-ms@npm:9.2.0"
@@ -2398,6 +2497,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"rimraf@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "rimraf@npm:6.0.1"
+  dependencies:
+    glob: "npm:^11.0.0"
+    package-json-from-dist: "npm:^1.0.0"
+  bin:
+    rimraf: dist/esm/bin.mjs
+  checksum: 10c0/b30b6b072771f0d1e73b4ca5f37bb2944ee09375be9db5f558fcd3310000d29dfcfa93cf7734d75295ad5a7486dc8e40f63089ced1722a664539ffc0c3ece8c6
+  languageName: node
+  linkType: hard
+
 "run-parallel@npm:^1.1.9":
   version: 1.2.0
   resolution: "run-parallel@npm:1.2.0"
@@ -2481,6 +2592,24 @@ __metadata:
   languageName: node
   linkType: hard
 
+"sock-daemon@npm:^1.4.2":
+  version: 1.4.2
+  resolution: "sock-daemon@npm:1.4.2"
+  dependencies:
+    rimraf: "npm:^5.0.5"
+    signal-exit: "npm:^4.1.0"
+    socket-post-message: "npm:^1.0.3"
+  checksum: 10c0/1b5e0b02fdd8cd5454fc7de80557c11aac5d88085d0cee80ead08b8f4df5e3c0a4b50ebb2ae2113dab94f36dc88b5d3b7d4b1c2c8e53bbcfbddfc741abf3bd00
+  languageName: node
+  linkType: hard
+
+"socket-post-message@npm:^1.0.3":
+  version: 1.0.3
+  resolution: "socket-post-message@npm:1.0.3"
+  checksum: 10c0/d3ffb51dad97754856aaa6709e036196f4b8b674f00366b71591ead122bcdbc073827f67d17c8b03c9a28c921b2c7cb277c581f6ca318d472034eae7afc169d1
+  languageName: node
+  linkType: hard
+
 "sprintf-js@npm:~1.0.2":
   version: 1.0.3
   resolution: "sprintf-js@npm:1.0.3"
@@ -2613,6 +2742,27 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tsimp@npm:^2.0.12":
+  version: 2.0.12
+  resolution: "tsimp@npm:2.0.12"
+  dependencies:
+    "@isaacs/cached": "npm:^1.0.1"
+    "@isaacs/catcher": "npm:^1.0.4"
+    foreground-child: "npm:^3.1.1"
+    mkdirp: "npm:^3.0.1"
+    pirates: "npm:^4.0.6"
+    rimraf: "npm:^6.0.1"
+    signal-exit: "npm:^4.1.0"
+    sock-daemon: "npm:^1.4.2"
+    walk-up-path: "npm:^4.0.0"
+  peerDependencies:
+    typescript: ^5.1.0
+  bin:
+    tsimp: dist/esm/bin.mjs
+  checksum: 10c0/c56c03a6a4df3ab5ebcefcc0b473992cbb7150173c331be6bda01670d5ae3965e65f30c42757cd391100a1c21485e167a05a350d875f41826b35c45008e5fac8
+  languageName: node
+  linkType: hard
+
 "tslib@npm:^2.4.0":
   version: 2.8.1
   resolution: "tslib@npm:2.8.1"
@@ -2669,6 +2819,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"walk-up-path@npm:^4.0.0":
+  version: 4.0.0
+  resolution: "walk-up-path@npm:4.0.0"
+  checksum: 10c0/fabe344f91387d1d41df230af962ef18bf703dd4178006d55cd6412caacd187b54440002d4d53a982d4f7f0455567dcffb6d3884533c8b2268928eca3ebd8a19
+  languageName: node
+  linkType: hard
+
 "wasm-sjlj@npm:^1.0.6":
   version: 1.0.6
   resolution: "wasm-sjlj@npm:1.0.6"