feat: performance improvements, fix zip

src/manifest.rs

@@ -13,8 +13,7 @@ use napi::{
 use serde_json::json;
 use uuid::Uuid;
 
-use crate::version::utils::create_backend_for_path;
-
+use crate::version::{types::VersionBackend, utils::DropletHandler};
 
 const CHUNK_SIZE: usize = 1024 * 1024 * 64;
 
@@ -36,15 +35,18 @@ pub fn call_alt_thread_func(tsfn: Arc<ThreadsafeFunction<()>>) -> Result<(), Str
 }
 
 #[napi]
-pub fn generate_manifest(
+pub fn generate_manifest<'a>(
+  droplet_handler: &mut DropletHandler,
   dir: String,
   progress_sfn: ThreadsafeFunction<i32>,
   log_sfn: ThreadsafeFunction<String>,
   callback_sfn: ThreadsafeFunction<String>,
-) -> Result<(), String> {
+) -> Result<()> {
+  let backend: &mut Box<dyn VersionBackend + Send> =
+    droplet_handler.create_backend_for_path(dir).ok_or(napi::Error::from_reason("Could not create backend for path."))?;
+  let backend: &'static mut Box<dyn VersionBackend + Send> =
+    unsafe { std::mem::transmute(backend) };
   thread::spawn(move || {
-    let base_dir = Path::new(&dir);
-    let mut backend = create_backend_for_path(base_dir).unwrap();
     let files = backend.list_files();
 
     // Filepath to chunk data
@@ -54,7 +56,7 @@ pub fn generate_manifest(
     let mut i: i32 = 0;
 
     for version_file in files {
-      let raw_reader= backend.reader(&version_file).unwrap();
+      let raw_reader = backend.reader(&version_file).unwrap();
       let mut reader = BufReader::with_capacity(CHUNK_SIZE, raw_reader);
 
       let mut chunk_data = ChunkData {
@@ -106,4 +108,4 @@ pub fn generate_manifest(
   });
 
   Ok(())
-}
+}

src/version/backends.rs

@@ -1,24 +1,34 @@
-use core::arch;
 #[cfg(unix)]
 use std::os::unix::fs::PermissionsExt;
 use std::{
-  fs::File,
-  io::{self, Read, Seek},
-  path::PathBuf,
-  pin::Pin,
-  rc::Rc,
+  fs::{self, metadata, File},
+  io::{self, Read, Sink},
+  path::{Path, PathBuf},
   sync::Arc,
 };
 
+use flate2::read::DeflateDecoder;
 use rawzip::{
-  FileReader, ReaderAt, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, RECOMMENDED_BUFFER_SIZE,
+  FileReader, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, ZipReader, RECOMMENDED_BUFFER_SIZE,
 };
 
-use crate::version::{
-  types::{MinimumFileObject, Skippable, VersionBackend, VersionFile},
-  utils::_list_files,
-};
+use crate::version::types::{MinimumFileObject, Skippable, VersionBackend, VersionFile};
 
+pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
+  if metadata(path).unwrap().is_dir() {
+    let paths = fs::read_dir(path).unwrap();
+    for path_result in paths {
+      let full_path = path_result.unwrap().path();
+      if metadata(&full_path).unwrap().is_dir() {
+        _list_files(vec, &full_path);
+      } else {
+        vec.push(full_path);
+      }
+    }
+  }
+}
+
+#[derive(Clone)]
 pub struct PathVersionBackend {
   pub base_dir: PathBuf,
 }
@@ -30,40 +40,55 @@ impl VersionBackend for PathVersionBackend {
     let mut results = Vec::new();
 
     for pathbuf in vec.iter() {
-      let file = File::open(pathbuf.clone()).unwrap();
       let relative = pathbuf.strip_prefix(self.base_dir.clone()).unwrap();
-      let metadata = file.try_clone().unwrap().metadata().unwrap();
-      let permission_object = metadata.permissions();
-      let permissions = {
-        let perm: u32;
-        #[cfg(target_family = "unix")]
-        {
-          perm = permission_object.mode();
-        }
-        #[cfg(not(target_family = "unix"))]
-        {
-          perm = 0
-        }
-        perm
-      };
 
-      results.push(VersionFile {
-        relative_filename: relative.to_string_lossy().to_string(),
-        permission: permissions,
-        size: metadata.len(),
-      });
+      results.push(
+        self
+          .peek_file(relative.to_str().unwrap().to_owned())
+          .unwrap(),
+      );
     }
 
     results
   }
 
-  fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject + 'static)>> {
+  fn reader(&mut self, file: &VersionFile) -> Option<Box<dyn MinimumFileObject + 'static>> {
     let file = File::open(self.base_dir.join(file.relative_filename.clone())).ok()?;
 
     return Some(Box::new(file));
   }
+
+  fn peek_file(&mut self, sub_path: String) -> Option<VersionFile> {
+    let pathbuf = self.base_dir.join(sub_path.clone());
+    if !pathbuf.exists() {
+      return None;
+    };
+
+    let file = File::open(pathbuf.clone()).unwrap();
+    let metadata = file.try_clone().unwrap().metadata().unwrap();
+    let permission_object = metadata.permissions();
+    let permissions = {
+      let perm: u32;
+      #[cfg(target_family = "unix")]
+      {
+        perm = permission_object.mode();
+      }
+      #[cfg(not(target_family = "unix"))]
+      {
+        perm = 0
+      }
+      perm
+    };
+
+    Some(VersionFile {
+      relative_filename: sub_path,
+      permission: permissions,
+      size: metadata.len(),
+    })
+  }
 }
 
+#[derive(Clone)]
 pub struct ZipVersionBackend {
   archive: Arc<ZipArchive<FileReader>>,
 }
@@ -75,46 +100,51 @@ impl ZipVersionBackend {
     }
   }
 
-  pub fn new_entry(
+  pub fn new_entry<'archive>(
     &self,
-    entry: ZipEntry<'_, FileReader>,
-    wayfinder: ZipArchiveEntryWayfinder,
-  ) -> ZipFileWrapper {
-    let (offset, end_offset) = entry.compressed_data_range();
-    ZipFileWrapper {
-      archive: self.archive.clone(),
-      wayfinder,
-      offset,
-      end_offset,
-    }
+    entry: ZipEntry<'archive, FileReader>,
+  ) -> ZipFileWrapper<'archive> {
+    let deflater = DeflateDecoder::new(entry.reader());
+    ZipFileWrapper { reader: deflater }
   }
 }
 
-pub struct ZipFileWrapper {
-  pub archive: Arc<ZipArchive<FileReader>>,
-  wayfinder: ZipArchiveEntryWayfinder,
-  offset: u64,
-  end_offset: u64,
+pub struct ZipFileWrapper<'archive> {
+  reader: DeflateDecoder<ZipReader<'archive, FileReader>>,
 }
 
-impl Read for ZipFileWrapper {
+impl<'a> Read for ZipFileWrapper<'a> {
   fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-    let read_size = buf.len().min((self.end_offset - self.offset) as usize);
-    let read = self
-      .archive
-      .get_ref()
-      .read_at(&mut buf[..read_size], self.offset)?;
-    self.offset += read as u64;
+    let read = self.reader.read(buf)?;
     Ok(read)
   }
 }
-impl Skippable for ZipFileWrapper {
+impl<'a> Skippable for ZipFileWrapper<'a> {
   fn skip(&mut self, amount: u64) {
-    self.offset += amount;
+    io::copy(&mut self.reader.by_ref().take(amount), &mut Sink::default()).unwrap();
   }
 }
-impl MinimumFileObject for ZipFileWrapper {}
+impl<'a> MinimumFileObject for ZipFileWrapper<'a> {}
 
+impl ZipVersionBackend {
+  fn find_wayfinder(&mut self, filename: &str) -> Option<ZipArchiveEntryWayfinder> {
+    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
+    let mut entries = self.archive.entries(read_buffer);
+    let entry = loop {
+      if let Some(v) = entries.next_entry().unwrap() {
+        if v.file_path().try_normalize().unwrap().as_ref() == filename {
+          break Some(v);
+        }
+      } else {
+        break None;
+      }
+    }?;
+
+    let wayfinder = entry.wayfinder();
+
+    Some(wayfinder)
+  }
+}
 impl VersionBackend for ZipVersionBackend {
   fn list_files(&mut self) -> Vec<VersionFile> {
     let mut results = Vec::new();
@@ -133,24 +163,22 @@ impl VersionBackend for ZipVersionBackend {
     results
   }
 
-  fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject)>> {
-    let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
-    let mut entries = self.archive.entries(read_buffer);
-    let entry = loop {
-      if let Some(v) = entries.next_entry().unwrap() {
-        if v.file_path().try_normalize().unwrap().as_ref() == &file.relative_filename {
-          break Some(v);
-        }
-      } else {
-        break None;
-      }
-    }?;
-
-    let wayfinder = entry.wayfinder();
+  fn reader(&mut self, file: &VersionFile) -> Option<Box<dyn MinimumFileObject + '_>> {
+    let wayfinder = self.find_wayfinder(&file.relative_filename)?;
     let local_entry = self.archive.get_entry(wayfinder).unwrap();
 
-    let wrapper = self.new_entry(local_entry, wayfinder);
+    let wrapper = self.new_entry(local_entry);
 
     Some(Box::new(wrapper))
   }
+
+  fn peek_file(&mut self, sub_path: String) -> Option<VersionFile> {
+    let entry = self.find_wayfinder(&sub_path)?;
+
+    Some(VersionFile {
+      relative_filename: sub_path,
+      permission: 0,
+      size: entry.uncompressed_size_hint(),
+    })
+  }
 }

src/version/types.rs

@@ -1,7 +1,9 @@
 use std::{
-  fmt::Debug, io::{Read, Seek, SeekFrom}
+  fmt::Debug,
+  io::{Read, Seek, SeekFrom},
 };
 
+use dyn_clone::DynClone;
 use tokio::io::{self, AsyncRead};
 
 #[derive(Debug, Clone)]
@@ -27,12 +29,11 @@ pub trait MinimumFileObject: Read + Send + Skippable {}
 impl<T: Read + Send + Seek> MinimumFileObject for T {}
 
 // Intentionally not a generic, because of types in read_file
-pub struct ReadToAsyncRead {
-  pub inner: Box<(dyn Read + Send)>,
-  pub backend: Box<(dyn VersionBackend + Send)>,
+pub struct ReadToAsyncRead<'a> {
+  pub inner: Box<dyn Read + Send + 'a>,
 }
 
-impl AsyncRead for ReadToAsyncRead {
+impl<'a> AsyncRead for ReadToAsyncRead<'a> {
   fn poll_read(
     mut self: std::pin::Pin<&mut Self>,
     _cx: &mut std::task::Context<'_>,
@@ -40,13 +41,16 @@ impl AsyncRead for ReadToAsyncRead {
   ) -> std::task::Poll<io::Result<()>> {
     let mut read_buf = [0u8; 8192];
     let var_name = self.inner.read(&mut read_buf).unwrap();
-    let amount = var_name;
+    let amount = var_name.min(buf.remaining());
     buf.put_slice(&read_buf[0..amount]);
     std::task::Poll::Ready(Ok(()))
   }
 }
 
-pub trait VersionBackend {
+pub trait VersionBackend: DynClone {
   fn list_files(&mut self) -> Vec<VersionFile>;
-  fn reader(&mut self, file: &VersionFile) -> Option<Box<(dyn MinimumFileObject)>>;
+  fn peek_file(&mut self, sub_path: String) -> Option<VersionFile>;
+  fn reader(&mut self, file: &VersionFile) -> Option<Box<dyn MinimumFileObject + '_>>;
 }
+
+dyn_clone::clone_trait_object!(VersionBackend);

src/version/utils.rs

@@ -1,10 +1,6 @@
-use std::{
-  fs::{self, metadata, File},
-  io::Read,
-  path::{Path, PathBuf},
-};
+use std::{collections::HashMap, fs::File, io::Read, path::Path};
 
-use napi::{bindgen_prelude::*, tokio_stream::StreamExt};
+use napi::{bindgen_prelude::*, sys::napi_value__, tokio_stream::StreamExt};
 use tokio_util::codec::{BytesCodec, FramedRead};
 
 use crate::version::{
@@ -12,120 +8,160 @@ use crate::version::{
   types::{ReadToAsyncRead, VersionBackend, VersionFile},
 };
 
-pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
-  if metadata(path).unwrap().is_dir() {
-    let paths = fs::read_dir(path).unwrap();
-    for path_result in paths {
-      let full_path = path_result.unwrap().path();
-      if metadata(&full_path).unwrap().is_dir() {
-        _list_files(vec, &full_path);
-      } else {
-        vec.push(full_path);
-      }
-    }
+/**
+ * Append new backends here
+ */
+pub fn create_backend_constructor<'a>(
+  path: &Path,
+) -> Option<Box<dyn FnOnce() -> Box<dyn VersionBackend + Send + 'a>>> {
+  if !path.exists() {
+    return None;
   }
-}
 
-pub fn create_backend_for_path<'a>(path: &Path) -> Option<Box<(dyn VersionBackend + Send + 'a)>> {
   let is_directory = path.is_dir();
   if is_directory {
-    return Some(Box::new(PathVersionBackend {
-      base_dir: path.to_path_buf(),
-    }));
+    let base_dir = path.to_path_buf();
+    return Some(Box::new(move || Box::new(PathVersionBackend { base_dir })));
   };
 
   if path.to_string_lossy().ends_with(".zip") {
     let f = File::open(path.to_path_buf()).unwrap();
-    return Some(Box::new(ZipVersionBackend::new(f)));
+    return Some(Box::new(|| Box::new(ZipVersionBackend::new(f))));
   }
 
   None
 }
 
-#[napi]
-pub fn has_backend_for_path(path: String) -> bool {
-  let path = Path::new(&path);
-
-  let has_backend = create_backend_for_path(path).is_some();
-
-  has_backend
-}
-
-#[napi]
-pub fn list_files(path: String) -> Result<Vec<String>> {
-  let path = Path::new(&path);
-  let mut backend =
-    create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
-  let files = backend.list_files();
-  Ok(files.into_iter().map(|e| e.relative_filename).collect())
-}
-
 /**
- * This is inefficient, but is used in attempt to keep the interface simple
+ * Persistent object so we can cache things between commands
  */
-#[napi]
-pub fn peek_file(path: String, sub_path: String) -> Result<u64> {
-  let path = Path::new(&path);
-  let mut backend =
-    create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
-  let files = backend.list_files();
-
-  let file = files
-    .iter()
-    .find(|e| e.relative_filename == sub_path)
-    .ok_or(napi::Error::from_reason("Can't find file to peek"))?;
-  return Ok(file.size.try_into().unwrap());
+#[napi(js_name = "DropletHandler")]
+pub struct DropletHandler<'a> {
+  backend_cache: HashMap<String, Box<dyn VersionBackend + Send + 'a>>,
 }
 
 #[napi]
-pub fn read_file(
-  path: String,
-  sub_path: String,
-  env: &Env,
-  start: Option<BigInt>,
-  end: Option<BigInt>,
-) -> Option<ReadableStream<'_, BufferSlice<'_>>> {
-  let path = Path::new(&path);
-  let mut backend = create_backend_for_path(path).unwrap();
-  let version_file = VersionFile {
-    relative_filename: sub_path,
-    permission: 0, // Shouldn't matter
-    size: 0,       // Shouldn't matter
-  };
-  // Use `?` operator for cleaner error propagation from `Option`
-  let mut reader = backend.reader(&version_file)?;
-
-  // Skip the 'start' amount of bytes without seek
-  if let Some(skip) = start.clone() {
-    reader.skip(skip.get_u64().1.into());
-    // io::copy(&mut reader.by_ref().take(skip.into()), &mut io::sink()).unwrap();
+impl<'a> DropletHandler<'a> {
+  #[napi(constructor)]
+  pub fn new() -> Self {
+    DropletHandler {
+      backend_cache: HashMap::new(),
+    }
   }
 
-  let async_reader = if let Some(limit) = end {
-    let amount = limit.get_u64().1 - start.map_or(Some(0), |v| Some(v.get_u64().1)).unwrap();
-    ReadToAsyncRead {
-      inner: Box::new(reader.take(amount.into())),
-      backend,
-    }
-  } else {
-    ReadToAsyncRead {
-      inner: reader,
-      backend,
-    }
-  };
+  pub fn create_backend_for_path(
+    &mut self,
+    path: String,
+  ) -> Option<&mut Box<dyn VersionBackend + Send + 'a>> {
+    let fs_path = Path::new(&path);
+    let constructor = create_backend_constructor(fs_path)?;
 
-  // Create a FramedRead stream with BytesCodec for chunking
-  let stream = FramedRead::new(async_reader, BytesCodec::new())
-    // Use StreamExt::map to transform each Result item
-    .map(|result_item| {
-      result_item
-        // Apply Result::map to transform Ok(BytesMut) to Ok(Vec<u8>)
-        .map(|bytes| bytes.to_vec())
-        // Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
-        .map_err(|e| napi::Error::from(e)) // napi::Error implements From<tokio::io::Error>
+    let existing_backend = self.backend_cache.entry(path).or_insert_with(|| {
+      let backend = constructor();
+      backend
     });
-  // Create the napi-rs ReadableStream from the tokio_stream::Stream
-  // The unwrap() here means if stream creation fails, it will panic.
-  // For a production system, consider returning Result<Option<...>> and handling this.
-  Some(ReadableStream::create_with_stream_bytes(env, stream).unwrap())
+
+    Some(existing_backend)
+  }
+
+  #[napi]
+  pub fn has_backend_for_path(&self, path: String) -> bool {
+    let path = Path::new(&path);
+
+    let has_backend = create_backend_constructor(path).is_some();
+
+    has_backend
+  }
+
+  #[napi]
+  pub fn list_files(&mut self, path: String) -> Result<Vec<String>> {
+    let backend = self
+      .create_backend_for_path(path)
+      .ok_or(napi::Error::from_reason("No backend for path"))?;
+    let files = backend.list_files();
+    Ok(files.into_iter().map(|e| e.relative_filename).collect())
+  }
+
+  #[napi]
+  pub fn peek_file(&mut self, path: String, sub_path: String) -> Result<u64> {
+    let backend = self
+      .create_backend_for_path(path)
+      .ok_or(napi::Error::from_reason("No backend for path"))?;
+
+    let file = backend
+      .peek_file(sub_path)
+      .ok_or(napi::Error::from_reason("Can't find file to peek"))?;
+
+    return Ok(file.size.try_into().unwrap());
+  }
+
+  #[napi]
+  pub fn read_file(
+    &mut self,
+    reference: Reference<DropletHandler<'static>>,
+    path: String,
+    sub_path: String,
+    env: Env,
+    start: Option<BigInt>,
+    end: Option<BigInt>,
+  ) -> Result<JsDropStreamable> {
+    let stream = reference.share_with(env, |handler| {
+      let backend = handler
+        .create_backend_for_path(path)
+        .ok_or(napi::Error::from_reason("Failed to create backend."))?;
+      let version_file = VersionFile {
+        relative_filename: sub_path,
+        permission: 0, // Shouldn't matter
+        size: 0,       // Shouldn't matter
+      };
+      // Use `?` operator for cleaner error propagation from `Option`
+      let mut reader = backend.reader(&version_file).ok_or(napi::Error::from_reason("Failed to create reader."))?;
+
+      if let Some(skip) = start.clone() {
+        reader.skip(skip.get_u64().1.into());
+        // io::copy(&mut reader.by_ref().take(skip.into()), &mut io::sink()).unwrap();
+      }
+
+      let async_reader = if let Some(limit) = end {
+        let amount = limit.get_u64().1 - start.map_or(Some(0), |v| Some(v.get_u64().1)).unwrap();
+        ReadToAsyncRead {
+          inner: Box::new(reader.take(amount.into())),
+        }
+      } else {
+        ReadToAsyncRead { inner: reader }
+      };
+
+      // Create a FramedRead stream with BytesCodec for chunking
+      let stream = FramedRead::new(async_reader, BytesCodec::new())
+        // Use StreamExt::map to transform each Result item
+        .map(|result_item| {
+          result_item
+            // Apply Result::map to transform Ok(BytesMut) to Ok(Vec<u8>)
+            .map(|bytes| bytes.to_vec())
+            // Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
+            .map_err(|e| napi::Error::from(e)) // napi::Error implements From<tokio::io::Error>
+        });
+      // Create the napi-rs ReadableStream from the tokio_stream::Stream
+      // The unwrap() here means if stream creation fails, it will panic.
+      // For a production system, consider returning Result<Option<...>> and handling this.
+      Ok(ReadableStream::create_with_stream_bytes(&env, stream).unwrap())
+    })?;
+
+    Ok(JsDropStreamable {
+      inner: stream,
+    })
+  }
 }
+
+#[napi]
+pub struct JsDropStreamable {
+  inner: SharedReference<DropletHandler<'static>, ReadableStream<'static, BufferSlice<'static>>>,
+}
+
+#[napi]
+impl JsDropStreamable {
+  #[napi]
+  pub fn get_stream(&self) -> *mut napi_value__ {
+    self.inner.raw()
+  }
+}