fix: chunk size

__test__/manifest.spec.mjs

@@ -2,7 +2,7 @@ import test from "ava";
 import fs from "node:fs";
 import path from "path";
 
-import { generateManifest } from "../index.js";
+import { generateManifest, listFiles } from "../index.js";
 
 test("numerous small file", async (t) => {
   // Setup test dir

index.d.ts

@@ -3,6 +3,8 @@
 
 /* auto-generated by NAPI-RS */
 
+export declare function hasBackendForPath(path: string): boolean
+export declare function listFiles(path: string): Array<string>
 export declare function callAltThreadFunc(callback: (...args: any[]) => any): void
 export declare function generateManifest(dir: string, progress: (...args: any[]) => any, log: (...args: any[]) => any, callback: (...args: any[]) => any): void
 export declare function generateRootCa(): Array<string>

index.js

@@ -310,8 +310,10 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { callAltThreadFunc, generateManifest, generateRootCa, generateClientCertificate, verifyClientCertificate, signNonce, verifyNonce } = nativeBinding
+const { hasBackendForPath, listFiles, callAltThreadFunc, generateManifest, generateRootCa, generateClientCertificate, verifyClientCertificate, signNonce, verifyNonce } = nativeBinding
 
+module.exports.hasBackendForPath = hasBackendForPath
+module.exports.listFiles = listFiles
 module.exports.callAltThreadFunc = callAltThreadFunc
 module.exports.generateManifest = generateManifest
 module.exports.generateRootCa = generateRootCa

npm/win32-ia32-msvc/README.md

@@ -1,3 +0,0 @@
-# `@drop-oss/droplet-win32-ia32-msvc`
-
-This is the **i686-pc-windows-msvc** binary for `@drop-oss/droplet`

npm/win32-ia32-msvc/package.json

@@ -1,21 +0,0 @@
-{
-  "name": "@drop-oss/droplet-win32-ia32-msvc",
-  "version": "0.0.0",
-  "os": [
-    "win32"
-  ],
-  "cpu": [
-    "ia32"
-  ],
-  "main": "droplet.win32-ia32-msvc.node",
-  "files": [
-    "droplet.win32-ia32-msvc.node"
-  ],
-  "license": "MIT",
-  "engines": {
-    "node": ">= 10"
-  },
-  "repository": {
-    "url": "https://github.com/Drop-OSS/droplet"
-  }
-}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "0.7.1",
+  "version": "1.1.2",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -14,7 +14,6 @@
         "armv7-unknown-linux-gnueabihf",
         "armv7-unknown-linux-musleabihf",
         "x86_64-unknown-linux-musl",
-        "i686-pc-windows-msvc",
         "universal-apple-darwin",
         "riscv64gc-unknown-linux-gnu"
       ]
@@ -43,6 +42,6 @@
   },
   "packageManager": "yarn@4.7.0",
   "repository": {
-    "url": "https://github.com/Drop-OSS/droplet"
+    "url": "git+https://github.com/Drop-OSS/droplet.git"
   }
 }

src/file_utils.rs

@@ -1,8 +1,14 @@
+#[cfg(unix)]
+use std::os::unix::fs::PermissionsExt;
 use std::{
-  fs::{self, metadata},
+  fs::{self, metadata, File},
+  io::BufReader,
   path::{Path, PathBuf},
 };
 
+const CHUNK_SIZE: usize = 1024 * 1024 * 64;
+
+
 fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
   if metadata(path).unwrap().is_dir() {
     let paths = fs::read_dir(path).unwrap();
@@ -17,8 +23,101 @@ fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
   }
 }
 
-pub fn list_files(path: &Path) -> Vec<PathBuf> {
-  let mut vec = Vec::new();
-  _list_files(&mut vec, path);
-  vec
+pub struct VersionFile {
+  pub relative_filename: String,
+  pub permission: u32,
 }
+
+pub trait VersionBackend: 'static {
+  fn list_files(&self, path: &Path) -> Vec<VersionFile>;
+  fn reader(&self, file: &VersionFile) -> BufReader<File>;
+}
+
+pub struct PathVersionBackend {
+  pub base_dir: PathBuf,
+}
+impl VersionBackend for PathVersionBackend {
+  fn list_files(&self, path: &Path) -> Vec<VersionFile> {
+    let mut vec = Vec::new();
+    _list_files(&mut vec, path);
+
+    let mut results = Vec::new();
+
+    for pathbuf in vec.iter() {
+      let file = File::open(pathbuf.clone()).unwrap();
+      let relative = pathbuf.strip_prefix(path).unwrap();
+      let metadata = file.try_clone().unwrap().metadata().unwrap();
+      let permission_object = metadata.permissions();
+      let permissions = {
+        let perm: u32;
+        #[cfg(target_family = "unix")]
+        {
+          perm = permission_object.mode();
+        }
+        #[cfg(not(target_family = "unix"))]
+        {
+          perm = 0
+        }
+        perm
+      };
+
+      results.push(VersionFile {
+        relative_filename: relative.to_string_lossy().to_string(),
+        permission: permissions,
+      });
+    }
+
+    results
+  }
+
+  fn reader(&self, file: &VersionFile) -> BufReader<File> {
+    let file = File::open(self.base_dir.join(file.relative_filename.clone())).unwrap();
+    let reader = BufReader::with_capacity(CHUNK_SIZE, file);
+    return reader;
+  }
+}
+
+// Todo implementation for archives
+// Split into a separate impl for each type of archive
+pub struct ArchiveVersionBackend {}
+impl VersionBackend for ArchiveVersionBackend {
+  fn list_files(&self, path: &Path) -> Vec<VersionFile> {
+    todo!()
+  }
+
+  fn reader(&self, file: &VersionFile) -> BufReader<File> {
+    todo!()
+  }
+}
+
+pub fn create_backend_for_path(path: &Path) -> Option<Box<(dyn VersionBackend)>> {
+  let is_directory = path.is_dir();
+  if is_directory {
+    return Some(Box::new(PathVersionBackend {
+      base_dir: path.to_path_buf(),
+    }));
+  };
+
+  /*
+    Insert checks for whatever backend you like
+  */
+
+  None
+}
+
+#[napi]
+pub fn has_backend_for_path(path: String) -> bool {
+  let path = Path::new(&path);
+
+  let has_backend = create_backend_for_path(path).is_some();
+
+  has_backend
+}
+
+#[napi]
+pub fn list_files(path: String) -> Vec<String> {
+  let path = Path::new(&path);
+  let backend = create_backend_for_path(path).unwrap();
+  let files = backend.list_files(path);
+  files.into_iter().map(|e| e.relative_filename).collect()
+}

src/manifest.rs

@@ -16,9 +16,7 @@ use napi::{
 use serde_json::json;
 use uuid::Uuid;
 
-use crate::file_utils::list_files;
-
-const CHUNK_SIZE: usize = 1024 * 1024 * 64;
+use crate::file_utils::create_backend_for_path;
 
 #[derive(serde::Serialize)]
 struct ChunkData {
@@ -64,7 +62,8 @@ pub fn generate_manifest(
 
   thread::spawn(move || {
     let base_dir = Path::new(&dir);
-    let files = list_files(base_dir);
+    let backend = create_backend_for_path(base_dir).unwrap();
+    let files = backend.list_files(base_dir);
 
     // Filepath to chunk data
     let mut chunks: HashMap<String, ChunkData> = HashMap::new();
@@ -72,27 +71,11 @@ pub fn generate_manifest(
     let total: i32 = files.len() as i32;
     let mut i: i32 = 0;
 
-    for file_path in files {
-      let file = File::open(file_path.clone()).unwrap();
-      let relative = file_path.strip_prefix(base_dir).unwrap();
-      let permission_object = file.try_clone().unwrap().metadata().unwrap().permissions();
-      let permissions = {
-        let perm: u32;
-        #[cfg(target_family = "unix")]
-        {
-          perm = permission_object.mode();
-        }
-        #[cfg(not(target_family = "unix"))]
-        {
-          perm = 0
-        }
-        perm
-      };
-
-      let mut reader = BufReader::with_capacity(CHUNK_SIZE, file);
+    for version_file in files {
+      let mut reader = backend.reader(&version_file);
 
       let mut chunk_data = ChunkData {
-        permissions,
+        permissions: version_file.permission,
         ids: Vec::new(),
         checksums: Vec::new(),
         lengths: Vec::new(),
@@ -119,7 +102,7 @@ pub fn generate_manifest(
         let log_str = format!(
           "Processed chunk {} for {}",
           chunk_index,
-          relative.to_str().unwrap()
+          &version_file.relative_filename
         );
         log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
 
@@ -127,7 +110,7 @@ pub fn generate_manifest(
         chunk_index += 1;
       }
 
-      chunks.insert(relative.to_str().unwrap().to_string(), chunk_data);
+      chunks.insert(version_file.relative_filename, chunk_data);
 
       i += 1;
       let progress = i * 100 / total;