feat: move to bigints for larger file sizes

Bump rawzip to 0.3

Cargo.lock

@@ -629,7 +629,9 @@ dependencies = [
 
 [[package]]
 name = "rawzip"
-version = "0.2.0"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e512201a808c46ad2a0c810057db306d66b58d5516304548d2445a53db933499"
 
 [[package]]
 name = "rcgen"

Cargo.toml

@@ -10,7 +10,7 @@ crate-type = ["cdylib"]
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 napi = { version = "3.0.0-beta.11", default-features = false, features = [
-    "napi4",
+    "napi6",
     "async",
     "web_stream",
 ] }
@@ -24,14 +24,11 @@ webpki = "0.22.4"
 ring = "0.17.14"
 tokio = { version = "1.45.1", features = ["fs", "io-util"] }
 tokio-util = { version = "0.7.15", features = ["codec"] }
-rawzip = "0.2.0"
+rawzip = "0.3.0"
 
 [package.metadata.patch]
 crates = ["rawzip"]
 
-[patch.crates-io]
-rawzip = { path="./target/patch/rawzip-0.2.0" }
-
 [dependencies.x509-parser]
 version = "0.17.0"
 features = ["verify"]

__test__/utils.spec.mjs

@@ -53,7 +53,7 @@ test("read file offset", async (t) => {
   const testString = "0123456789";
   fs.writeFileSync(dirName + "/TESTFILE", testString);
 
-  const stream = droplet.readFile(dirName, "TESTFILE", 1, 4);
+  const stream = droplet.readFile(dirName, "TESTFILE", BigInt(1), BigInt(4));
 
   let finalString = "";
 

index.d.ts

@@ -15,9 +15,9 @@ export declare function listFiles(path: string): Array<string>
 /**
  * This is inefficient, but is used in attempt to keep the interface simple
  */
-export declare function peekFile(path: string, subPath: string): number
+export declare function peekFile(path: string, subPath: string): bigint
 
-export declare function readFile(path: string, subPath: string, start?: number | undefined | null, end?: number | undefined | null): ReadableStream<Buffer> | null
+export declare function readFile(path: string, subPath: string, start?: bigint | undefined | null, end?: bigint | undefined | null): ReadableStream<Buffer> | null
 
 export declare function signNonce(privateKey: string, nonce: string): string
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drop-oss/droplet",
-  "version": "1.5.3",
+  "version": "1.6.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {

patches/rawzip+0.2.0.patch

@@ -1,26 +0,0 @@
-diff --git a/src/archive.rs b/src/archive.rs
-index 1203015..837c405 100644
---- a/src/archive.rs
-+++ b/src/archive.rs
-@@ -275,7 +275,7 @@ impl<'data> Iterator for ZipSliceEntries<'data> {
- /// ```
- #[derive(Debug, Clone)]
- pub struct ZipArchive<R> {
--    pub(crate) reader: R,
-+    pub reader: R,
-     pub(crate) comment: ZipString,
-     pub(crate) eocd: EndOfCentralDirectory,
- }
-@@ -431,9 +431,9 @@ where
- #[derive(Debug, Clone)]
- pub struct ZipEntry<'archive, R> {
-     archive: &'archive ZipArchive<R>,
--    body_offset: u64,
--    body_end_offset: u64,
--    entry: ZipArchiveEntryWayfinder,
-+    pub body_offset: u64,
-+    pub body_end_offset: u64,
-+    pub entry: ZipArchiveEntryWayfinder,
- }
- 
- impl<'archive, R> ZipEntry<'archive, R>

src/version/backends.rs

@@ -75,12 +75,17 @@ impl ZipVersionBackend {
     }
   }
 
-  pub fn new_entry(&self, entry: ZipEntry<'_, FileReader>) -> ZipFileWrapper {
+  pub fn new_entry(
+    &self,
+    entry: ZipEntry<'_, FileReader>,
+    wayfinder: ZipArchiveEntryWayfinder,
+  ) -> ZipFileWrapper {
+    let (offset, end_offset) = entry.compressed_data_range();
     ZipFileWrapper {
       archive: self.archive.clone(),
-      wayfinder: entry.entry,
+      wayfinder,
-      offset: entry.body_offset,
+      offset,
-      end_offset: entry.body_end_offset,
+      end_offset,
     }
   }
 }
@@ -97,7 +102,7 @@ impl Read for ZipFileWrapper {
     let read_size = buf.len().min((self.end_offset - self.offset) as usize);
     let read = self
       .archive
-      .reader
+      .get_ref()
       .read_at(&mut buf[..read_size], self.offset)?;
     self.offset += read as u64;
     Ok(read)
@@ -120,8 +125,8 @@ impl VersionBackend for ZipVersionBackend {
         continue;
       }
       results.push(VersionFile {
-        relative_filename: entry.file_safe_path().unwrap().to_string(),
+        relative_filename: String::from(entry.file_path().try_normalize().unwrap()),
-        permission: 744, // apparently ZIPs with permissions are not supported by this library, so we let the owner do anything
+        permission: entry.mode().permissions(),
         size: entry.uncompressed_size_hint(),
       });
     }
@@ -133,7 +138,7 @@ impl VersionBackend for ZipVersionBackend {
     let mut entries = self.archive.entries(read_buffer);
     let entry = loop {
       if let Some(v) = entries.next_entry().unwrap() {
-        if v.file_safe_path().unwrap().to_string() == file.relative_filename {
+        if v.file_path().try_normalize().unwrap().as_ref() == &file.relative_filename {
           break Some(v);
         }
       } else {
@@ -144,7 +149,7 @@ impl VersionBackend for ZipVersionBackend {
     let wayfinder = entry.wayfinder();
     let local_entry = self.archive.get_entry(wayfinder).unwrap();
 
-    let wrapper = self.new_entry(local_entry);
+    let wrapper = self.new_entry(local_entry, wayfinder);
 
     Some(Box::new(wrapper))
   }

src/version/utils.rs

@@ -64,7 +64,7 @@ pub fn list_files(path: String) -> Result<Vec<String>> {
  * This is inefficient, but is used in attempt to keep the interface simple
  */
 #[napi]
-pub fn peek_file(path: String, sub_path: String) -> Result<u32> {
+pub fn peek_file(path: String, sub_path: String) -> Result<u64> {
   let path = Path::new(&path);
   let mut backend =
     create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
@@ -82,8 +82,8 @@ pub fn read_file(
   path: String,
   sub_path: String,
   env: &Env,
-  start: Option<u32>,
+  start: Option<BigInt>,
-  end: Option<u32>,
+  end: Option<BigInt>,
 ) -> Option<ReadableStream<'_, BufferSlice<'_>>> {
   let path = Path::new(&path);
   let mut backend = create_backend_for_path(path).unwrap();
@@ -96,13 +96,13 @@ pub fn read_file(
   let mut reader = backend.reader(&version_file)?;
 
   // Skip the 'start' amount of bytes without seek
-  if let Some(skip) = start {
+  if let Some(skip) = start.clone() {
-    reader.skip(skip.into());
+    reader.skip(skip.get_u64().1.into());
     // io::copy(&mut reader.by_ref().take(skip.into()), &mut io::sink()).unwrap();
   }
 
   let async_reader = if let Some(limit) = end {
-    let amount = limit - start.or(Some(0)).unwrap();
+    let amount = limit.get_u64().1 - start.map_or(Some(0), |v| Some(v.get_u64().1)).unwrap();
     ReadToAsyncRead {
       inner: Box::new(reader.take(amount.into())),
       backend,