mirror of
https://github.com/Drop-OSS/droplet.git
synced 2026-06-22 04:11:40 +10:00
feat: use new droplet-rs crate
This commit is contained in:
DecDuck
Generated
+67
-22
@@ -101,6 +101,17 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.89"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.0"
|
||||
@@ -328,9 +339,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.9.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db05ffb6856bf0ecdf6367558a76a0e8a77b1713044eb92845c692100ed50190"
|
||||
checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
@@ -349,9 +360,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "ctor"
|
||||
version = "0.6.1"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ffc71fcdcdb40d6f087edddf7f8f1f8f79e6cf922f555a9ee8779752d4819bd"
|
||||
checksum = "eb230974aaf0aca4d71665bed0aca156cf43b764fcb9583b69c6c3e686f35e72"
|
||||
dependencies = [
|
||||
"ctor-proc-macro",
|
||||
"dtor",
|
||||
@@ -453,10 +464,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "droplet-rs"
|
||||
version = "0.8.1"
|
||||
source = "git+https://github.com/Drop-OSS/droplet-rs.git#32a7ec91c35f3f676341c52124e19f7953c63a5d"
|
||||
version = "0.9.2"
|
||||
source = "git+https://github.com/Drop-OSS/droplet-rs.git#d8f37886d479d8d9fec7e51523628e863306dddb"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"dyn-clone",
|
||||
"hex",
|
||||
"rcgen",
|
||||
@@ -850,9 +862,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.177"
|
||||
version = "0.2.178"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
|
||||
checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
@@ -907,10 +919,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "napi"
|
||||
version = "3.5.2"
|
||||
name = "mio"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e917a98ac74187a5d486604a269ed69cd7901dd4824453d5573fb051f69b1b3"
|
||||
checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi"
|
||||
version = "3.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3af30fe8e799dda3a555c496c59e960e4cff1e931b63acbaf3a3b25d9fad22b6"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags",
|
||||
@@ -933,9 +956,9 @@ checksum = "d376940fd5b723c6893cd1ee3f33abbfd86acb1cd1ec079f3ab04a2a3bc4d3b1"
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive"
|
||||
version = "3.3.3"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a258a6521951715e00568b258b8fb7a44c6087f588c371dc6b84a413f2728fdb"
|
||||
checksum = "47cffa09ea668c4cc5d7b1198780882e28780ed1804a903b80680725426223d9"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"ctor",
|
||||
@@ -947,9 +970,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive-backend"
|
||||
version = "3.0.2"
|
||||
version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77c36636292fe04366a1eec028adc25bc72f4fd7cce35bdcc310499ef74fb7de"
|
||||
checksum = "5e186227ec22f4675267a176d98dffecb27e6cc88926cbb7efb5427268565c0f"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
@@ -960,9 +983,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi-sys"
|
||||
version = "3.1.1"
|
||||
version = "3.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50ef9c1086f16aea2417c3788dbefed7591c3bccd800b827f4dfb271adff1149"
|
||||
checksum = "8eb602b84d7c1edae45e50bbf1374696548f36ae179dfa667f577e384bb90c2b"
|
||||
dependencies = [
|
||||
"libloading",
|
||||
]
|
||||
@@ -1350,7 +1373,7 @@ dependencies = [
|
||||
"getrandom 0.2.16",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1456,6 +1479,15 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.1"
|
||||
@@ -1637,7 +1669,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1726,9 +1762,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.18.1"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
|
||||
checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"js-sys",
|
||||
@@ -1739,9 +1775,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "uuid-macro-internal"
|
||||
version = "1.18.1"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9384a660318abfbd7f8932c34d67e4d1ec511095f95972ddc01e19d7ba8413f"
|
||||
checksum = "39d11901c36b3650df7acb0f9ebe624f35b5ac4e1922ecd3c57f444648429594"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1839,6 +1875,15 @@ dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
import test from "ava";
|
||||
import { DropletHandler, generateManifest } from "../index.js";
|
||||
|
||||
test.skip("debug", async (t) => {
|
||||
const handler = new DropletHandler();
|
||||
|
||||
console.log("created handler");
|
||||
|
||||
const manifest = JSON.parse(
|
||||
await new Promise((r, e) =>
|
||||
generateManifest(
|
||||
handler,
|
||||
"./assets/TheGame.zip",
|
||||
(_, __) => {},
|
||||
(_, __) => {},
|
||||
(err, manifest) => (err ? e(err) : r(manifest))
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
return t.pass();
|
||||
});
|
||||
+14
-32
@@ -2,7 +2,7 @@ import test from "ava";
|
||||
import fs from "node:fs";
|
||||
import path from "path";
|
||||
|
||||
import { DropletHandler, generateManifest } from "../index.js";
|
||||
import { generateManifest } from "../index.js";
|
||||
|
||||
test("numerous small file", async (t) => {
|
||||
// Setup test dir
|
||||
@@ -18,17 +18,11 @@ test("numerous small file", async (t) => {
|
||||
fs.writeFileSync(fileName, i.toString());
|
||||
}
|
||||
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
const manifest = JSON.parse(
|
||||
await new Promise((r, e) =>
|
||||
generateManifest(
|
||||
dropletHandler,
|
||||
dirName,
|
||||
(_, __) => {},
|
||||
(_, __) => {},
|
||||
(err, manifest) => (err ? e(err) : r(manifest))
|
||||
)
|
||||
await generateManifest(
|
||||
dirName,
|
||||
(_, __) => {},
|
||||
(_, __) => {}
|
||||
)
|
||||
);
|
||||
|
||||
@@ -75,17 +69,11 @@ test.skip("performance test", async (t) => {
|
||||
randomStream.on("end", r);
|
||||
});
|
||||
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
const start = Date.now();
|
||||
await new Promise((r, e) =>
|
||||
generateManifest(
|
||||
dropletHandler,
|
||||
dirName,
|
||||
(_, __) => {},
|
||||
(_, __) => {},
|
||||
(err, manifest) => (err ? e(err) : r(manifest))
|
||||
)
|
||||
await generateManifest(
|
||||
dirName,
|
||||
(_, __) => {},
|
||||
(_, __) => {}
|
||||
);
|
||||
const end = Date.now();
|
||||
|
||||
@@ -108,17 +96,11 @@ test("special characters", async (t) => {
|
||||
fs.writeFileSync(fileName, i.toString());
|
||||
}
|
||||
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
const manifest = JSON.parse(
|
||||
await new Promise((r, e) =>
|
||||
generateManifest(
|
||||
dropletHandler,
|
||||
dirName,
|
||||
(_, __) => {},
|
||||
(_, __) => {},
|
||||
(err, manifest) => (err ? e(err) : r(manifest))
|
||||
)
|
||||
await generateManifest(
|
||||
dirName,
|
||||
(_, __) => {},
|
||||
(_, __) => {}
|
||||
)
|
||||
);
|
||||
|
||||
@@ -136,4 +118,4 @@ test("special characters", async (t) => {
|
||||
}
|
||||
|
||||
fs.rmSync(dirName, { recursive: true });
|
||||
});
|
||||
});
|
||||
|
||||
+19
-42
@@ -4,7 +4,7 @@ import path from "path";
|
||||
import { createHash } from "node:crypto";
|
||||
import prettyBytes from "pretty-bytes";
|
||||
|
||||
import droplet, { DropletHandler, generateManifest } from "../index.js";
|
||||
import droplet, { generateManifest, listFiles, readFile } from "../index.js";
|
||||
|
||||
test("check alt thread util", async (t) => {
|
||||
let endtime1, endtime2;
|
||||
@@ -17,6 +17,7 @@ test("check alt thread util", async (t) => {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
endtime2 = Date.now();
|
||||
|
||||
if (!endtime1) return t.fail("alt thread function never ran");
|
||||
const difference = endtime2 - endtime1;
|
||||
if (difference >= 600) {
|
||||
t.fail("likely isn't multithreaded, difference: " + difference);
|
||||
@@ -36,8 +37,7 @@ test("list files", async (t) => {
|
||||
fs.writeFileSync(dirName + "/subdir/one.txt", "the first subdir");
|
||||
fs.writeFileSync(dirName + "/subddir/two.txt", "the second");
|
||||
|
||||
const dropletHandler = new DropletHandler();
|
||||
const files = dropletHandler.listFiles(dirName);
|
||||
const files = await listFiles(dirName);
|
||||
|
||||
t.assert(
|
||||
files.sort().join("\n"),
|
||||
@@ -56,9 +56,7 @@ test("read file", async (t) => {
|
||||
|
||||
fs.writeFileSync(dirName + "/TESTFILE", testString);
|
||||
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
const stream = dropletHandler.readFile(
|
||||
const stream = await readFile(
|
||||
dirName,
|
||||
"TESTFILE",
|
||||
BigInt(0),
|
||||
@@ -84,13 +82,7 @@ test("read file offset", async (t) => {
|
||||
const testString = "0123456789";
|
||||
fs.writeFileSync(dirName + "/TESTFILE", testString);
|
||||
|
||||
const dropletHandler = new DropletHandler();
|
||||
const stream = dropletHandler.readFile(
|
||||
dirName,
|
||||
"TESTFILE",
|
||||
BigInt(1),
|
||||
BigInt(4)
|
||||
);
|
||||
const stream = await readFile(dirName, "TESTFILE", BigInt(1), BigInt(4));
|
||||
|
||||
let finalString = "";
|
||||
|
||||
@@ -110,9 +102,8 @@ test("read file offset", async (t) => {
|
||||
|
||||
test.skip("zip speed test", async (t) => {
|
||||
t.timeout(100_000_000);
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
const stream = dropletHandler.readFile("./assets/TheGame.zip", "setup.exe");
|
||||
const stream = await readFile("./assets/TheGame.zip", "setup.exe");
|
||||
|
||||
let totalRead = 0;
|
||||
let totalSeconds = 0;
|
||||
@@ -148,19 +139,14 @@ test.skip("zip speed test", async (t) => {
|
||||
|
||||
test("zip manifest test", async (t) => {
|
||||
const zipFiles = fs.readdirSync("./assets").filter((v) => v.endsWith(".zip"));
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
for (const zipFile of zipFiles) {
|
||||
console.log("generating manifest for " + zipFile);
|
||||
const manifest = JSON.parse(
|
||||
await new Promise((r, e) =>
|
||||
generateManifest(
|
||||
dropletHandler,
|
||||
"./assets/" + zipFile,
|
||||
(_, __) => {},
|
||||
(_, __) => {},
|
||||
(err, manifest) => (err ? e(err) : r(manifest))
|
||||
)
|
||||
await generateManifest(
|
||||
"./assets/" + zipFile,
|
||||
(_, __) => {},
|
||||
(_, __) => {}
|
||||
)
|
||||
);
|
||||
|
||||
@@ -168,15 +154,12 @@ test("zip manifest test", async (t) => {
|
||||
let start = 0;
|
||||
for (const [chunkIndex, length] of data.lengths.entries()) {
|
||||
const hash = createHash("md5");
|
||||
const stream = (
|
||||
await dropletHandler.readFile(
|
||||
"./assets/" + zipFile,
|
||||
filename,
|
||||
BigInt(start),
|
||||
BigInt(start + length)
|
||||
)
|
||||
const stream = await readFile(
|
||||
"./assets/" + zipFile,
|
||||
filename,
|
||||
BigInt(start),
|
||||
BigInt(start + length)
|
||||
);
|
||||
console.log(stream);
|
||||
|
||||
let streamLength = 0;
|
||||
await stream.pipeTo(
|
||||
@@ -208,17 +191,11 @@ test("zip manifest test", async (t) => {
|
||||
});
|
||||
|
||||
test.skip("partially compress zip test", async (t) => {
|
||||
const dropletHandler = new DropletHandler();
|
||||
|
||||
const manifest = JSON.parse(
|
||||
await new Promise((r, e) =>
|
||||
generateManifest(
|
||||
dropletHandler,
|
||||
"./assets/my horror game.zip",
|
||||
(_, __) => {},
|
||||
(_, __) => {},
|
||||
(err, manifest) => (err ? e(err) : r(manifest))
|
||||
)
|
||||
await generateManifest(
|
||||
"./assets/my horror game.zip",
|
||||
(_, __) => {},
|
||||
(_, __) => {}
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
Vendored
+9
-11
@@ -1,15 +1,5 @@
|
||||
/* auto-generated by NAPI-RS */
|
||||
/* eslint-disable */
|
||||
/** * Persistent object so we can cache things between commands
|
||||
*/
|
||||
export declare class DropletHandler {
|
||||
constructor()
|
||||
hasBackendForPath(path: string): boolean
|
||||
listFiles(path: string): Array<string>
|
||||
peekFile(path: string, subPath: string): bigint
|
||||
readFile(path: string, subPath: string, start?: bigint | undefined | null, end?: bigint | undefined | null): ReadableStream
|
||||
}
|
||||
|
||||
export declare class Script {
|
||||
|
||||
}
|
||||
@@ -26,10 +16,18 @@ export declare function callAltThreadFunc(tsfn: ((err: Error | null, ) => any)):
|
||||
|
||||
export declare function generateClientCertificate(clientId: string, clientName: string, rootCa: string, rootCaPrivate: string): Array<string>
|
||||
|
||||
export declare function generateManifest(dropletHandler: DropletHandler, dir: string, progressSfn: ((err: Error | null, arg: number) => any), logSfn: ((err: Error | null, arg: string) => any), callbackSfn: ((err: Error | null, arg: string) => any)): void
|
||||
export declare function generateManifest(dir: string, progressSfn: ((err: Error | null, arg: number) => any), logSfn: ((err: Error | null, arg: string) => any)): Promise<string>
|
||||
|
||||
export declare function generateRootCa(): Array<string>
|
||||
|
||||
export declare function hasBackendForPath(path: string): boolean
|
||||
|
||||
export declare function listFiles(path: string): Promise<Array<string>>
|
||||
|
||||
export declare function peekFile(path: string, subPath: string): Promise<bigint>
|
||||
|
||||
export declare function readFile(path: string, subPath: string, start?: bigint | undefined | null, end?: bigint | undefined | null): ReadableStream<Buffer>
|
||||
|
||||
export declare function signNonce(privateKey: string, nonce: string): string
|
||||
|
||||
export declare function verifyClientCertificate(clientCert: string, rootCa: string): boolean
|
||||
|
||||
@@ -376,13 +376,16 @@ if (!nativeBinding) {
|
||||
}
|
||||
|
||||
module.exports = nativeBinding
|
||||
module.exports.DropletHandler = nativeBinding.DropletHandler
|
||||
module.exports.Script = nativeBinding.Script
|
||||
module.exports.ScriptEngine = nativeBinding.ScriptEngine
|
||||
module.exports.callAltThreadFunc = nativeBinding.callAltThreadFunc
|
||||
module.exports.generateClientCertificate = nativeBinding.generateClientCertificate
|
||||
module.exports.generateManifest = nativeBinding.generateManifest
|
||||
module.exports.generateRootCa = nativeBinding.generateRootCa
|
||||
module.exports.hasBackendForPath = nativeBinding.hasBackendForPath
|
||||
module.exports.listFiles = nativeBinding.listFiles
|
||||
module.exports.peekFile = nativeBinding.peekFile
|
||||
module.exports.readFile = nativeBinding.readFile
|
||||
module.exports.signNonce = nativeBinding.signNonce
|
||||
module.exports.verifyClientCertificate = nativeBinding.verifyClientCertificate
|
||||
module.exports.verifyNonce = nativeBinding.verifyNonce
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@drop-oss/droplet",
|
||||
"version": "3.5.1",
|
||||
"version": "4.0.0",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
"napi": {
|
||||
|
||||
+72
-101
@@ -1,14 +1,14 @@
|
||||
use std::{collections::HashMap, sync::Arc, thread};
|
||||
|
||||
use droplet_rs::versions::types::VersionBackend;
|
||||
use napi::{
|
||||
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
|
||||
Result,
|
||||
};
|
||||
use serde_json::json;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::version::DropletHandler;
|
||||
use crate::version::create_backend_for_path;
|
||||
|
||||
const CHUNK_SIZE: usize = 1024 * 1024 * 64;
|
||||
|
||||
@@ -30,117 +30,88 @@ pub fn call_alt_thread_func(tsfn: Arc<ThreadsafeFunction<()>>) -> Result<(), Str
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn generate_manifest<'a>(
|
||||
droplet_handler: &mut DropletHandler,
|
||||
pub async fn generate_manifest<'a>(
|
||||
dir: String,
|
||||
progress_sfn: ThreadsafeFunction<i32>,
|
||||
log_sfn: ThreadsafeFunction<String>,
|
||||
callback_sfn: ThreadsafeFunction<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
let backend: &mut Box<dyn VersionBackend + Send> = droplet_handler
|
||||
.create_backend_for_path(dir)
|
||||
.ok_or(napi::Error::from_reason(
|
||||
) -> anyhow::Result<String> {
|
||||
let mut backend = create_backend_for_path(dir).ok_or(napi::Error::from_reason(
|
||||
"Could not create backend for path.",
|
||||
))?;
|
||||
|
||||
// This is unsafe (obviously)
|
||||
// But it's allg as long the DropletHandler doesn't get
|
||||
// dropped while we're generating the manifest.
|
||||
let backend: &'static mut Box<dyn VersionBackend + Send> =
|
||||
unsafe { std::mem::transmute(backend) };
|
||||
let required_single_file = backend.require_whole_files();
|
||||
|
||||
let required_single_file = backend.require_whole_files();
|
||||
let files = backend.list_files().await?;
|
||||
// Filepath to chunk data
|
||||
let mut chunks: HashMap<String, ChunkData> = HashMap::new();
|
||||
|
||||
thread::spawn(move || {
|
||||
let callback_borrow = &callback_sfn;
|
||||
let total: i32 = files.len() as i32;
|
||||
let mut i: i32 = 0;
|
||||
|
||||
let mut inner = move || -> Result<()> {
|
||||
let files = backend.list_files()?;
|
||||
let mut buf = [0u8; 1024 * 16];
|
||||
|
||||
// Filepath to chunk data
|
||||
let mut chunks: HashMap<String, ChunkData> = HashMap::new();
|
||||
for version_file in files {
|
||||
let mut reader = backend.reader(&version_file, 0, 0).await?;
|
||||
|
||||
let total: i32 = files.len() as i32;
|
||||
let mut i: i32 = 0;
|
||||
|
||||
let mut buf = [0u8; 1024 * 16];
|
||||
|
||||
for version_file in files {
|
||||
let mut reader = backend.reader(&version_file, 0, 0)?;
|
||||
|
||||
let mut chunk_data = ChunkData {
|
||||
permissions: version_file.permission,
|
||||
ids: Vec::new(),
|
||||
checksums: Vec::new(),
|
||||
lengths: Vec::new(),
|
||||
};
|
||||
|
||||
let mut chunk_index = 0;
|
||||
loop {
|
||||
let mut length = 0;
|
||||
let mut buffer: Vec<u8> = Vec::new();
|
||||
let mut file_empty = false;
|
||||
|
||||
loop {
|
||||
let read = reader.read(&mut buf)?;
|
||||
|
||||
length += read;
|
||||
|
||||
// If we're out of data, add this chunk and then move onto the next file
|
||||
if read == 0 {
|
||||
file_empty = true;
|
||||
break;
|
||||
}
|
||||
|
||||
buffer.extend_from_slice(&buf[0..read]);
|
||||
|
||||
if length >= CHUNK_SIZE && !required_single_file {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let chunk_id = Uuid::new_v4();
|
||||
let checksum = md5::compute(buffer).0;
|
||||
let checksum_string = hex::encode(checksum);
|
||||
|
||||
chunk_data.ids.push(chunk_id.to_string());
|
||||
chunk_data.checksums.push(checksum_string);
|
||||
chunk_data.lengths.push(length);
|
||||
|
||||
let log_str = format!(
|
||||
"Processed chunk {} for {}",
|
||||
chunk_index, &version_file.relative_filename
|
||||
);
|
||||
|
||||
log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
|
||||
|
||||
chunk_index += 1;
|
||||
|
||||
if file_empty {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
chunks.insert(version_file.relative_filename, chunk_data);
|
||||
|
||||
i += 1;
|
||||
let progress = i * 100 / total;
|
||||
progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
|
||||
}
|
||||
|
||||
callback_borrow.call(
|
||||
Ok(json!(chunks).to_string()),
|
||||
ThreadsafeFunctionCallMode::Blocking,
|
||||
);
|
||||
|
||||
Ok(())
|
||||
let mut chunk_data = ChunkData {
|
||||
permissions: version_file.permission,
|
||||
ids: Vec::new(),
|
||||
checksums: Vec::new(),
|
||||
lengths: Vec::new(),
|
||||
};
|
||||
|
||||
let result = inner();
|
||||
if let Err(generate_err) = result {
|
||||
callback_borrow.call(Err(generate_err), ThreadsafeFunctionCallMode::Blocking);
|
||||
}
|
||||
});
|
||||
let mut chunk_index = 0;
|
||||
loop {
|
||||
let mut length = 0;
|
||||
let mut buffer: Vec<u8> = Vec::new();
|
||||
let mut file_empty = false;
|
||||
|
||||
Ok(())
|
||||
loop {
|
||||
let read = reader.read(&mut buf).await?;
|
||||
|
||||
length += read;
|
||||
|
||||
// If we're out of data, add this chunk and then move onto the next file
|
||||
if read == 0 {
|
||||
file_empty = true;
|
||||
break;
|
||||
}
|
||||
|
||||
buffer.extend_from_slice(&buf[0..read]);
|
||||
|
||||
if length >= CHUNK_SIZE && !required_single_file {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let chunk_id = Uuid::new_v4();
|
||||
let checksum = md5::compute(buffer).0;
|
||||
let checksum_string = hex::encode(checksum);
|
||||
|
||||
chunk_data.ids.push(chunk_id.to_string());
|
||||
chunk_data.checksums.push(checksum_string);
|
||||
chunk_data.lengths.push(length);
|
||||
|
||||
let log_str = format!(
|
||||
"Processed chunk {} for {}",
|
||||
chunk_index, &version_file.relative_filename
|
||||
);
|
||||
|
||||
log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
|
||||
|
||||
chunk_index += 1;
|
||||
|
||||
if file_empty {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
chunks.insert(version_file.relative_filename, chunk_data);
|
||||
|
||||
i += 1;
|
||||
let progress = i * 100 / total;
|
||||
progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
|
||||
}
|
||||
|
||||
Ok(json!(chunks).to_string())
|
||||
}
|
||||
|
||||
+77
-91
@@ -6,116 +6,102 @@ use std::{
|
||||
};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use droplet_rs::versions::{create_backend_constructor, types::{ReadToAsyncRead, VersionBackend, VersionFile}};
|
||||
use napi::{bindgen_prelude::*, sys::napi_value__, tokio_stream::StreamExt};
|
||||
use droplet_rs::versions::{
|
||||
create_backend_constructor,
|
||||
types::{VersionBackend, VersionFile},
|
||||
};
|
||||
use napi::{
|
||||
bindgen_prelude::*,
|
||||
sys::napi_value__,
|
||||
tokio_stream::{wrappers::ReceiverStream, StreamExt},
|
||||
};
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
use tokio_util::codec::{BytesCodec, FramedRead};
|
||||
|
||||
pub fn create_backend_for_path(path: String) -> Option<Box<dyn VersionBackend + Send>> {
|
||||
let fs_path = Path::new(&path);
|
||||
let constructor = create_backend_constructor(fs_path)?;
|
||||
|
||||
/**
|
||||
* Persistent object so we can cache things between commands
|
||||
*/
|
||||
#[napi(js_name = "DropletHandler")]
|
||||
pub struct DropletHandler<'a> {
|
||||
backend_cache: HashMap<String, Box<dyn VersionBackend + Send + 'a>>,
|
||||
Some(constructor().ok()?)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl<'a> DropletHandler<'a> {
|
||||
#[napi(constructor)]
|
||||
pub fn new() -> Self {
|
||||
DropletHandler {
|
||||
backend_cache: HashMap::new(),
|
||||
}
|
||||
}
|
||||
pub fn has_backend_for_path(path: String) -> bool {
|
||||
let path = Path::new(&path);
|
||||
|
||||
pub fn create_backend_for_path(
|
||||
&mut self,
|
||||
path: String,
|
||||
) -> Option<&mut Box<dyn VersionBackend + Send + 'a>> {
|
||||
let fs_path = Path::new(&path);
|
||||
let constructor = create_backend_constructor(fs_path)?;
|
||||
let has_backend = create_backend_constructor(path).is_some();
|
||||
|
||||
let existing_backend = match self.backend_cache.entry(path) {
|
||||
std::collections::hash_map::Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
|
||||
std::collections::hash_map::Entry::Vacant(vacant_entry) => {
|
||||
let backend = constructor().ok()?;
|
||||
vacant_entry.insert(backend)
|
||||
}
|
||||
};
|
||||
has_backend
|
||||
}
|
||||
|
||||
Some(existing_backend)
|
||||
}
|
||||
#[napi]
|
||||
pub async fn list_files(path: String) -> Result<Vec<String>> {
|
||||
let mut backend =
|
||||
create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
|
||||
let files = backend.list_files().await?;
|
||||
Ok(files.into_iter().map(|e| e.relative_filename).collect())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn has_backend_for_path(&self, path: String) -> bool {
|
||||
let path = Path::new(&path);
|
||||
#[napi]
|
||||
pub async fn peek_file(path: String, sub_path: String) -> Result<u64> {
|
||||
let mut backend =
|
||||
create_backend_for_path(path).ok_or(napi::Error::from_reason("No backend for path"))?;
|
||||
|
||||
let has_backend = create_backend_constructor(path).is_some();
|
||||
let file = backend.peek_file(sub_path).await?;
|
||||
|
||||
has_backend
|
||||
}
|
||||
Ok(file.size)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn list_files(&mut self, path: String) -> Result<Vec<String>> {
|
||||
let backend = self
|
||||
.create_backend_for_path(path)
|
||||
.ok_or(napi::Error::from_reason("No backend for path"))?;
|
||||
let files = backend.list_files()?;
|
||||
Ok(files.into_iter().map(|e| e.relative_filename).collect())
|
||||
}
|
||||
#[napi]
|
||||
pub fn read_file(
|
||||
path: String,
|
||||
sub_path: String,
|
||||
env: &Env,
|
||||
start: Option<BigInt>,
|
||||
end: Option<BigInt>,
|
||||
) -> anyhow::Result<ReadableStream<BufferSlice>> {
|
||||
let mut backend = create_backend_for_path(path).ok_or(anyhow!("Failed to create backend."))?;
|
||||
let version_file = VersionFile {
|
||||
relative_filename: sub_path,
|
||||
permission: 0, // Shouldn't matter
|
||||
size: 0, // Shouldn't matter
|
||||
};
|
||||
|
||||
#[napi]
|
||||
pub fn peek_file(&mut self, path: String, sub_path: String) -> Result<u64> {
|
||||
let backend = self
|
||||
.create_backend_for_path(path)
|
||||
.ok_or(napi::Error::from_reason("No backend for path"))?;
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(100);
|
||||
|
||||
let file = backend.peek_file(sub_path)?;
|
||||
|
||||
Ok(file.size)
|
||||
}
|
||||
|
||||
#[napi(ts_return_type = "ReadableStream")]
|
||||
pub fn read_file(
|
||||
&mut self,
|
||||
reference: Reference<DropletHandler<'static>>,
|
||||
path: String,
|
||||
sub_path: String,
|
||||
env: Env,
|
||||
start: Option<BigInt>,
|
||||
end: Option<BigInt>,
|
||||
) -> anyhow::Result<*mut napi_value__> {
|
||||
let stream = reference.share_with(env, |handler| {
|
||||
let backend = handler
|
||||
.create_backend_for_path(path)
|
||||
.ok_or(anyhow!("Failed to create backend."))?;
|
||||
let version_file = VersionFile {
|
||||
relative_filename: sub_path,
|
||||
permission: 0, // Shouldn't matter
|
||||
size: 0, // Shouldn't matter
|
||||
};
|
||||
// Use `?` operator for cleaner error propagation from `Option`
|
||||
let reader = backend.reader(
|
||||
spawn(async move {
|
||||
// Use `?` operator for cleaner error propagation from `Option`
|
||||
let reader = backend
|
||||
.reader(
|
||||
&version_file,
|
||||
start.map(|e| e.get_u64().1).unwrap_or(0),
|
||||
end.map(|e| e.get_u64().1).unwrap_or(0),
|
||||
)?;
|
||||
)
|
||||
.await
|
||||
.expect("failed to open file");
|
||||
|
||||
let async_reader = ReadToAsyncRead { inner: reader };
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Create a FramedRead stream with BytesCodec for chunking
|
||||
let stream = FramedRead::new(async_reader, BytesCodec::new())
|
||||
// Use StreamExt::map to transform each Result item
|
||||
.map(|result_item| {
|
||||
result_item
|
||||
// Apply Result::map to transform Ok(BytesMut) to Ok(Vec<u8>)
|
||||
.map(|bytes| bytes.to_vec())
|
||||
// Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
|
||||
.map_err(napi::Error::from) // napi::Error implements From<tokio::io::Error>
|
||||
});
|
||||
ReadableStream::create_with_stream_bytes(&env, stream)
|
||||
})?;
|
||||
let mut read_buf = [0u8; 4096];
|
||||
|
||||
Ok(stream.raw())
|
||||
}
|
||||
loop {
|
||||
let amount = reader.read(&mut read_buf).await;
|
||||
if amount.is_err() {
|
||||
let _ = tx.send(Err(napi::Error::from_reason(
|
||||
amount.unwrap_err().to_string(),
|
||||
))).await;
|
||||
break;
|
||||
}
|
||||
let amount = amount.unwrap();
|
||||
if amount == 0 {
|
||||
break;
|
||||
}
|
||||
tx.send(Ok(read_buf[0..amount].to_vec())).await.expect("failed to send data");
|
||||
}
|
||||
});
|
||||
|
||||
return Ok(ReadableStream::create_with_stream_bytes(
|
||||
env,
|
||||
ReceiverStream::new(rx),
|
||||
)?);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user