14 Commits

Author SHA1 Message Date
96c1b15de7 remove unneeded deps 2025-10-02 17:14:26 +10:00
bd6d7060fd feat: the 7z update 2025-10-02 17:06:58 +10:00
0431eebaa7 fix: remove lua tests 2025-08-25 13:02:00 +10:00
e66a6581cb fix: temporary remove luajit for compliation reasons 2025-08-25 12:43:23 +10:00
817c3cf503 feat: script backend, fixes 2025-08-25 12:35:12 +10:00
0d01809fd0 feat: no panik 2025-08-25 12:20:51 +10:00
ba35ca9a14 feat: start of scripting engine 2025-08-24 13:50:44 +10:00
ae4648845e feat: add support for partially deflated zips 2025-08-17 11:21:09 +10:00
bd30464a08 fix: manifest generation with multiple chunks 2025-08-15 21:56:33 +10:00
c67cca4ee0 fix: remove debug println 2025-08-15 21:41:48 +10:00
cae208a3e0 fix: zip read sizing 2025-08-15 21:30:25 +10:00
4276b9d668 fix: skip zip test 2025-08-15 19:47:50 +10:00
4fb9bb7563 fix: manifest sizing for slow backends 2025-08-15 16:49:18 +10:00
913dc2f58d feat: add zip speed test 2025-08-15 12:17:10 +10:00
18 changed files with 1607 additions and 332 deletions

893
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -9,14 +9,9 @@ crate-type = ["cdylib"]
[dependencies]
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
napi = { version = "3.0.0-beta.11", default-features = false, features = [
"napi6",
"async",
"web_stream",
] }
napi = { version = "3.0.0-beta.11", default-features = false, features = ["napi6", "async", "web_stream", "error_anyhow"] }
napi-derive = "3.0.0-beta.11"
hex = "0.4.3"
serde_json = "1.0.128"
md5 = "0.7.0"
time-macros = "0.2.22"
time = "0.3.41"
@ -24,12 +19,12 @@ webpki = "0.22.4"
ring = "0.17.14"
tokio = { version = "1.45.1", features = ["fs", "io-util"] }
tokio-util = { version = "0.7.15", features = ["codec"] }
rawzip = "0.3.0"
dyn-clone = "1.0.20"
flate2 = "1.1.2"
[package.metadata.patch]
crates = ["rawzip"]
rhai = "1.22.2"
# mlua = { version = "0.11.2", features = ["luajit"] }
boa_engine = "0.20.0"
serde_json = "1.0.143"
anyhow = "1.0.99"
[dependencies.x509-parser]
version = "0.17.0"

22
__test__/debug.spec.mjs Normal file
View File

@ -0,0 +1,22 @@
import test from "ava";
import { DropletHandler, generateManifest } from "../index.js";
test.skip("debug", async (t) => {
const handler = new DropletHandler();
console.log("created handler");
const manifest = JSON.parse(
await new Promise((r, e) =>
generateManifest(
handler,
"./assets/TheGame.zip",
(_, __) => {},
(_, __) => {},
(err, manifest) => (err ? e(err) : r(manifest))
)
)
);
return t.pass();
});

View File

@ -93,3 +93,47 @@ test.skip("performance test", async (t) => {
fs.rmSync(dirName, { recursive: true });
});
test("special characters", async (t) => {
// Setup test dir
const dirName = "./.test/sc";
if (fs.existsSync(dirName)) fs.rmSync(dirName, { recursive: true });
fs.mkdirSync(dirName, { recursive: true });
// Config
const fileNames = ["Technická podpora.rtf", "Servicio técnico.rtf"];
for (let i = 0; i < fileNames.length; i++) {
const fileName = path.join(dirName, fileNames[i]);
fs.writeFileSync(fileName, i.toString());
}
const dropletHandler = new DropletHandler();
const manifest = JSON.parse(
await new Promise((r, e) =>
generateManifest(
dropletHandler,
dirName,
(_, __) => {},
(_, __) => {},
(err, manifest) => (err ? e(err) : r(manifest))
)
)
);
// Check the first few checksums
const checksums = [
"cfcd208495d565ef66e7dff9f98764da",
"c4ca4238a0b923820dcc509a6f75849b",
];
for (let index in checksums) {
const entry = manifest[fileNames[index]];
if (!entry) return t.fail(`manifest missing file ${index}`);
const checksum = entry.checksums[0];
t.is(checksum, checksums[index], `checksums do not match for ${index}`);
}
fs.rmSync(dirName, { recursive: true });
});

62
__test__/script.spec.mjs Normal file
View File

@ -0,0 +1,62 @@
import test from "ava";
import { ScriptEngine } from "../index.js";
test.skip("lua syntax fail", (t) => {
const scriptEngine = new ScriptEngine();
const luaIshCode = `
print("hello world);
`;
try {
const script = scriptEngine.buildLuaScript(luaIshCode);
} catch {
return t.pass();
}
t.fail();
});
test("js syntax fail", (t) => {
const scriptEngine = new ScriptEngine();
const jsIshCode = `
const v = "hello world;
`;
try {
const script = scriptEngine.buildJsScript(jsIshCode);
} catch {
return t.pass();
}
t.fail();
});
test("js", (t) => {
const scriptEngine = new ScriptEngine();
const jsModule = `
const v = "1" + "2";
["1", "2", "3", v]
`;
const script = scriptEngine.buildJsScript(jsModule);
scriptEngine.fetchStrings(script);
t.pass();
});
test.skip("lua", (t) => {
const scriptEngine = new ScriptEngine();
const luaModule = `
local arr = {"1", "2"};
return arr;
`;
const script = scriptEngine.buildLuaScript(luaModule);
scriptEngine.fetchStrings(script);
t.pass();
});

View File

@ -1,6 +1,8 @@
import test from "ava";
import fs from "node:fs";
import path from "path";
import { createHash } from "node:crypto";
import prettyBytes from "pretty-bytes";
import droplet, { DropletHandler, generateManifest } from "../index.js";
@ -56,7 +58,12 @@ test("read file", async (t) => {
const dropletHandler = new DropletHandler();
const stream = dropletHandler.readFile(dirName, "TESTFILE");
const stream = dropletHandler.readFile(
dirName,
"TESTFILE",
BigInt(0),
BigInt(testString.length)
);
let finalString = "";
@ -78,7 +85,12 @@ test("read file offset", async (t) => {
fs.writeFileSync(dirName + "/TESTFILE", testString);
const dropletHandler = new DropletHandler();
const stream = dropletHandler.readFile(dirName, "TESTFILE", BigInt(1), BigInt(4));
const stream = dropletHandler.readFile(
dirName,
"TESTFILE",
BigInt(1),
BigInt(4)
);
let finalString = "";
@ -96,10 +108,45 @@ test("read file offset", async (t) => {
fs.rmSync(dirName, { recursive: true });
});
test("zip file reader", async (t) => {
return t.pass();
test.skip("zip speed test", async (t) => {
t.timeout(100_000_000);
const dropletHandler = new DropletHandler();
t.timeout(10_000);
const stream = dropletHandler.readFile("./assets/TheGame.zip", "setup.exe");
let totalRead = 0;
let totalSeconds = 0;
let lastTime = process.hrtime.bigint();
const timeThreshold = BigInt(1_000_000_000);
let runningTotal = 0;
let runningTime = BigInt(0);
for await (const chunk of stream.getStream()) {
// Do something with each 'chunk'
const currentTime = process.hrtime.bigint();
const timeDiff = currentTime - lastTime;
lastTime = currentTime;
runningTime += timeDiff;
runningTotal += chunk.length;
if (runningTime >= timeThreshold) {
console.log(`${prettyBytes(runningTotal)}/s`);
totalRead += runningTotal;
totalSeconds += 1;
runningTime = BigInt(0);
runningTotal = 0;
}
}
const roughAverage = totalRead / totalSeconds;
console.log(`total rough average: ${prettyBytes(roughAverage)}/s`);
t.pass();
});
test.skip("zip manifest test", async (t) => {
const dropletHandler = new DropletHandler();
const manifest = JSON.parse(
await new Promise((r, e) =>
@ -113,20 +160,61 @@ test("zip file reader", async (t) => {
)
);
const stream = dropletHandler.readFile(
"./assets/TheGame.zip",
"setup.exe",
BigInt(10),
BigInt(20)
);
for (const [filename, data] of Object.entries(manifest)) {
let start = 0;
for (const [chunkIndex, length] of data.lengths.entries()) {
const hash = createHash("md5");
const stream = (
await dropletHandler.readFile(
"./assets/TheGame.zip",
filename,
BigInt(start),
BigInt(start + length)
)
).getStream();
let streamLength = 0;
await stream.pipeTo(
new WritableStream({
write(chunk) {
streamLength += chunk.length;
hash.update(chunk);
},
})
);
let finalString = "";
for await (const chunk of stream.getStream()) {
// Do something with each 'chunk'
finalString = String.fromCharCode.apply(null, chunk);
if(finalString.length > 100) break;
if (streamLength != length)
return t.fail(
`stream length for chunk index ${chunkIndex} was not expected: real: ${streamLength} vs expected: ${length}`
);
const digest = hash.digest("hex");
if (data.checksums[chunkIndex] != digest)
return t.fail(
`checksums did not match for chunk index ${chunkIndex}: real: ${digest} vs expected: ${data.checksums[chunkIndex]}`
);
start += length;
}
}
t.pass();
});
test.skip("partially compress zip test", async (t) => {
const dropletHandler = new DropletHandler();
const manifest = JSON.parse(
await new Promise((r, e) =>
generateManifest(
dropletHandler,
"./assets/my horror game.zip",
(_, __) => {},
(_, __) => {},
(err, manifest) => (err ? e(err) : r(manifest))
)
)
);
return t.pass();
});

View File

@ -1,3 +1,4 @@
yes "droplet is awesome" | dd of=./setup.exe bs=1024 count=1000000
# yes "droplet is awesome" | dd of=./setup.exe bs=1024 count=1000000
dd if=/dev/random of=./setup.exe bs=1024 count=1000000
zip TheGame.zip setup.exe
rm setup.exe

12
index.d.ts vendored
View File

@ -15,6 +15,18 @@ export declare class JsDropStreamable {
getStream(): any
}
export declare class Script {
}
export declare class ScriptEngine {
constructor()
buildRhaiScript(content: string): Script
buildJsScript(content: string): Script
execute(script: Script): void
fetchStrings(script: Script): Array<string>
}
export declare function callAltThreadFunc(tsfn: ((err: Error | null, ) => any)): void
export declare function generateClientCertificate(clientId: string, clientName: string, rootCa: string, rootCaPrivate: string): Array<string>

View File

@ -378,6 +378,8 @@ if (!nativeBinding) {
module.exports = nativeBinding
module.exports.DropletHandler = nativeBinding.DropletHandler
module.exports.JsDropStreamable = nativeBinding.JsDropStreamable
module.exports.Script = nativeBinding.Script
module.exports.ScriptEngine = nativeBinding.ScriptEngine
module.exports.callAltThreadFunc = nativeBinding.callAltThreadFunc
module.exports.generateClientCertificate = nativeBinding.generateClientCertificate
module.exports.generateManifest = nativeBinding.generateManifest

View File

@ -1,6 +1,6 @@
{
"name": "@drop-oss/droplet",
"version": "2.0.2",
"version": "3.1.0",
"main": "index.js",
"types": "index.d.ts",
"napi": {
@ -24,7 +24,9 @@
"devDependencies": {
"@napi-rs/cli": "3.0.0-alpha.91",
"@types/node": "^22.13.10",
"ava": "^6.2.0"
"ava": "^6.2.0",
"pretty-bytes": "^7.0.1",
"tsimp": "^2.0.12"
},
"ava": {
"timeout": "3m",
@ -44,15 +46,12 @@
"build": "napi build --platform --release",
"build:debug": "napi build --platform",
"prepublishOnly": "napi prepublish -t npm",
"test": "ava",
"test": "ava ",
"universal": "napi universalize",
"version": "napi version"
},
"packageManager": "yarn@4.7.0",
"repository": {
"url": "git+https://github.com/Drop-OSS/droplet.git"
},
"dependencies": {
"tsimp": "^2.0.12"
}
}

View File

@ -1,9 +1,14 @@
#![deny(clippy::all)]
#![deny(clippy::unwrap_used)]
#![deny(clippy::expect_used)]
#![deny(clippy::panic)]
#![feature(trait_alias)]
#![feature(iterator_try_collect)]
pub mod manifest;
pub mod script;
pub mod ssl;
pub mod version;
#[macro_use]
extern crate napi_derive;
extern crate napi_derive;

View File

@ -1,10 +1,4 @@
use std::{
collections::HashMap,
io::{BufRead, BufReader},
path::Path,
sync::Arc,
thread,
};
use std::{collections::HashMap, sync::Arc, thread};
use napi::{
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
@ -41,70 +35,110 @@ pub fn generate_manifest<'a>(
progress_sfn: ThreadsafeFunction<i32>,
log_sfn: ThreadsafeFunction<String>,
callback_sfn: ThreadsafeFunction<String>,
) -> Result<()> {
let backend: &mut Box<dyn VersionBackend + Send> =
droplet_handler.create_backend_for_path(dir).ok_or(napi::Error::from_reason("Could not create backend for path."))?;
) -> anyhow::Result<()> {
let backend: &mut Box<dyn VersionBackend + Send> = droplet_handler
.create_backend_for_path(dir)
.ok_or(napi::Error::from_reason(
"Could not create backend for path.",
))?;
// This is unsafe (obviously)
// But it's allg as long the DropletHandler doesn't get
// dropped while we're generating the manifest.
let backend: &'static mut Box<dyn VersionBackend + Send> =
unsafe { std::mem::transmute(backend) };
let required_single_file = backend.require_whole_files();
thread::spawn(move || {
let files = backend.list_files();
let callback_borrow = &callback_sfn;
// Filepath to chunk data
let mut chunks: HashMap<String, ChunkData> = HashMap::new();
let mut inner = move || -> Result<()> {
let files = backend.list_files()?;
let total: i32 = files.len() as i32;
let mut i: i32 = 0;
// Filepath to chunk data
let mut chunks: HashMap<String, ChunkData> = HashMap::new();
for version_file in files {
let raw_reader = backend.reader(&version_file).unwrap();
let mut reader = BufReader::with_capacity(CHUNK_SIZE, raw_reader);
let total: i32 = files.len() as i32;
let mut i: i32 = 0;
let mut chunk_data = ChunkData {
permissions: version_file.permission,
ids: Vec::new(),
checksums: Vec::new(),
lengths: Vec::new(),
};
let mut buf = [0u8; 1024 * 16];
let mut chunk_index = 0;
loop {
let mut buffer: Vec<u8> = Vec::new();
reader.fill_buf().unwrap().clone_into(&mut buffer);
let length = buffer.len();
for version_file in files {
let mut reader = backend.reader(&version_file, 0, 0)?;
if length == 0 {
break;
let mut chunk_data = ChunkData {
permissions: version_file.permission,
ids: Vec::new(),
checksums: Vec::new(),
lengths: Vec::new(),
};
let mut chunk_index = 0;
loop {
let mut length = 0;
let mut buffer: Vec<u8> = Vec::new();
let mut file_empty = false;
loop {
let read = reader.read(&mut buf)?;
length += read;
// If we're out of data, add this chunk and then move onto the next file
if read == 0 {
file_empty = true;
break;
}
buffer.extend_from_slice(&buf[0..read]);
if length >= CHUNK_SIZE && !required_single_file {
break;
}
}
let chunk_id = Uuid::new_v4();
let checksum = md5::compute(buffer).0;
let checksum_string = hex::encode(checksum);
chunk_data.ids.push(chunk_id.to_string());
chunk_data.checksums.push(checksum_string);
chunk_data.lengths.push(length);
let log_str = format!(
"Processed chunk {} for {}",
chunk_index, &version_file.relative_filename
);
log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
chunk_index += 1;
if file_empty {
break;
}
}
let chunk_id = Uuid::new_v4();
let checksum = md5::compute(buffer).0;
let checksum_string = hex::encode(checksum);
chunks.insert(version_file.relative_filename, chunk_data);
chunk_data.ids.push(chunk_id.to_string());
chunk_data.checksums.push(checksum_string);
chunk_data.lengths.push(length);
let log_str = format!(
"Processed chunk {} for {}",
chunk_index, &version_file.relative_filename
);
log_sfn.call(Ok(log_str), ThreadsafeFunctionCallMode::Blocking);
reader.consume(length);
chunk_index += 1;
i += 1;
let progress = i * 100 / total;
progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
}
chunks.insert(version_file.relative_filename, chunk_data);
callback_borrow.call(
Ok(json!(chunks).to_string()),
ThreadsafeFunctionCallMode::Blocking,
);
i += 1;
let progress = i * 100 / total;
progress_sfn.call(Ok(progress), ThreadsafeFunctionCallMode::Blocking);
Ok(())
};
let result = inner();
if let Err(generate_err) = result {
callback_borrow.call(Err(generate_err), ThreadsafeFunctionCallMode::Blocking);
}
callback_sfn.call(
Ok(json!(chunks).to_string()),
ThreadsafeFunctionCallMode::Blocking,
);
});
Ok(())

133
src/script/mod.rs Normal file
View File

@ -0,0 +1,133 @@
use boa_engine::{Context, JsValue, Source};
// use mlua::{FromLuaMulti, Function, Lua};
use napi::Result;
use rhai::AST;
pub enum ScriptType {
Rhai,
Lua,
Javascript,
}
#[napi]
pub struct Script(ScriptInner);
pub enum ScriptInner {
Rhai { script: AST },
// Lua { script: Function },
Javascript { script: boa_engine::Script },
}
#[napi]
pub struct ScriptEngine {
rhai_engine: rhai::Engine,
// lua_engine: Lua,
js_engine: Context,
}
#[napi]
impl ScriptEngine {
#[napi(constructor)]
pub fn new() -> Self {
ScriptEngine {
rhai_engine: rhai::Engine::new(),
// lua_engine: Lua::new(),
js_engine: Context::default(),
}
}
#[napi]
pub fn build_rhai_script(&self, content: String) -> Result<Script> {
let script = self
.rhai_engine
.compile(content.clone())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
Ok(Script(ScriptInner::Rhai { script }))
}
/*
#[napi]
pub fn build_lua_script(&self, content: String) -> Result<Script> {
let func = self
.lua_engine
.load(content.clone())
.into_function()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
Ok(Script(ScriptInner::Lua { script: func }))
}
*/
#[napi]
pub fn build_js_script(&mut self, content: String) -> Result<Script> {
let source = Source::from_bytes(content.as_bytes());
let script = boa_engine::Script::parse(source, None, &mut self.js_engine)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
Ok(Script(ScriptInner::Javascript { script }))
}
fn execute_rhai_script<T>(&self, ast: &AST) -> Result<T>
where
T: Clone + 'static,
{
let v = self
.rhai_engine
.eval_ast::<T>(ast)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
Ok(v)
}
/*
fn execute_lua_script<T>(&self, function: &Function) -> Result<T>
where
T: FromLuaMulti,
{
let v = function
.call::<T>(())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
Ok(v)
}
*/
fn execute_js_script(&mut self, func: &boa_engine::Script) -> Result<JsValue> {
let v = func
.evaluate(&mut self.js_engine)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
Ok(v)
}
#[napi]
pub fn execute(&mut self, script: &mut Script) -> Result<()> {
match &script.0 {
ScriptInner::Rhai { script } => {
self.execute_rhai_script::<()>(script)?;
}
/*ScriptInner::Lua { script } => {
self.execute_lua_script::<()>(script)?;
}*/
ScriptInner::Javascript { script } => {
self.execute_js_script(script)?;
}
};
Ok(())
}
#[napi]
pub fn fetch_strings(&mut self, script: &mut Script) -> Result<Vec<String>> {
Ok(match &script.0 {
ScriptInner::Rhai { script } => self.execute_rhai_script(script)?,
//ScriptInner::Lua { script } => self.execute_lua_script(script)?,
ScriptInner::Javascript { script } => {
let v = self.execute_js_script(script)?;
serde_json::from_value(
v.to_json(&mut self.js_engine)
.map_err(|e| napi::Error::from_reason(e.to_string()))?,
)
.map_err(|e| napi::Error::from_reason(e.to_string()))?
}
})
}
}

View File

@ -1,4 +1,4 @@
use napi::Error;
use anyhow::anyhow;
use rcgen::{
CertificateParams, DistinguishedName, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData,
SubjectPublicKeyInfo,
@ -10,7 +10,7 @@ use x509_parser::parse_x509_certificate;
use x509_parser::pem::Pem;
#[napi]
pub fn generate_root_ca() -> Result<Vec<String>, Error> {
pub fn generate_root_ca() -> anyhow::Result<Vec<String>> {
let mut params = CertificateParams::default();
let mut name = DistinguishedName::new();
@ -22,7 +22,7 @@ pub fn generate_root_ca() -> Result<Vec<String>, Error> {
params.not_before = OffsetDateTime::now_utc();
params.not_after = OffsetDateTime::now_utc()
.checked_add(Duration::days(365 * 1000))
.unwrap();
.ok_or(anyhow!("failed to calculate end date"))?;
params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
@ -32,9 +32,8 @@ pub fn generate_root_ca() -> Result<Vec<String>, Error> {
KeyUsagePurpose::DigitalSignature,
];
let key_pair = KeyPair::generate().map_err(|e| napi::Error::from_reason(e.to_string()))?;
let certificate = CertificateParams::self_signed(params, &key_pair)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
let key_pair = KeyPair::generate()?;
let certificate = CertificateParams::self_signed(params, &key_pair)?;
// Returns certificate, then private key
Ok(vec![certificate.pem(), key_pair.serialize_pem()])
@ -46,13 +45,10 @@ pub fn generate_client_certificate(
_client_name: String,
root_ca: String,
root_ca_private: String,
) -> Result<Vec<String>, Error> {
let root_key_pair =
KeyPair::from_pem(&root_ca_private).map_err(|e| napi::Error::from_reason(e.to_string()))?;
let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
) -> anyhow::Result<Vec<String>> {
let root_key_pair = KeyPair::from_pem(&root_ca_private)?;
let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca)?;
let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair)?;
let mut params = CertificateParams::default();
@ -66,28 +62,24 @@ pub fn generate_client_certificate(
KeyUsagePurpose::DataEncipherment,
];
let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384)?;
let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair)?;
// Returns certificate, then private key
Ok(vec![certificate.pem(), key_pair.serialize_pem()])
}
#[napi]
pub fn verify_client_certificate(client_cert: String, root_ca: String) -> Result<bool, Error> {
pub fn verify_client_certificate(client_cert: String, root_ca: String) -> anyhow::Result<bool> {
let root_ca = Pem::iter_from_buffer(root_ca.as_bytes())
.next()
.unwrap()
.unwrap();
let root_ca = root_ca.parse_x509().unwrap();
.ok_or(anyhow!("no certificates in root ca"))??;
let root_ca = root_ca.parse_x509()?;
let client_cert = Pem::iter_from_buffer(client_cert.as_bytes())
.next()
.unwrap()
.unwrap();
let client_cert = client_cert.parse_x509().unwrap();
.ok_or(anyhow!("No client certs in chain."))??;
let client_cert = client_cert.parse_x509()?;
let valid = root_ca
.verify_signature(Some(client_cert.public_key()))
@ -97,31 +89,33 @@ pub fn verify_client_certificate(client_cert: String, root_ca: String) -> Result
}
#[napi]
pub fn sign_nonce(private_key: String, nonce: String) -> Result<String, Error> {
pub fn sign_nonce(private_key: String, nonce: String) -> anyhow::Result<String> {
let rng = SystemRandom::new();
let key_pair = KeyPair::from_pem(&private_key).unwrap();
let key_pair = KeyPair::from_pem(&private_key)?;
let key_pair = EcdsaKeyPair::from_pkcs8(
&ring::signature::ECDSA_P384_SHA384_FIXED_SIGNING,
&key_pair.serialize_der(),
&rng,
)
.unwrap();
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
let signature = key_pair.sign(&rng, nonce.as_bytes()).unwrap();
let signature = key_pair
.sign(&rng, nonce.as_bytes())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
let hex_signature = hex::encode(signature);
Ok(hex_signature)
}
#[napi]
pub fn verify_nonce(public_cert: String, nonce: String, signature: String) -> Result<bool, Error> {
let (_, pem) = x509_parser::pem::parse_x509_pem(public_cert.as_bytes()).unwrap();
let (_, spki) = parse_x509_certificate(&pem.contents).unwrap();
let public_key = SubjectPublicKeyInfo::from_der(spki.public_key().raw).unwrap();
pub fn verify_nonce(public_cert: String, nonce: String, signature: String) -> anyhow::Result<bool> {
let (_, pem) = x509_parser::pem::parse_x509_pem(public_cert.as_bytes())?;
let (_, spki) = parse_x509_certificate(&pem.contents)?;
let public_key = SubjectPublicKeyInfo::from_der(spki.public_key().raw)?;
let raw_signature = hex::decode(signature).unwrap();
let raw_signature = hex::decode(signature)?;
let valid = ring::signature::ECDSA_P384_SHA384_FIXED
.verify(

View File

@ -1,31 +1,32 @@
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::{
cell::LazyCell,
fs::{self, metadata, File},
io::{self, Read, Sink},
io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink},
path::{Path, PathBuf},
sync::Arc,
process::{Child, ChildStdout, Command, Stdio},
sync::{Arc, LazyLock},
};
use flate2::read::DeflateDecoder;
use rawzip::{
FileReader, ZipArchive, ZipArchiveEntryWayfinder, ZipEntry, ZipReader, RECOMMENDED_BUFFER_SIZE,
};
use anyhow::anyhow;
use crate::version::types::{MinimumFileObject, Skippable, VersionBackend, VersionFile};
use crate::version::types::{MinimumFileObject, VersionBackend, VersionFile};
pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) {
if metadata(path).unwrap().is_dir() {
let paths = fs::read_dir(path).unwrap();
pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) -> napi::Result<()> {
if metadata(path)?.is_dir() {
let paths = fs::read_dir(path)?;
for path_result in paths {
let full_path = path_result.unwrap().path();
if metadata(&full_path).unwrap().is_dir() {
_list_files(vec, &full_path);
let full_path = path_result?.path();
if metadata(&full_path)?.is_dir() {
_list_files(vec, &full_path)?;
} else {
vec.push(full_path);
}
}
}
};
Ok(())
}
#[derive(Clone)]
@ -33,39 +34,55 @@ pub struct PathVersionBackend {
pub base_dir: PathBuf,
}
impl VersionBackend for PathVersionBackend {
fn list_files(&mut self) -> Vec<VersionFile> {
fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
let mut vec = Vec::new();
_list_files(&mut vec, &self.base_dir);
_list_files(&mut vec, &self.base_dir)?;
let mut results = Vec::new();
for pathbuf in vec.iter() {
let relative = pathbuf.strip_prefix(self.base_dir.clone()).unwrap();
let relative = pathbuf.strip_prefix(self.base_dir.clone())?;
results.push(
self
.peek_file(relative.to_str().unwrap().to_owned())
.unwrap(),
self.peek_file(
relative
.to_str()
.ok_or(napi::Error::from_reason("Could not parse path"))?
.to_owned(),
)?,
);
}
results
Ok(results)
}
fn reader(&mut self, file: &VersionFile) -> Option<Box<dyn MinimumFileObject + 'static>> {
let file = File::open(self.base_dir.join(file.relative_filename.clone())).ok()?;
fn reader(
&mut self,
file: &VersionFile,
start: u64,
end: u64,
) -> anyhow::Result<Box<dyn MinimumFileObject + 'static>> {
let mut file = File::open(self.base_dir.join(file.relative_filename.clone()))?;
return Some(Box::new(file));
if start != 0 {
file.seek(SeekFrom::Start(start))?;
}
if end != 0 {
return Ok(Box::new(file.take(end - start)));
}
Ok(Box::new(file))
}
fn peek_file(&mut self, sub_path: String) -> Option<VersionFile> {
fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
let pathbuf = self.base_dir.join(sub_path.clone());
if !pathbuf.exists() {
return None;
return Err(anyhow!("Path doesn't exist."));
};
let file = File::open(pathbuf.clone()).unwrap();
let metadata = file.try_clone().unwrap().metadata().unwrap();
let file = File::open(pathbuf.clone())?;
let metadata = file.try_clone()?.metadata()?;
let permission_object = metadata.permissions();
let permissions = {
let perm: u32;
@ -80,105 +97,120 @@ impl VersionBackend for PathVersionBackend {
perm
};
Some(VersionFile {
Ok(VersionFile {
relative_filename: sub_path,
permission: permissions,
size: metadata.len(),
})
}
fn require_whole_files(&self) -> bool {
false
}
}
pub static SEVEN_ZIP_INSTALLED: LazyLock<bool> =
LazyLock::new(|| Command::new("7z").output().is_ok());
#[derive(Clone)]
pub struct ZipVersionBackend {
archive: Arc<ZipArchive<FileReader>>,
path: String,
}
impl ZipVersionBackend {
pub fn new(archive: File) -> Self {
let archive = ZipArchive::from_file(archive, &mut [0u8; RECOMMENDED_BUFFER_SIZE]).unwrap();
Self {
archive: Arc::new(archive),
}
}
pub fn new_entry<'archive>(
&self,
entry: ZipEntry<'archive, FileReader>,
) -> ZipFileWrapper<'archive> {
let deflater = DeflateDecoder::new(entry.reader());
ZipFileWrapper { reader: deflater }
}
}
pub struct ZipFileWrapper<'archive> {
reader: DeflateDecoder<ZipReader<'archive, FileReader>>,
}
impl<'a> Read for ZipFileWrapper<'a> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let read = self.reader.read(buf)?;
Ok(read)
}
}
impl<'a> Skippable for ZipFileWrapper<'a> {
fn skip(&mut self, amount: u64) {
io::copy(&mut self.take(amount), &mut Sink::default()).unwrap();
}
}
impl<'a> MinimumFileObject for ZipFileWrapper<'a> {}
impl ZipVersionBackend {
fn find_wayfinder(&mut self, filename: &str) -> Option<ZipArchiveEntryWayfinder> {
let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
let mut entries = self.archive.entries(read_buffer);
let entry = loop {
if let Some(v) = entries.next_entry().unwrap() {
if v.file_path().try_normalize().unwrap().as_ref() == filename {
break Some(v);
}
} else {
break None;
}
}?;
let wayfinder = entry.wayfinder();
Some(wayfinder)
}
}
impl VersionBackend for ZipVersionBackend {
fn list_files(&mut self) -> Vec<VersionFile> {
let mut results = Vec::new();
let read_buffer = &mut [0u8; RECOMMENDED_BUFFER_SIZE];
let mut budget_iterator = self.archive.entries(read_buffer);
while let Some(entry) = budget_iterator.next_entry().unwrap() {
if entry.is_dir() {
continue;
}
results.push(VersionFile {
relative_filename: String::from(entry.file_path().try_normalize().unwrap()),
permission: entry.mode().permissions(),
size: entry.uncompressed_size_hint(),
});
}
results
}
fn reader(&mut self, file: &VersionFile) -> Option<Box<dyn MinimumFileObject + '_>> {
let wayfinder = self.find_wayfinder(&file.relative_filename)?;
let local_entry = self.archive.get_entry(wayfinder).unwrap();
let wrapper = self.new_entry(local_entry);
Some(Box::new(wrapper))
}
fn peek_file(&mut self, sub_path: String) -> Option<VersionFile> {
let entry = self.find_wayfinder(&sub_path)?;
Some(VersionFile {
relative_filename: sub_path,
permission: 0,
size: entry.uncompressed_size_hint(),
pub fn new(path: PathBuf) -> anyhow::Result<Self> {
Ok(Self {
path: path.to_str().expect("invalid utf path").to_owned(),
})
}
}
pub struct ZipFileWrapper {
command: Child,
reader: BufReader<ChildStdout>
}
impl ZipFileWrapper {
pub fn new(mut command: Child) -> Self {
let stdout = command.stdout.take().expect("failed to access stdout of 7z");
let reader = BufReader::new(stdout);
ZipFileWrapper { command, reader }
}
}
/**
* This read implemention is a result of debugging hell
* It should probably be replaced with a .take() call.
*/
impl Read for ZipFileWrapper {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.reader.read(buf)
}
}
impl Drop for ZipFileWrapper {
fn drop(&mut self) {
self.command.wait().expect("failed to wait for 7z exit");
}
}
impl VersionBackend for ZipVersionBackend {
fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
let mut list_command = Command::new("7z");
list_command.args(vec!["l", "-ba", &self.path]);
let result = list_command.output()?;
if !result.status.success() {
return Err(anyhow!(
"failed to list files: code {:?}",
result.status.code()
));
}
let raw_result = String::from_utf8(result.stdout)?;
let files = raw_result.split("\n").filter(|v| v.len() > 0).map(|v| v.split(" ").filter(|v| v.len() > 0));
let mut results = Vec::new();
for mut file in files {
let (date, time, attrs, size, compress, name) = (
file.next().unwrap(),
file.next().unwrap(),
file.next().unwrap(),
file.next().unwrap(),
file.next().unwrap(),
file.next().unwrap(),
);
println!("got line: {} {} {} {} {} {}", date, time, attrs, size, compress, name);
results.push(VersionFile {
relative_filename: name.to_owned(),
permission: 0,
size: size.parse().unwrap(),
});
}
Ok(results)
}
fn reader(
&mut self,
file: &VersionFile,
start: u64,
end: u64,
) -> anyhow::Result<Box<dyn MinimumFileObject + '_>> {
let mut read_command = Command::new("7z");
read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]);
let output = read_command.stdout(Stdio::piped()).spawn().expect("failed to spawn 7z");
Ok(Box::new(ZipFileWrapper::new(output)))
}
fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
let files = self.list_files()?;
let file = files
.iter()
.find(|v| v.relative_filename == sub_path)
.expect("file not found");
Ok(file.clone())
}
fn require_whole_files(&self) -> bool {
true
}
}

View File

@ -1,7 +1,4 @@
use std::{
fmt::Debug,
io::{Read, Seek, SeekFrom},
};
use std::{fmt::Debug, io::Read};
use dyn_clone::DynClone;
use tokio::io::{self, AsyncRead};
@ -13,44 +10,46 @@ pub struct VersionFile {
pub size: u64,
}
pub trait Skippable {
fn skip(&mut self, amount: u64);
}
impl<T> Skippable for T
where
T: Seek,
{
fn skip(&mut self, amount: u64) {
self.seek(SeekFrom::Start(amount)).unwrap();
}
}
pub trait MinimumFileObject: Read + Send + Skippable {}
impl<T: Read + Send + Seek> MinimumFileObject for T {}
pub trait MinimumFileObject: Read + Send {}
impl<T: Read + Send> MinimumFileObject for T {}
// Intentionally not a generic, because of types in read_file
pub struct ReadToAsyncRead<'a> {
pub inner: Box<dyn Read + Send + 'a>,
}
const ASYNC_READ_BUFFER_SIZE: usize = 8128;
impl<'a> AsyncRead for ReadToAsyncRead<'a> {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<io::Result<()>> {
let mut read_buf = [0u8; 8192];
let var_name = self.inner.read(&mut read_buf).unwrap();
let amount = var_name.min(buf.remaining());
buf.put_slice(&read_buf[0..amount]);
std::task::Poll::Ready(Ok(()))
let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE];
let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining());
match self.inner.read(&mut read_buf[0..read_size]) {
Ok(read) => {
buf.put_slice(&read_buf[0..read]);
std::task::Poll::Ready(Ok(()))
}
Err(err) => {
std::task::Poll::Ready(Err(err))
},
}
}
}
pub trait VersionBackend: DynClone {
fn list_files(&mut self) -> Vec<VersionFile>;
fn peek_file(&mut self, sub_path: String) -> Option<VersionFile>;
fn reader(&mut self, file: &VersionFile) -> Option<Box<dyn MinimumFileObject + '_>>;
fn require_whole_files(&self) -> bool;
fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>>;
fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile>;
fn reader(
&mut self,
file: &VersionFile,
start: u64,
end: u64,
) -> anyhow::Result<Box<dyn MinimumFileObject + '_>>;
}
dyn_clone::clone_trait_object!(VersionBackend);
dyn_clone::clone_trait_object!(VersionBackend);

View File

@ -1,10 +1,16 @@
use std::{collections::HashMap, fs::File, io::Read, path::Path};
use std::{
collections::HashMap,
fs::File,
path::Path,
process::{Command, ExitStatus},
};
use anyhow::anyhow;
use napi::{bindgen_prelude::*, sys::napi_value__, tokio_stream::StreamExt};
use tokio_util::codec::{BytesCodec, FramedRead};
use crate::version::{
backends::{PathVersionBackend, ZipVersionBackend},
backends::{PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED},
types::{ReadToAsyncRead, VersionBackend, VersionFile},
};
@ -13,7 +19,7 @@ use crate::version::{
*/
pub fn create_backend_constructor<'a>(
path: &Path,
) -> Option<Box<dyn FnOnce() -> Box<dyn VersionBackend + Send + 'a>>> {
) -> Option<Box<dyn FnOnce() -> Result<Box<dyn VersionBackend + Send + 'a>>>> {
if !path.exists() {
return None;
}
@ -21,12 +27,21 @@ pub fn create_backend_constructor<'a>(
let is_directory = path.is_dir();
if is_directory {
let base_dir = path.to_path_buf();
return Some(Box::new(move || Box::new(PathVersionBackend { base_dir })));
return Some(Box::new(move || {
Ok(Box::new(PathVersionBackend { base_dir }))
}));
};
if path.to_string_lossy().ends_with(".zip") {
let f = File::open(path.to_path_buf()).unwrap();
return Some(Box::new(|| Box::new(ZipVersionBackend::new(f))));
if *SEVEN_ZIP_INSTALLED {
let mut test = Command::new("7z");
test.args(vec!["t", path.to_str().expect("invalid utf path")]);
let status = test.status().ok()?;
if status.code().unwrap_or(1) == 0 {
let buf = path.to_path_buf();
return Some(Box::new(move || {
Ok(Box::new(ZipVersionBackend::new(buf)?))
}));
}
}
None
@ -56,10 +71,13 @@ impl<'a> DropletHandler<'a> {
let fs_path = Path::new(&path);
let constructor = create_backend_constructor(fs_path)?;
let existing_backend = self.backend_cache.entry(path).or_insert_with(|| {
let backend = constructor();
backend
});
let existing_backend = match self.backend_cache.entry(path) {
std::collections::hash_map::Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
std::collections::hash_map::Entry::Vacant(vacant_entry) => {
let backend = constructor().ok()?;
vacant_entry.insert(backend)
}
};
Some(existing_backend)
}
@ -78,7 +96,7 @@ impl<'a> DropletHandler<'a> {
let backend = self
.create_backend_for_path(path)
.ok_or(napi::Error::from_reason("No backend for path"))?;
let files = backend.list_files();
let files = backend.list_files()?;
Ok(files.into_iter().map(|e| e.relative_filename).collect())
}
@ -88,11 +106,9 @@ impl<'a> DropletHandler<'a> {
.create_backend_for_path(path)
.ok_or(napi::Error::from_reason("No backend for path"))?;
let file = backend
.peek_file(sub_path)
.ok_or(napi::Error::from_reason("Can't find file to peek"))?;
let file = backend.peek_file(sub_path)?;
return Ok(file.size.try_into().unwrap());
Ok(file.size)
}
#[napi]
@ -104,32 +120,24 @@ impl<'a> DropletHandler<'a> {
env: Env,
start: Option<BigInt>,
end: Option<BigInt>,
) -> Result<JsDropStreamable> {
) -> anyhow::Result<JsDropStreamable> {
let stream = reference.share_with(env, |handler| {
let backend = handler
.create_backend_for_path(path)
.ok_or(napi::Error::from_reason("Failed to create backend."))?;
.ok_or(anyhow!("Failed to create backend."))?;
let version_file = VersionFile {
relative_filename: sub_path,
permission: 0, // Shouldn't matter
size: 0, // Shouldn't matter
};
// Use `?` operator for cleaner error propagation from `Option`
let mut reader = backend.reader(&version_file).ok_or(napi::Error::from_reason("Failed to create reader."))?;
let reader = backend.reader(
&version_file,
start.map(|e| e.get_u64().1).unwrap_or(0),
end.map(|e| e.get_u64().1).unwrap_or(0),
)?;
if let Some(skip) = start.clone() {
reader.skip(skip.get_u64().1.into());
// io::copy(&mut reader.by_ref().take(skip.into()), &mut io::sink()).unwrap();
}
let async_reader = if let Some(limit) = end {
let amount = limit.get_u64().1 - start.map_or(Some(0), |v| Some(v.get_u64().1)).unwrap();
ReadToAsyncRead {
inner: Box::new(reader.take(amount.into())),
}
} else {
ReadToAsyncRead { inner: reader }
};
let async_reader = ReadToAsyncRead { inner: reader };
// Create a FramedRead stream with BytesCodec for chunking
let stream = FramedRead::new(async_reader, BytesCodec::new())
@ -139,17 +147,15 @@ impl<'a> DropletHandler<'a> {
// Apply Result::map to transform Ok(BytesMut) to Ok(Vec<u8>)
.map(|bytes| bytes.to_vec())
// Apply Result::map_err to transform Err(std::io::Error) to Err(napi::Error)
.map_err(|e| napi::Error::from(e)) // napi::Error implements From<tokio::io::Error>
.map_err(napi::Error::from) // napi::Error implements From<tokio::io::Error>
});
// Create the napi-rs ReadableStream from the tokio_stream::Stream
// The unwrap() here means if stream creation fails, it will panic.
// For a production system, consider returning Result<Option<...>> and handling this.
Ok(ReadableStream::create_with_stream_bytes(&env, stream).unwrap())
ReadableStream::create_with_stream_bytes(&env, stream)
})?;
Ok(JsDropStreamable {
inner: stream,
})
Ok(JsDropStreamable { inner: stream })
}
}
@ -164,4 +170,4 @@ impl JsDropStreamable {
pub fn get_stream(&self) -> *mut napi_value__ {
self.inner.raw()
}
}
}

View File

@ -12,6 +12,7 @@ __metadata:
"@napi-rs/cli": "npm:3.0.0-alpha.91"
"@types/node": "npm:^22.13.10"
ava: "npm:^6.2.0"
pretty-bytes: "npm:^7.0.1"
tsimp: "npm:^2.0.12"
languageName: unknown
linkType: soft
@ -2432,6 +2433,13 @@ __metadata:
languageName: node
linkType: hard
"pretty-bytes@npm:^7.0.1":
version: 7.0.1
resolution: "pretty-bytes@npm:7.0.1"
checksum: 10c0/14ffb503d2de3588042c722848062a4897e6faece1694e0c83ba5669ec003d73311d946d50d2b3c6099a6a306760011b8446ee3cf9cf86eca13a454a8f1c47cb
languageName: node
linkType: hard
"pretty-ms@npm:^9.1.0":
version: 9.2.0
resolution: "pretty-ms@npm:9.2.0"