mirror of
https://github.com/Drop-OSS/droplet.git
synced 2025-11-24 13:41:16 +10:00
new chunking and cert utils
This commit is contained in:
DecDuck
109
src/chunker.rs
Normal file
109
src/chunker.rs
Normal file
@ -0,0 +1,109 @@
|
||||
use std::{collections::HashMap, fs::File, io::{self, BufRead, BufReader}, path::Path, sync::{Arc, Mutex}};
|
||||
use napi::Error;
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{file_utils::list_files, manifest::{generate_manifest, Manifest, ManifestChunk, ManifestRecord}};
|
||||
|
||||
const CHUNK_SIZE: usize = 1024 * 1024 * 64;
|
||||
|
||||
|
||||
fn compress(mut buffer: &[u8], output_path: &Path, chunk_id: Uuid) {
|
||||
let chunk_path = output_path.join(chunk_id.to_string() + ".bin");
|
||||
let mut chunk_file = File::create_new(chunk_path).unwrap();
|
||||
|
||||
io::copy(&mut buffer, &mut chunk_file).unwrap();
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn chunk(source: String, output: String) -> Result<(), Error> {
|
||||
let source_path = Path::new(&source);
|
||||
let output_path = Path::new(&output);
|
||||
|
||||
let files = list_files(source_path);
|
||||
|
||||
let num_of_threads: u64 = 8;
|
||||
|
||||
let pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(num_of_threads.try_into().unwrap())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let queue_size = Arc::new(Mutex::new(0));
|
||||
|
||||
pool.scope(|scope| {
|
||||
let mut manifest = Manifest {
|
||||
record: HashMap::new(),
|
||||
};
|
||||
|
||||
for file_path in files {
|
||||
let file = File::open(file_path.clone()).unwrap();
|
||||
let permissions = file.try_clone().unwrap().metadata().unwrap().permissions();
|
||||
let mut reader = BufReader::with_capacity(CHUNK_SIZE, file);
|
||||
let relative = file_path.strip_prefix(source_path).unwrap();
|
||||
|
||||
let mut record = ManifestRecord {
|
||||
chunks: Vec::new(),
|
||||
permissions: 0,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
{
|
||||
record.permissions = permissions.mode();
|
||||
}
|
||||
|
||||
let mut chunk_index = 0;
|
||||
loop {
|
||||
let mut buffer: Vec<u8> = Vec::new();
|
||||
reader.fill_buf().unwrap().clone_into(&mut buffer);
|
||||
let length = buffer.len();
|
||||
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
{
|
||||
*queue_size.lock().unwrap() += 1;
|
||||
}
|
||||
|
||||
let chunk_id: Uuid = Uuid::new_v4();
|
||||
|
||||
let queue_size_handle = queue_size.clone();
|
||||
scope.spawn(move |_scope| {
|
||||
compress(&buffer, output_path, chunk_id);
|
||||
let mut num = queue_size_handle.lock().unwrap();
|
||||
*num -= 1;
|
||||
});
|
||||
|
||||
reader.consume(length);
|
||||
|
||||
let chunk_record = ManifestChunk {
|
||||
uuid: chunk_id.to_string(),
|
||||
index: chunk_index,
|
||||
};
|
||||
record.chunks.push(chunk_record);
|
||||
chunk_index += 1;
|
||||
|
||||
loop {
|
||||
let num = queue_size.lock().unwrap();
|
||||
if *num < num_of_threads {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
manifest
|
||||
.record
|
||||
.insert(relative.to_str().unwrap().to_string(), record);
|
||||
|
||||
println!("Queued {}", file_path.to_str().unwrap());
|
||||
}
|
||||
|
||||
let manifest_path = output_path.join("manifest.drop");
|
||||
generate_manifest(manifest, &manifest_path);
|
||||
});
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
115
src/lib.rs
115
src/lib.rs
@ -1,121 +1,10 @@
|
||||
#![deny(clippy::all)]
|
||||
|
||||
const CHUNK_SIZE: usize = 1024 * 1024 * 16;
|
||||
|
||||
use file_utils::list_files;
|
||||
use manifest::{generate_manifest, Manifest, ManifestChunk, ManifestRecord};
|
||||
use napi::Error;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::File,
|
||||
io::{self, BufRead, BufReader, Write},
|
||||
path::Path,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
pub mod file_utils;
|
||||
pub mod manifest;
|
||||
pub mod chunker;
|
||||
pub mod ssl;
|
||||
|
||||
#[macro_use]
|
||||
extern crate napi_derive;
|
||||
|
||||
fn compress(buffer: &[u8], output_path: &Path, chunk_id: Uuid) {
|
||||
let chunk_path = output_path.join(chunk_id.to_string() + ".bin");
|
||||
let chunk_file = File::create_new(chunk_path).unwrap();
|
||||
|
||||
zstd::stream::copy_encode(buffer, chunk_file, 7).unwrap();
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn repack(source: String, output: String) -> Result<(), Error> {
|
||||
let source_path = Path::new(&source);
|
||||
let output_path = Path::new(&output);
|
||||
|
||||
let files = list_files(source_path);
|
||||
|
||||
let num_of_threads: u64 = 8;
|
||||
|
||||
let pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(num_of_threads.try_into().unwrap())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let queue_size = Arc::new(Mutex::new(0));
|
||||
|
||||
pool.scope(|scope| {
|
||||
let mut manifest = Manifest {
|
||||
record: HashMap::new(),
|
||||
};
|
||||
|
||||
for file_path in files {
|
||||
let file = File::open(file_path.clone()).unwrap();
|
||||
let permissions = file.try_clone().unwrap().metadata().unwrap().permissions();
|
||||
let mut reader = BufReader::with_capacity(CHUNK_SIZE, file);
|
||||
let relative = file_path.strip_prefix(source_path).unwrap();
|
||||
|
||||
let mut record = ManifestRecord {
|
||||
chunks: Vec::new(),
|
||||
permissions: 0,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
{
|
||||
record.permissions = permissions.mode();
|
||||
}
|
||||
|
||||
let mut chunk_index = 0;
|
||||
loop {
|
||||
let mut buffer: Vec<u8> = Vec::new();
|
||||
reader.fill_buf().unwrap().clone_into(&mut buffer);
|
||||
let length = buffer.len();
|
||||
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
{
|
||||
*queue_size.lock().unwrap() += 1;
|
||||
}
|
||||
|
||||
let chunk_id: Uuid = Uuid::new_v4();
|
||||
|
||||
let queue_size_handle = queue_size.clone();
|
||||
scope.spawn(move |_scope| {
|
||||
compress(&buffer, output_path, chunk_id);
|
||||
let mut num = queue_size_handle.lock().unwrap();
|
||||
*num -= 1;
|
||||
});
|
||||
|
||||
reader.consume(length);
|
||||
|
||||
let chunk_record = ManifestChunk {
|
||||
uuid: chunk_id.to_string(),
|
||||
index: chunk_index,
|
||||
};
|
||||
record.chunks.push(chunk_record);
|
||||
chunk_index += 1;
|
||||
|
||||
loop {
|
||||
let num = queue_size.lock().unwrap();
|
||||
if *num < num_of_threads {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
manifest
|
||||
.record
|
||||
.insert(relative.to_str().unwrap().to_string(), record);
|
||||
|
||||
println!("Queued {}", file_path.to_str().unwrap());
|
||||
}
|
||||
|
||||
let manifest_path = output_path.join("manifest.drop");
|
||||
generate_manifest(manifest, &manifest_path);
|
||||
});
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
84
src/ssl.rs
Normal file
84
src/ssl.rs
Normal file
@ -0,0 +1,84 @@
|
||||
use napi::Error;
|
||||
use rcgen::{
|
||||
date_time_ymd, generate_simple_self_signed, BasicConstraints, Certificate, CertificateParams,
|
||||
DistinguishedName, DnType, Ia5String, IsCa, KeyIdMethod, KeyPair, KeyUsagePurpose, SanType,
|
||||
PKCS_ECDSA_P384_SHA384,
|
||||
};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use x509_verify::der::DecodePem;
|
||||
|
||||
const YEAR: i64 = 60 * 60 * 24 * 365;
|
||||
|
||||
#[napi]
|
||||
pub fn generate_root_ca() -> Result<Vec<String>, Error> {
|
||||
let mut params = CertificateParams::new(Vec::new()).unwrap();
|
||||
|
||||
let mut name = DistinguishedName::new();
|
||||
name.push(DnType::CommonName, "Drop Root CA");
|
||||
name.push(DnType::OrganizationName, "Drop");
|
||||
|
||||
params.distinguished_name = name;
|
||||
params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
params.key_usages = vec![
|
||||
KeyUsagePurpose::DigitalSignature,
|
||||
KeyUsagePurpose::KeyCertSign,
|
||||
];
|
||||
|
||||
let key_pair = KeyPair::generate_for(&PKCS_ECDSA_P384_SHA384).unwrap();
|
||||
|
||||
let root_ca = CertificateParams::self_signed(params, &key_pair).unwrap();
|
||||
|
||||
return Ok(vec![
|
||||
key_pair.serialize_pem(),
|
||||
key_pair.public_key_pem(),
|
||||
root_ca.pem(),
|
||||
]);
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn generate_client_certificate(
|
||||
client_id: String,
|
||||
client_name: String,
|
||||
root_ca: String,
|
||||
root_ca_private: String,
|
||||
) -> Result<Vec<String>, Error> {
|
||||
let root_key_pair = KeyPair::from_pem(&root_ca_private).unwrap();
|
||||
let root_ca_params = CertificateParams::from_ca_cert_pem(&root_ca).unwrap();
|
||||
let root_ca = root_ca_params.self_signed(&root_key_pair).unwrap();
|
||||
|
||||
let mut params = CertificateParams::new(Vec::new()).unwrap();
|
||||
|
||||
let mut name = DistinguishedName::new();
|
||||
name.push(DnType::CommonName, client_id);
|
||||
name.push(DnType::OrganizationName, "Drop");
|
||||
|
||||
params.distinguished_name = name;
|
||||
params.subject_alt_names = vec![SanType::DnsName(Ia5String::try_from(client_name).unwrap())];
|
||||
params.key_usages = vec![
|
||||
KeyUsagePurpose::DigitalSignature,
|
||||
KeyUsagePurpose::KeyCertSign,
|
||||
];
|
||||
|
||||
let client_key_pair = KeyPair::generate_for(&PKCS_ECDSA_P384_SHA384).unwrap();
|
||||
let client_certificate = params
|
||||
.signed_by(&client_key_pair, &root_ca, &root_key_pair)
|
||||
.unwrap();
|
||||
|
||||
return Ok(vec![
|
||||
root_key_pair.serialize_pem(),
|
||||
root_key_pair.public_key_pem(),
|
||||
client_certificate.pem(),
|
||||
]);
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn verify_client_certificate(client_cert: String, root_ca: String) -> Result<bool, Error> {
|
||||
let ca = x509_verify::x509_cert::Certificate::from_pem(root_ca).unwrap();
|
||||
let client = x509_verify::x509_cert::Certificate::from_pem(client_cert).unwrap();
|
||||
|
||||
let key = x509_verify::VerifyingKey::try_from(&ca).unwrap();
|
||||
return Ok(match key.verify(&client) {
|
||||
Ok(_) => true,
|
||||
Err(_) => false,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user