mirror of
https://github.com/Drop-OSS/drop.git
synced 2026-06-22 04:11:32 +10:00
feat: libarchive backend
This commit is contained in:
DecDuck
@@ -0,0 +1,3 @@
|
||||
[submodule "libarchive-rust"]
|
||||
path = libarchive-rust
|
||||
url = git@github.com:Drop-OSS/libarchive-rust.git
|
||||
Generated
+166
-20
@@ -2,6 +2,21 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.25.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.100"
|
||||
@@ -48,8 +63,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
"syn 2.0.117",
|
||||
"synstructure 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -60,8 +75,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
"syn 2.0.117",
|
||||
"synstructure 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -72,7 +87,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -83,7 +98,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -92,6 +107,21 @@ version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
@@ -214,7 +244,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -228,11 +258,13 @@ dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"hex",
|
||||
"humansize",
|
||||
"libarchive",
|
||||
"rcgen",
|
||||
"ring",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"speedometer",
|
||||
"time",
|
||||
"tokio",
|
||||
"uuid",
|
||||
@@ -245,6 +277,28 @@ version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "failure"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"failure_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure_derive"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
"synstructure 0.12.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.31"
|
||||
@@ -301,7 +355,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -367,6 +421,12 @@ dependencies = [
|
||||
"wasip2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.32.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
@@ -405,10 +465,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.171"
|
||||
name = "libarchive"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"libarchive3-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libarchive3-sys"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
|
||||
checksum = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.182"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
@@ -428,6 +506,15 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.1.0"
|
||||
@@ -483,6 +570,15 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.37.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oid-registry"
|
||||
version = "0.7.1"
|
||||
@@ -529,6 +625,12 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
@@ -587,6 +689,12 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
|
||||
|
||||
[[package]]
|
||||
name = "rusticata-macros"
|
||||
version = "4.1.0"
|
||||
@@ -641,7 +749,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -690,16 +798,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.100"
|
||||
name = "speedometer"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
|
||||
checksum = "2789736092fa21b44baf8590acb4b360cb91f0f597bd6c1f1741ca9644c95c1e"
|
||||
dependencies = [
|
||||
"failure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.12.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.1"
|
||||
@@ -708,7 +848,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -737,7 +877,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -748,7 +888,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -805,7 +945,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -820,6 +960,12 @@ version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
@@ -890,7 +1036,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.117",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
|
||||
@@ -11,7 +11,15 @@ hex = "0.4.3"
|
||||
time = "0.3.41"
|
||||
ring = "0.17.14"
|
||||
dyn-clone = "1.0.20"
|
||||
tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "rt", "rt-multi-thread", "macros", "sync"] }
|
||||
tokio = { version = "^1.48.0", features = [
|
||||
"process",
|
||||
"fs",
|
||||
"io-util",
|
||||
"rt",
|
||||
"rt-multi-thread",
|
||||
"macros",
|
||||
"sync",
|
||||
] }
|
||||
anyhow = "1.0.100"
|
||||
async-trait = "0.1.89"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
@@ -21,6 +29,8 @@ uuid = { version = "1.19.0", features = ["v4"] }
|
||||
sha2 = "0.10.9"
|
||||
futures = "0.3.31"
|
||||
getrandom = "0.3.4"
|
||||
libarchive = { version = "*", path = "./libarchive-rust" }
|
||||
speedometer = "0.2.2"
|
||||
|
||||
[dependencies.x509-parser]
|
||||
version = "0.17.0"
|
||||
|
||||
@@ -1,2 +1,8 @@
|
||||
# droplet-rs
|
||||
Core rust functionality for droplet
|
||||
|
||||
A Rust-based library for utilities and functionality required both by the server, client, and other tools.
|
||||
|
||||
## manifest generation
|
||||
`droplet-rs` contains the manifest generation code, held in `manifest.rs`.
|
||||
|
||||
It also includes the version backends to provide a unified read-write interface to files.
|
||||
Submodule
+1
Submodule libraries/droplet/libarchive-rust added at 279fd5f727
@@ -5,6 +5,9 @@ pub mod file_utils;
|
||||
pub mod ssl;
|
||||
pub mod versions;
|
||||
pub mod manifest;
|
||||
pub mod vm;
|
||||
|
||||
extern crate libarchive;
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod tests;
|
||||
|
||||
@@ -22,6 +22,8 @@ pub async fn main() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
return;
|
||||
|
||||
// Sanity checks
|
||||
for (_, chunk_data) in manifest.chunks {
|
||||
for file in chunk_data.files {
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
use std::{
|
||||
collections::HashMap, future::Future, path::Path, sync::{
|
||||
Arc, atomic::{AtomicU64, Ordering}
|
||||
}
|
||||
collections::HashMap,
|
||||
future::Future,
|
||||
mem,
|
||||
path::Path,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Error};
|
||||
use futures::{stream::FuturesUnordered, StreamExt};
|
||||
use anyhow::anyhow;
|
||||
use hex::ToHex as _;
|
||||
use humansize::{format_size, BINARY};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -14,6 +18,7 @@ use tokio::{
|
||||
io::AsyncReadExt as _,
|
||||
join,
|
||||
sync::{Mutex, Semaphore},
|
||||
task::JoinSet,
|
||||
};
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
@@ -39,18 +44,21 @@ pub struct Manifest {
|
||||
pub key: [u8; 16],
|
||||
}
|
||||
|
||||
const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
|
||||
const CHUNK_SIZE: u64 = 1024 * 1024; //* 64;
|
||||
const MAX_FILE_COUNT: usize = 512;
|
||||
|
||||
use crate::versions::{create_backend_constructor, types::VersionFile};
|
||||
use crate::versions::{
|
||||
create_backend_constructor,
|
||||
types::{VersionBackend, VersionFile},
|
||||
};
|
||||
|
||||
pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
|
||||
dir: &Path,
|
||||
progress_sfn: V,
|
||||
log_sfn: T,
|
||||
reader_semaphore: Option<&Semaphore>,
|
||||
reader_semaphore: Option<Arc<Semaphore>>,
|
||||
) -> anyhow::Result<Manifest> {
|
||||
let mut backend =
|
||||
let backend =
|
||||
create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?;
|
||||
|
||||
let required_single_file = backend.require_whole_files();
|
||||
@@ -137,18 +145,19 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
|
||||
let manifest: Arc<Mutex<HashMap<String, ChunkData>>> = Arc::new(Mutex::new(HashMap::new()));
|
||||
let total_manifest_length = Arc::new(AtomicU64::new(0));
|
||||
|
||||
let backend = Arc::new(Mutex::new(backend));
|
||||
// SAFETY: we .join_all() the futures using this
|
||||
let backend: &'static Box<dyn VersionBackend + Send + Sync> =
|
||||
unsafe { mem::transmute(&backend) };
|
||||
|
||||
let futures: FuturesUnordered<impl Future<Output = Result<(), Error>>> =
|
||||
FuturesUnordered::new();
|
||||
let mut futures: JoinSet<Result<(), anyhow::Error>> = JoinSet::new();
|
||||
let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16);
|
||||
let chunks_length = chunks.len();
|
||||
for (index, chunk) in chunks.into_iter().enumerate() {
|
||||
let send_log = send_log.clone();
|
||||
let backend = backend.clone();
|
||||
let total_manifest_length = total_manifest_length.clone();
|
||||
let manifest = manifest.clone();
|
||||
futures.push(async move {
|
||||
let reader_semaphore = reader_semaphore.clone();
|
||||
futures.spawn(async move {
|
||||
let mut read_buf = vec![0; 1024 * 1024 * 64];
|
||||
|
||||
let uuid = uuid::Uuid::new_v4().to_string();
|
||||
@@ -164,6 +173,8 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
|
||||
|
||||
let mut chunk_length = 0;
|
||||
|
||||
println!("starting chunk {}", index);
|
||||
|
||||
for (file, start, length) in chunk {
|
||||
let permit = if let Some(reader_semaphore) = &reader_semaphore {
|
||||
Some(reader_semaphore.acquire().await?)
|
||||
@@ -171,18 +182,20 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
|
||||
None
|
||||
};
|
||||
|
||||
let mut reader = {
|
||||
let mut backend_lock = backend.lock().await;
|
||||
let reader = backend_lock.reader(&file, start, start + length).await?;
|
||||
reader
|
||||
};
|
||||
let mut reader = backend.reader(&file, start, start + length).await?;
|
||||
|
||||
let mut total = 0;
|
||||
|
||||
loop {
|
||||
let amount = reader.read(&mut read_buf).await?;
|
||||
if amount == 0 {
|
||||
break;
|
||||
}
|
||||
total += amount;
|
||||
hasher.update(&read_buf[0..amount]);
|
||||
if total as u64 > length {
|
||||
panic!("read too much: target {}, got {}", length, total);
|
||||
}
|
||||
}
|
||||
|
||||
chunk_length += length;
|
||||
@@ -230,7 +243,7 @@ pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
|
||||
progress_sfn((current_progress / total_progress) * 100.0f32)
|
||||
}
|
||||
},
|
||||
futures.collect::<Vec<Result<(), Error>>>()
|
||||
futures.join_all()
|
||||
);
|
||||
|
||||
let manifest = manifest.lock().await;
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
use std::{path::PathBuf, task::Poll};
|
||||
|
||||
use anyhow::{anyhow};
|
||||
use async_trait::async_trait;
|
||||
use libarchive::{
|
||||
archive::{Entry, FileType, ReadCompression, ReadFormat},
|
||||
reader::{Builder, FileReader, Reader},
|
||||
};
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile};
|
||||
|
||||
pub struct ZipVersionBackend {
|
||||
path: PathBuf,
|
||||
}
|
||||
impl ZipVersionBackend {
|
||||
pub fn new(path: PathBuf) -> anyhow::Result<Self> {
|
||||
Ok(Self { path })
|
||||
}
|
||||
|
||||
fn open_archive(&self) -> Result<FileReader, anyhow::Error> {
|
||||
let mut archive = Builder::new();
|
||||
archive.support_format(ReadFormat::All)?;
|
||||
archive.support_compression(ReadCompression::All)?;
|
||||
let archive = archive.open_file(&self.path)?;
|
||||
|
||||
Ok(archive)
|
||||
}
|
||||
}
|
||||
|
||||
struct ArchiveReader {
|
||||
archive: FileReader,
|
||||
}
|
||||
|
||||
impl AsyncRead for ArchiveReader {
|
||||
fn poll_read(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
_cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
let block = match self.archive.read_block() {
|
||||
Ok(v) => v,
|
||||
Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))),
|
||||
};
|
||||
|
||||
let block = match block {
|
||||
Some(v) => v,
|
||||
None => return Poll::Ready(Ok(())),
|
||||
};
|
||||
|
||||
buf.put_slice(block);
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VersionBackend for ZipVersionBackend {
|
||||
async fn list_files(&self) -> anyhow::Result<Vec<VersionFile>> {
|
||||
let mut archive = self.open_archive()?;
|
||||
let mut results = Vec::new();
|
||||
|
||||
while let Some(header) = archive.next_header() {
|
||||
match header.filetype() {
|
||||
FileType::RegularFile => (),
|
||||
_ => {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
results.push(VersionFile {
|
||||
relative_filename: header.pathname().to_string(),
|
||||
permission: 0o744,
|
||||
size: header.size().try_into()?,
|
||||
});
|
||||
println!("storing file {}", header.pathname());
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn reader(
|
||||
&self,
|
||||
file: &VersionFile,
|
||||
_start: u64,
|
||||
_end: u64,
|
||||
) -> anyhow::Result<Box<dyn MinimumFileObject>> {
|
||||
let mut archive = self.open_archive()?;
|
||||
|
||||
// Find entry in archive
|
||||
loop {
|
||||
let entry = match archive.next_header() {
|
||||
Some(v) => v,
|
||||
None => return Err(anyhow!("entry not found:{}", file.relative_filename)),
|
||||
};
|
||||
if entry.pathname() == file.relative_filename {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Box::new(ArchiveReader { archive }))
|
||||
}
|
||||
|
||||
async fn peek_file(&self, sub_path: String) -> anyhow::Result<VersionFile> {
|
||||
let files = self.list_files().await?;
|
||||
let file = files
|
||||
.iter()
|
||||
.find(|v| v.relative_filename == sub_path)
|
||||
.expect("file not found");
|
||||
|
||||
Ok(file.clone())
|
||||
}
|
||||
|
||||
fn require_whole_files(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::{
|
||||
fs::{metadata, read_dir},
|
||||
io::SeekFrom,
|
||||
path::{Path, PathBuf},
|
||||
process::Stdio,
|
||||
sync::LazyLock,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use async_trait::async_trait;
|
||||
use tokio::{
|
||||
fs::File,
|
||||
io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader},
|
||||
process::Command,
|
||||
};
|
||||
|
||||
use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile};
|
||||
|
||||
pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) -> Result<()> {
|
||||
if metadata(path)?.is_dir() {
|
||||
let paths = read_dir(path)?;
|
||||
for path_result in paths {
|
||||
let full_path = path_result?.path();
|
||||
if metadata(&full_path)?.is_dir() {
|
||||
_list_files(vec, &full_path)?;
|
||||
} else {
|
||||
vec.push(full_path);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PathVersionBackend {
|
||||
pub base_dir: PathBuf,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VersionBackend for PathVersionBackend {
|
||||
async fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
|
||||
let mut vec = Vec::new();
|
||||
_list_files(&mut vec, &self.base_dir)?;
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
for pathbuf in vec.iter() {
|
||||
let relative = pathbuf.strip_prefix(self.base_dir.clone())?;
|
||||
|
||||
results.push(
|
||||
self.peek_file(
|
||||
relative
|
||||
.to_str()
|
||||
.ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))?
|
||||
.to_owned(),
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn reader(
|
||||
&mut self,
|
||||
file: &VersionFile,
|
||||
start: u64,
|
||||
end: u64,
|
||||
) -> anyhow::Result<Box<dyn MinimumFileObject>> {
|
||||
let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?;
|
||||
|
||||
if start != 0 {
|
||||
file.seek(SeekFrom::Start(start)).await?;
|
||||
}
|
||||
|
||||
if end != 0 {
|
||||
return Ok(Box::new(file.take(end - start)));
|
||||
}
|
||||
|
||||
Ok(Box::new(file))
|
||||
}
|
||||
|
||||
async fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
|
||||
let pathbuf = self.base_dir.join(sub_path.clone());
|
||||
if !pathbuf.exists() {
|
||||
return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy()));
|
||||
};
|
||||
|
||||
let file = File::open(pathbuf.clone()).await?;
|
||||
let metadata = file.try_clone().await?.metadata().await?;
|
||||
let permission_object = metadata.permissions();
|
||||
let permissions = {
|
||||
let perm: u32;
|
||||
#[cfg(target_family = "unix")]
|
||||
{
|
||||
perm = permission_object.mode();
|
||||
}
|
||||
#[cfg(not(target_family = "unix"))]
|
||||
{
|
||||
perm = 0
|
||||
}
|
||||
perm
|
||||
};
|
||||
|
||||
Ok(VersionFile {
|
||||
relative_filename: sub_path,
|
||||
permission: permissions,
|
||||
size: metadata.len(),
|
||||
})
|
||||
}
|
||||
|
||||
fn require_whole_files(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub static SEVEN_ZIP_INSTALLED: LazyLock<bool> =
|
||||
LazyLock::new(|| std::process::Command::new("7z").output().is_ok());
|
||||
// https://7-zip.opensource.jp/chm/general/formats.htm
|
||||
// Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above
|
||||
pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [
|
||||
"7z", "bz2", "bzip2", "tbz2", "tbz", "gz", "gzip", "tgz", "tar", "wim", "swm", "esd", "xz",
|
||||
"txz", "zip", "zipx", "jar", "xpi", "odt", "ods", "docx", "xlsx", "epub", "apm", "ar", "a",
|
||||
"deb", "lib", "arj", "cab", "chm", "chw", "chi", "chq", "msi", "msp", "doc", "xls", "ppt",
|
||||
"cpio", "cramfs", "dmg", "ext", "ext2", "ext3", "ext4", "img", "fat", "img", "hfs", "hfsx",
|
||||
"hxs", "hxr", "hxq", "hxw", "lit", "ihex", "iso", "img", "lzh", "lha", "lzma", "mbr", "mslz",
|
||||
"mub", "nsis", "ntfs", "img", "mbr", "rar", "r00", "rpm", "ppmd", "qcow", "qcow2", "qcow2c",
|
||||
"squashfs", "udf", "iso", "img", "scap", "uefif", "vdi", "vhd", "vmdk", "xar", "pkg", "z",
|
||||
"taz",
|
||||
];
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ZipVersionBackend {
|
||||
path: String,
|
||||
}
|
||||
impl ZipVersionBackend {
|
||||
pub fn new(path: PathBuf) -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
path: path.to_str().expect("invalid utf path").to_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VersionBackend for ZipVersionBackend {
|
||||
async fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>> {
|
||||
let mut list_command = Command::new("7z");
|
||||
list_command.args(vec!["l", &self.path]);
|
||||
let result = list_command.output().await?;
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!(
|
||||
"failed to list files: code {:?}",
|
||||
result.status.code()
|
||||
));
|
||||
}
|
||||
let raw_result = String::from_utf8(result.stdout)?;
|
||||
let mut lines = raw_result.split("\n").skip(11);
|
||||
|
||||
let mut column_lines = lines
|
||||
.find(|v| v.starts_with('-'))
|
||||
.ok_or(anyhow!("invalid 7z output"))?
|
||||
.chars();
|
||||
let file_lines = lines.take_while(|v| !v.starts_with("-"));
|
||||
|
||||
/*
|
||||
Date Time Attr Size Compressed Name
|
||||
------------------- ----- ------------ ------------ ------------------------
|
||||
*/
|
||||
let datetime_pos = 0usize;
|
||||
let attrs_pos = datetime_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
|
||||
let size_pos = attrs_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
|
||||
let compressed_size_pos = size_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
|
||||
let name_pos = compressed_size_pos + column_lines.position(|v| v == ' ').unwrap() + 1;
|
||||
|
||||
let mut results = Vec::new();
|
||||
for file in file_lines {
|
||||
let name = file[name_pos..].trim();
|
||||
let size = file[size_pos..compressed_size_pos].trim();
|
||||
|
||||
let size = str::parse::<u64>(size)?;
|
||||
|
||||
let version_file = VersionFile {
|
||||
relative_filename: name.to_string(),
|
||||
permission: 0o744,
|
||||
size,
|
||||
};
|
||||
|
||||
results.push(version_file);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn reader(
|
||||
&mut self,
|
||||
file: &VersionFile,
|
||||
_start: u64,
|
||||
_end: u64,
|
||||
) -> anyhow::Result<Box<dyn MinimumFileObject>> {
|
||||
let mut read_command = Command::new("7z");
|
||||
read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]);
|
||||
let mut output = read_command
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("failed to spawn 7z");
|
||||
let stdout = output
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or(anyhow!("no stdout on 7zip process"))?;
|
||||
let reader = BufReader::new(stdout);
|
||||
Ok(Box::new(reader))
|
||||
}
|
||||
|
||||
async fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile> {
|
||||
let files = self.list_files().await?;
|
||||
let file = files
|
||||
.iter()
|
||||
.find(|v| v.relative_filename == sub_path)
|
||||
.expect("file not found");
|
||||
|
||||
Ok(file.clone())
|
||||
}
|
||||
|
||||
fn require_whole_files(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,37 @@
|
||||
use std::path::Path;
|
||||
use std::{
|
||||
fs::{metadata, read_dir},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::versions::{
|
||||
backends::{
|
||||
PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED, SUPPORTED_FILE_EXTENSIONS,
|
||||
},
|
||||
types::VersionBackend,
|
||||
archive_backend::ZipVersionBackend, path_backend::PathVersionBackend, types::VersionBackend,
|
||||
};
|
||||
|
||||
pub mod backends;
|
||||
pub mod archive_backend;
|
||||
pub mod path_backend;
|
||||
|
||||
pub fn _list_files(vec: &mut Vec<PathBuf>, path: &Path) -> Result<()> {
|
||||
if metadata(path)?.is_dir() {
|
||||
let paths = read_dir(path)?;
|
||||
for path_result in paths {
|
||||
let full_path = path_result?.path();
|
||||
if metadata(&full_path)?.is_dir() {
|
||||
_list_files(vec, &full_path)?;
|
||||
} else {
|
||||
vec.push(full_path);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const SUPPORTED_FILE_EXTENSIONS: [&'static str; 11] = [
|
||||
"tar", "pax", "cpio", "zip", "jar", "ar", "xar", "rar", "rpm", "7z", "iso",
|
||||
];
|
||||
|
||||
pub mod types;
|
||||
pub fn create_backend_constructor<'a>(
|
||||
path: &Path,
|
||||
@@ -26,27 +48,17 @@ pub fn create_backend_constructor<'a>(
|
||||
}));
|
||||
};
|
||||
|
||||
if *SEVEN_ZIP_INSTALLED {
|
||||
/*
|
||||
Slow 7zip integrity test
|
||||
let mut test = Command::new("7z");
|
||||
test.args(vec!["t", path.to_str().expect("invalid utf path")]);
|
||||
let status = test.status().ok()?;
|
||||
if status.code().unwrap_or(1) == 0 {
|
||||
let buf = path.to_path_buf();
|
||||
return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
|
||||
}
|
||||
*/
|
||||
// Fast filename-based test
|
||||
if let Some(extension) = path.extension().and_then(|v| v.to_str()) {
|
||||
let supported = SUPPORTED_FILE_EXTENSIONS
|
||||
.iter()
|
||||
.any(|v| **v == *extension);
|
||||
if supported {
|
||||
let buf = path.to_path_buf();
|
||||
return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
|
||||
}
|
||||
}
|
||||
let file_extension = path
|
||||
.extension()
|
||||
.map(|v| v.to_str())
|
||||
.flatten()?;
|
||||
|
||||
if SUPPORTED_FILE_EXTENSIONS
|
||||
.iter()
|
||||
.any(|v| *v == file_extension)
|
||||
{
|
||||
let buf = path.to_path_buf();
|
||||
return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
|
||||
}
|
||||
|
||||
None
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::{
|
||||
io::SeekFrom,
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use async_trait::async_trait;
|
||||
use tokio::{
|
||||
fs::File,
|
||||
io::{AsyncReadExt as _, AsyncSeekExt as _},
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PathVersionBackend {
|
||||
pub base_dir: PathBuf,
|
||||
}
|
||||
|
||||
use crate::versions::{_list_files, types::{MinimumFileObject, VersionBackend, VersionFile}};
|
||||
|
||||
#[async_trait]
|
||||
impl VersionBackend for PathVersionBackend {
|
||||
async fn list_files(&self) -> anyhow::Result<Vec<VersionFile>> {
|
||||
let mut vec = Vec::new();
|
||||
_list_files(&mut vec, &self.base_dir)?;
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
for pathbuf in vec.iter() {
|
||||
let relative = pathbuf.strip_prefix(self.base_dir.clone())?;
|
||||
|
||||
results.push(
|
||||
self.peek_file(
|
||||
relative
|
||||
.to_str()
|
||||
.ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))?
|
||||
.to_owned(),
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn reader(
|
||||
&self,
|
||||
file: &VersionFile,
|
||||
start: u64,
|
||||
end: u64,
|
||||
) -> anyhow::Result<Box<dyn MinimumFileObject>> {
|
||||
let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?;
|
||||
|
||||
if start != 0 {
|
||||
file.seek(SeekFrom::Start(start)).await?;
|
||||
}
|
||||
|
||||
if end != 0 {
|
||||
return Ok(Box::new(file.take(end - start)));
|
||||
}
|
||||
|
||||
Ok(Box::new(file))
|
||||
}
|
||||
|
||||
async fn peek_file(&self, sub_path: String) -> anyhow::Result<VersionFile> {
|
||||
let pathbuf = self.base_dir.join(sub_path.clone());
|
||||
if !pathbuf.exists() {
|
||||
return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy()));
|
||||
};
|
||||
|
||||
let file = File::open(pathbuf.clone()).await?;
|
||||
let metadata = file.try_clone().await?.metadata().await?;
|
||||
let permission_object = metadata.permissions();
|
||||
let permissions = {
|
||||
let perm: u32;
|
||||
#[cfg(target_family = "unix")]
|
||||
{
|
||||
perm = permission_object.mode();
|
||||
}
|
||||
#[cfg(not(target_family = "unix"))]
|
||||
{
|
||||
perm = 0
|
||||
}
|
||||
perm
|
||||
};
|
||||
|
||||
Ok(VersionFile {
|
||||
relative_filename: sub_path,
|
||||
permission: permissions,
|
||||
size: metadata.len(),
|
||||
})
|
||||
}
|
||||
|
||||
fn require_whole_files(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::fmt::Debug;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use dyn_clone::DynClone;
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -16,16 +15,14 @@ impl<T: AsyncRead + Send + Unpin> MinimumFileObject for T {}
|
||||
|
||||
|
||||
#[async_trait]
|
||||
pub trait VersionBackend: DynClone {
|
||||
pub trait VersionBackend {
|
||||
fn require_whole_files(&self) -> bool;
|
||||
async fn list_files(&mut self) -> anyhow::Result<Vec<VersionFile>>;
|
||||
async fn peek_file(&mut self, sub_path: String) -> anyhow::Result<VersionFile>;
|
||||
async fn list_files(&self) -> anyhow::Result<Vec<VersionFile>>;
|
||||
async fn peek_file(&self, sub_path: String) -> anyhow::Result<VersionFile>;
|
||||
async fn reader(
|
||||
&mut self,
|
||||
&self,
|
||||
file: &VersionFile,
|
||||
start: u64,
|
||||
end: u64,
|
||||
) -> anyhow::Result<Box<dyn MinimumFileObject>>;
|
||||
}
|
||||
|
||||
dyn_clone::clone_trait_object!(VersionBackend);
|
||||
|
||||
Reference in New Issue
Block a user