Decompress files in parallel

This commit is contained in:
Vinícius R. Miguel 2023-01-05 21:15:12 -03:00
parent 4754a6d4e3
commit 23e33412a4
3 changed files with 99 additions and 11 deletions

82
Cargo.lock generated
View File

@ -208,6 +208,40 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.14" version = "0.8.14"
@ -509,6 +543,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.6.2" version = "0.6.2"
@ -527,6 +570,16 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "num_cpus"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi 0.2.6",
"libc",
]
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.17.0" version = "1.17.0"
@ -563,6 +616,7 @@ dependencies = [
"parse-display", "parse-display",
"proptest", "proptest",
"rand", "rand",
"rayon",
"same-file", "same-file",
"snap", "snap",
"tar", "tar",
@ -754,6 +808,28 @@ dependencies = [
"rand_core", "rand_core",
] ]
[[package]]
name = "rayon"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.16" version = "0.2.16"
@ -836,6 +912,12 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.152" version = "1.0.152"

View File

@ -23,6 +23,7 @@ libc = "0.2.139"
linked-hash-map = "0.5.6" linked-hash-map = "0.5.6"
lzzzz = "1.0.4" lzzzz = "1.0.4"
once_cell = "1.17.0" once_cell = "1.17.0"
rayon = "1.6.1"
same-file = "1.0.6" same-file = "1.0.6"
snap = "1.1.0" snap = "1.1.0"
tar = "0.4.38" tar = "0.4.38"

View File

@ -9,6 +9,7 @@ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
use rayon::prelude::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
use utils::colors; use utils::colors;
use crate::{ use crate::{
@ -272,17 +273,21 @@ pub fn run(
PathBuf::from(".") PathBuf::from(".")
}; };
for ((input_path, formats), file_name) in files.iter().zip(formats).zip(output_paths) { files
let output_file_path = output_dir.join(file_name); // Path used by single file format archives .par_iter()
decompress_file( .zip(formats)
input_path, .zip(output_paths)
formats, .try_for_each(|((input_path, formats), file_name)| {
&output_dir, let output_file_path = output_dir.join(file_name); // Path used by single file format archives
output_file_path, decompress_file(
question_policy, input_path,
args.quiet, formats,
)?; &output_dir,
} output_file_path,
question_policy,
args.quiet,
)
})?;
} }
Subcommand::List { archives: files, tree } => { Subcommand::List { archives: files, tree } => {
let mut formats = vec![]; let mut formats = vec![];