From 0ec7d4489d4827ff6f3f4c776e85a4f7e12443c7 Mon Sep 17 00:00:00 2001 From: nalabrie Date: Sun, 14 Jul 2024 02:54:12 -0400 Subject: [PATCH] Added multithreading by default to zstd compression. Bumped zstd package version. Added num_cpus package to get core count. --- CHANGELOG.md | 2 ++ Cargo.lock | 5 +++-- Cargo.toml | 3 ++- src/commands/compress.rs | 8 +++++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c69577..7598567 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ Categories Used: ### New Features +- Add multithreading support for `zstd` compression [\#688](https://github.com/ouch-org/ouch/pull/688) ([nalabrie](https://github.com/nalabrie)) + ### Bug Fixes - Fix output corrupted on parallel decompression [\#642](https://github.com/ouch-org/ouch/pull/642) ([AntoniosBarotsis](https://github.com/AntoniosBarotsis)) diff --git a/Cargo.lock b/Cargo.lock index e535ffc..3b6fa56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -848,6 +848,7 @@ dependencies = [ "libc", "linked-hash-map", "lz4_flex", + "num_cpus", "once_cell", "parse-display", "proptest", @@ -1796,9 +1797,9 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] diff --git a/Cargo.toml b/Cargo.toml index 15a9fba..991f2a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ ignore = "0.4.22" libc = "0.2.155" linked-hash-map = "0.5.6" lz4_flex = "0.11.3" +num_cpus = "1.16.0" once_cell = "1.19.0" rayon = "1.10.0" same-file = "1.0.6" @@ -34,7 +35,7 @@ time = { version = "0.3.36", default-features = false } unrar = { version = "0.5.3", optional = true } xz2 = "0.1.7" zip = { version = "0.6.6", default-features = false, features = ["time"] } -zstd = { version = "0.13.1", default-features = false } +zstd = { version = "0.13.2", default-features = false, features = ["zstdmt"]} [target.'cfg(not(unix))'.dependencies] is_executable = "1.0.1" diff --git a/src/commands/compress.rs b/src/commands/compress.rs index 10fa660..55625ca 100644 --- a/src/commands/compress.rs +++ b/src/commands/compress.rs @@ -69,16 +69,18 @@ pub fn compress_files( .from_writer(encoder), ), Zstd => { - let zstd_encoder = zstd::stream::write::Encoder::new( + let mut zstd_encoder = zstd::stream::write::Encoder::new( encoder, level.map_or(zstd::DEFAULT_COMPRESSION_LEVEL, |l| { (l as i32).clamp(zstd::zstd_safe::min_c_level(), zstd::zstd_safe::max_c_level()) }), - ); + )?; + // Use all available PHYSICAL cores for compression + zstd_encoder.multithread(num_cpus::get_physical() as u32)?; // Safety: // Encoder::new() can only fail if `level` is invalid, but the level // is `clamp`ed and therefore guaranteed to be valid - Box::new(zstd_encoder.unwrap().auto_finish()) + Box::new(zstd_encoder.auto_finish()) } Tar | Zip | Rar | SevenZip => unreachable!(), };