github/ouch
1
1
Fork 0
mirror of https://github.com/ouch-org/ouch.git synced 2025-06-06 11:35:45 +00:00
Code Issues 15 Packages Projects Releases Wiki Activity

Add progress bar for compressing/decompressing

Browse Source
This commit is contained in:
Nbiba Bedis 2021-11-25 11:57:47 +01:00
parent b31f407011
commit 0976970e8c
10 changed files with 291 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Cargo.lock generated
View File

@ -172,6 +172,19 @@ dependencies = [
"clap",
]
[[package]]
name = "console"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"terminal_size",
"winapi",
]
[[package]]
name = "crc32fast"
version = "1.2.1"
@ -199,6 +212,12 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "filetime"
version = "0.2.15"
@ -281,6 +300,18 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "indicatif"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
]
[[package]]
name = "infer"
version = "0.5.0"
@ -382,6 +413,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "number_prefix"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.8.0"
@ -408,6 +445,7 @@ dependencies = [
"clap_generate",
"flate2",
"fs-err",
"indicatif",
"infer",
"libc",
"linked-hash-map",
@ -749,6 +787,16 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "terminal_size"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "termtree"
version = "0.2.3"

1
Cargo.toml
View File

@ -28,6 +28,7 @@ xz2 = "0.1.6"
zip = { version = "0.5.13", default-features = false }
zstd = { version = "0.9.0", default-features = false }
tempfile = "3.2.0"
indicatif = "0.16.2"
[build-dependencies]
clap = "=3.0.0-beta.5"

17
src/archive/tar.rs
View File

@ -15,15 +15,16 @@ use crate::{
info,
list::FileInArchive,
utils::{self, Bytes},
QuestionPolicy,
};
/// Unpacks the archive given by `archive` into the folder given by `into`.
/// Assumes that output_folder is empty
pub fn unpack_archive(
reader: Box<dyn Read>,
output_folder: &Path,
question_policy: QuestionPolicy,
mut display_handle: impl Write,
) -> crate::Result<Vec<PathBuf>> {
assert!(output_folder.read_dir().expect("dir exists").count() == 0);
let mut archive = tar::Archive::new(reader);
let mut files_unpacked = vec![];
@ -31,18 +32,13 @@ pub fn unpack_archive(
let mut file = file?;
let file_path = output_folder.join(file.path()?);
if !utils::clear_path(&file_path, question_policy)? {
// User doesn't want to overwrite
continue;
}
file.unpack_in(output_folder)?;
// This is printed for every file in the archive and has little
// importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "{:?} extracted. ({})", output_folder.join(file.path()?), Bytes::new(file.size()));
info!(@display_handle, inaccessible, "{:?} extracted. ({})", output_folder.join(file.path()?), Bytes::new(file.size()));
files_unpacked.push(file_path);
}
@ -68,9 +64,10 @@ pub fn list_archive(reader: Box<dyn Read>) -> crate::Result<Vec<FileInArchive>>
}
/// Compresses the archives given by `input_filenames` into the file given previously to `writer`.
pub fn build_archive_from_paths<W>(input_filenames: &[PathBuf], writer: W) -> crate::Result<W>
pub fn build_archive_from_paths<W, D>(input_filenames: &[PathBuf], writer: W, mut display_handle: D) -> crate::Result<W>
where
W: Write,
D: Write,
{
let mut builder = tar::Builder::new(writer);
@ -88,7 +85,7 @@ where
// little importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "Compressing '{}'.", utils::to_utf(path));
info!(@display_handle, inaccessible, "Compressing '{}'.", utils::to_utf(path));
if path.is_dir() {
builder.append_dir(path, path)?;

31
src/archive/zip.rs
View File

@ -15,21 +15,23 @@ use crate::{
info,
list::FileInArchive,
utils::{
cd_into_same_dir_as, clear_path, concatenate_os_str_list, dir_is_empty, get_invalid_utf8_paths, strip_cur_dir,
to_utf, Bytes,
cd_into_same_dir_as, concatenate_os_str_list, dir_is_empty, get_invalid_utf8_paths, strip_cur_dir, to_utf,
Bytes,
},
QuestionPolicy,
};
/// Unpacks the archive given by `archive` into the folder given by `into`.
pub fn unpack_archive<R>(
/// Unpacks the archive given by `archive` into the folder given by `output_folder`.
/// Assumes that output_folder is empty
pub fn unpack_archive<R, D>(
mut archive: ZipArchive<R>,
into: &Path,
question_policy: QuestionPolicy,
output_folder: &Path,
mut display_handle: D,
) -> crate::Result<Vec<PathBuf>>
where
R: Read + Seek,
D: Write,
{
assert!(output_folder.read_dir().expect("dir exists").count() == 0);
let mut unpacked_files = vec![];
for idx in 0..archive.len() {
let mut file = archive.by_index(idx)?;
@ -38,11 +40,7 @@ where
None => continue,
};
let file_path = into.join(file_path);
if !clear_path(&file_path, question_policy)? {
// User doesn't want to overwrite
continue;
}
let file_path = output_folder.join(file_path);
check_for_comments(&file);
@ -52,7 +50,7 @@ where
// importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "File {} extracted to \"{}\"", idx, file_path.display());
info!(@display_handle, inaccessible, "File {} extracted to \"{}\"", idx, file_path.display());
fs::create_dir_all(&file_path)?;
}
_is_file @ false => {
@ -64,7 +62,7 @@ where
let file_path = strip_cur_dir(file_path.as_path());
// same reason is in _is_dir: long, often not needed text
info!(inaccessible, "{:?} extracted. ({})", file_path.display(), Bytes::new(file.size()));
info!(@display_handle, inaccessible, "{:?} extracted. ({})", file_path.display(), Bytes::new(file.size()));
let mut output_file = fs::File::create(&file_path)?;
io::copy(&mut file, &mut output_file)?;
@ -102,9 +100,10 @@ where
}
/// Compresses the archives given by `input_filenames` into the file given previously to `writer`.
pub fn build_archive_from_paths<W>(input_filenames: &[PathBuf], writer: W) -> crate::Result<W>
pub fn build_archive_from_paths<W, D>(input_filenames: &[PathBuf], writer: W, mut display_handle: D) -> crate::Result<W>
where
W: Write + Seek,
D: Write,
{
let mut writer = zip::ZipWriter::new(writer);
let options = zip::write::FileOptions::default();
@ -134,7 +133,7 @@ where
// little importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "Compressing '{}'.", to_utf(path));
info!(@display_handle, inaccessible, "Compressing '{}'.", to_utf(path));
if path.is_dir() {
if dir_is_empty(path) {

3
src/cli.rs
View File

@ -13,7 +13,8 @@ use once_cell::sync::OnceCell;
use crate::{Opts, QuestionPolicy, Subcommand};
/// Whether to enable accessible output (removes info output and reduces other
/// output, removes visual markers like '[' and ']')
/// output, removes visual markers like '[' and ']').
/// Removes th progress bar as well
pub static ACCESSIBLE: OnceCell<bool> = OnceCell::new();
impl Opts {

85
src/commands.rs
View File

@ -21,6 +21,7 @@ use crate::{
},
info,
list::{self, ListOptions},
progress::Progress,
utils::{
self, concatenate_os_str_list, dir_is_empty, nice_directory_display, to_utf, try_infer_extension,
user_wants_to_continue_decompressing,
@ -280,6 +281,18 @@ pub fn run(args: Opts, question_policy: QuestionPolicy) -> crate::Result<()> {
// formats contains each format necessary for compression, example: [Tar, Gz] (in compression order)
// output_file is the resulting compressed file name, example: "compressed.tar.gz"
fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs::File) -> crate::Result<()> {
// The next lines are for displaying the progress bar
// If the input files contain a directory, then the total size will be underestimated
let (total_input_size, precise) = files
.iter()
.map(|f| (f.metadata().expect("file exists").len(), f.is_file()))
.fold((0, true), |(total_size, and_precise), (size, precise)| (total_size + size, and_precise & precise));
//NOTE: canonicalize is here to avoid a weird bug:
// > If output_file_path is a nested path and it exists and the user overwrite it
// >> output_file_path.exists() will always return false (somehow)
// - canonicalize seems to fix this
let output_file_path = output_file.path().canonicalize()?;
let file_writer = BufWriter::with_capacity(BUFFER_CAPACITY, output_file);
let mut writer: Box<dyn Write> = Box::new(file_writer);
@ -309,12 +322,28 @@ fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs:
match formats[0].compression_formats[0] {
Gzip | Bzip | Lz4 | Lzma | Zstd => {
let _progress = Progress::new_accessible_aware(
total_input_size,
precise,
Some(Box::new(move || output_file_path.metadata().expect("file exists").len())),
);
writer = chain_writer_encoder(&formats[0].compression_formats[0], writer)?;
let mut reader = fs::File::open(&files[0]).unwrap();
io::copy(&mut reader, &mut writer)?;
}
Tar => {
let mut writer = archive::tar::build_archive_from_paths(&files, writer)?;
let mut progress = Progress::new_accessible_aware(
total_input_size,
precise,
Some(Box::new(move || output_file_path.metadata().expect("file exists").len())),
);
archive::tar::build_archive_from_paths(
&files,
&mut writer,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)?;
writer.flush()?;
}
Zip => {
@ -328,7 +357,27 @@ fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs:
eprintln!("\tThe design of .zip makes it impossible to compress via stream.");
let mut vec_buffer = io::Cursor::new(vec![]);
archive::zip::build_archive_from_paths(&files, &mut vec_buffer)?;
let current_position_fn = {
let vec_buffer_ptr = {
struct FlyPtr(*const io::Cursor<Vec<u8>>);
unsafe impl Send for FlyPtr {}
FlyPtr(&vec_buffer as *const _)
};
Box::new(move || {
let vec_buffer_ptr = &vec_buffer_ptr;
// Safety: ptr is valid and vec_buffer is still alive
unsafe { &*vec_buffer_ptr.0 }.position()
})
};
let mut progress = Progress::new_accessible_aware(total_input_size, precise, Some(current_position_fn));
archive::zip::build_archive_from_paths(
&files,
&mut vec_buffer,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)?;
let vec_buffer = vec_buffer.into_inner();
io::copy(&mut vec_buffer.as_slice(), &mut writer)?;
}
@ -351,6 +400,7 @@ fn decompress_file(
question_policy: QuestionPolicy,
) -> crate::Result<()> {
assert!(output_dir.exists());
let total_input_size = input_file_path.metadata().expect("file exists").len();
let reader = fs::File::open(&input_file_path)?;
// Zip archives are special, because they require io::Seek, so it requires it's logic separated
// from decoder chaining.
@ -362,7 +412,14 @@ fn decompress_file(
if formats.len() == 1 && *formats[0].compression_formats == [Zip] {
let zip_archive = zip::ZipArchive::new(reader)?;
let files = if let ControlFlow::Continue(files) = smart_unpack(
Box::new(move |output_dir| crate::archive::zip::unpack_archive(zip_archive, output_dir, question_policy)),
Box::new(move |output_dir| {
let mut progress = Progress::new_accessible_aware(total_input_size, true, None);
crate::archive::zip::unpack_archive(
zip_archive,
output_dir,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)
}),
output_dir,
&output_file_path,
question_policy,
@ -419,12 +476,25 @@ fn decompress_file(
}
let mut writer = writer.unwrap();
let current_position_fn = Box::new({
let output_file_path = output_file_path.clone();
move || output_file_path.clone().metadata().expect("file exists").len()
});
let _progress = Progress::new_accessible_aware(total_input_size, true, Some(current_position_fn));
io::copy(&mut reader, &mut writer)?;
files_unpacked = vec![output_file_path];
}
Tar => {
files_unpacked = if let ControlFlow::Continue(files) = smart_unpack(
Box::new(move |output_dir| crate::archive::tar::unpack_archive(reader, output_dir, question_policy)),
Box::new(move |output_dir| {
let mut progress = Progress::new_accessible_aware(total_input_size, true, None);
crate::archive::tar::unpack_archive(
reader,
output_dir,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)
}),
output_dir,
&output_file_path,
question_policy,
@ -448,7 +518,12 @@ fn decompress_file(
files_unpacked = if let ControlFlow::Continue(files) = smart_unpack(
Box::new(move |output_dir| {
crate::archive::zip::unpack_archive(zip_archive, output_dir, question_policy)
let mut progress = Progress::new_accessible_aware(total_input_size, true, None);
crate::archive::zip::unpack_archive(
zip_archive,
output_dir,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)
}),
output_dir,
&output_file_path,

1
src/lib.rs
View File

@ -15,6 +15,7 @@ pub mod commands;
pub mod error;
pub mod extension;
pub mod list;
pub mod progress;
pub mod utils;
/// CLI argparsing definitions, using `clap`.

29
src/macros.rs
View File

@ -14,32 +14,45 @@
/// while it would generate long and hard to navigate text for blind people
/// who have to have each line of output read to them aloud, whithout to
/// ability to skip some lines deemed not important like a seeing person would.
///
/// By default `info` outputs to Stdout, if you want to specify the output you can use
/// `@display_handle` modifier
#[macro_export]
macro_rules! info {
// Accessible (short/important) info message.
// Show info message even in ACCESSIBLE mode
(accessible, $($arg:tt)*) => {
info!(@::std::io::stdout(), accessible, $($arg)*);
};
(@$display_handle: expr, accessible, $($arg:tt)*) => {
let display_handle = &mut $display_handle;
// if in ACCESSIBLE mode, suppress the "[INFO]" and just print the message
if (!$crate::cli::ACCESSIBLE.get().unwrap()) {
$crate::macros::_info_helper();
if !(*$crate::cli::ACCESSIBLE.get().unwrap()) {
$crate::macros::_info_helper(display_handle);
}
println!($($arg)*);
writeln!(display_handle, $($arg)*).unwrap();
};
// Inccessible (long/no important) info message.
// Print info message if ACCESSIBLE is not turned on
(inaccessible, $($arg:tt)*) => {
if (!$crate::cli::ACCESSIBLE.get().unwrap()) {
$crate::macros::_info_helper();
println!($($arg)*);
info!(@::std::io::stdout(), inaccessible, $($arg)*);
};
(@$display_handle: expr, inaccessible, $($arg:tt)*) => {
if (!$crate::cli::ACCESSIBLE.get().unwrap())
{
let display_handle = &mut $display_handle;
$crate::macros::_info_helper(display_handle);
writeln!(display_handle, $($arg)*).unwrap();
}
};
}
/// Helper to display "\[INFO\]", colored yellow
pub fn _info_helper() {
pub fn _info_helper(handle: &mut impl std::io::Write) {
use crate::utils::colors::{RESET, YELLOW};
print!("{}[INFO]{} ", *YELLOW, *RESET);
write!(handle, "{}[INFO]{} ", *YELLOW, *RESET).unwrap();
}
/// Macro that prints \[WARNING\] messages, wraps [`println`].

115
src/progress.rs Normal file
View File

@ -0,0 +1,115 @@
//! Module that provides functions to display progress bars for compressing and decompressing files.
use std::{
io,
sync::mpsc::{self, Receiver, Sender},
thread,
time::Duration,
};
use indicatif::{ProgressBar, ProgressStyle};
/// Draw a ProgressBar using a function that checks periodically for the progress
pub struct Progress {
draw_stop: Sender<()>,
clean_done: Receiver<()>,
display_handle: DisplayHandle,
}
/// Writes to this struct will be displayed on the progress bar or stdout depending on the
/// ProgressBarPolicy
struct DisplayHandle {
buf: Vec<u8>,
sender: Sender<String>,
}
impl io::Write for DisplayHandle {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.buf.extend(buf);
// Newline is the signal to flush
if matches!(buf.last(), Some(&b'\n')) {
self.buf.pop();
self.flush()?;
}
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
fn io_error<X>(_: X) -> io::Error {
io::Error::new(io::ErrorKind::Other, "failed to flush buffer")
}
self.sender.send(String::from_utf8(self.buf.drain(..).collect()).map_err(io_error)?).map_err(io_error)
}
}
impl Progress {
/// Create a ProgressBar using a function that checks periodically for the progress
/// If precise is true, the total_input_size will be displayed as the total_bytes size
/// If ACCESSIBLE is set, this function returns None
pub fn new_accessible_aware(
total_input_size: u64,
precise: bool,
current_position_fn: Option<Box<dyn Fn() -> u64 + Send>>,
) -> Option<Self> {
if *crate::cli::ACCESSIBLE.get().unwrap() {
return None;
}
Some(Self::new(total_input_size, precise, current_position_fn))
}
fn new(total_input_size: u64, precise: bool, current_position_fn: Option<Box<dyn Fn() -> u64 + Send>>) -> Self {
let (draw_tx, draw_rx) = mpsc::channel();
let (clean_tx, clean_rx) = mpsc::channel();
let (msg_tx, msg_rx) = mpsc::channel();
thread::spawn(move || {
let template = {
let mut t = String::new();
t += "{prefix} [{elapsed_precise}] ";
if precise && current_position_fn.is_some() {
t += "[{wide_bar:.cyan/blue}] ";
} else {
t += "{spinner:.green} ";
}
if current_position_fn.is_some() {
t += "{bytes}/ ";
}
if precise {
t += "{total_bytes} ";
}
t += "({bytes_per_sec}, {eta}) {path}";
t
};
let pb = ProgressBar::new(total_input_size);
pb.set_style(ProgressStyle::default_bar().template(&template).progress_chars("#>-"));
while draw_rx.try_recv().is_err() {
if let Some(ref pos_fn) = current_position_fn {
pb.set_position(pos_fn());
} else {
pb.tick();
}
if let Ok(msg) = msg_rx.try_recv() {
pb.set_prefix(msg);
}
thread::sleep(Duration::from_millis(100));
}
pb.finish();
let _ = clean_tx.send(());
});
Progress {
draw_stop: draw_tx,
clean_done: clean_rx,
display_handle: DisplayHandle { buf: Vec::new(), sender: msg_tx },
}
}
pub(crate) fn display_handle(&mut self) -> &mut dyn io::Write {
&mut self.display_handle
}
}
impl Drop for Progress {
fn drop(&mut self) {
let _ = self.draw_stop.send(());
let _ = self.clean_done.recv();
}
}

2
src/utils/fs.rs
View File

@ -3,7 +3,7 @@
use std::{
env,
fs::ReadDir,
io::Read,
io::{Read, Write},
path::{Path, PathBuf},
};