github/ouch
1
1
Fork 0
mirror of https://github.com/ouch-org/ouch.git synced 2025-06-06 11:35:45 +00:00
Code Issues 15 Packages Projects Releases Wiki Activity

Merge pull request #210 from sigmaSd/progress

Browse Source 
Add progress to compressing/decompressing
This commit is contained in:
João Marcos Bezerra 2021-11-25 10:09:32 -03:00 committed by GitHub
commit bafc2d31b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 291 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Cargo.lock generated
View File

@ -172,6 +172,19 @@ dependencies = [
"clap",
]
[[package]]
name = "console"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"terminal_size",
"winapi",
]
[[package]]
name = "crc32fast"
version = "1.2.1"
@ -199,6 +212,12 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "filetime"
version = "0.2.15"
@ -281,6 +300,18 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "indicatif"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
]
[[package]]
name = "infer"
version = "0.5.0"
@ -382,6 +413,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "number_prefix"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.8.0"
@ -408,6 +445,7 @@ dependencies = [
"clap_generate",
"flate2",
"fs-err",
"indicatif",
"infer",
"libc",
"linked-hash-map",
@ -749,6 +787,16 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "terminal_size"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "termtree"
version = "0.2.3"

1
Cargo.toml
View File

@ -28,6 +28,7 @@ xz2 = "0.1.6"
zip = { version = "0.5.13", default-features = false }
zstd = { version = "0.9.0", default-features = false }
tempfile = "3.2.0"
indicatif = "0.16.2"
[build-dependencies]
clap = "=3.0.0-beta.5"

17
src/archive/tar.rs
View File

@ -15,15 +15,16 @@ use crate::{
info,
list::FileInArchive,
utils::{self, Bytes},
QuestionPolicy,
};
/// Unpacks the archive given by `archive` into the folder given by `into`.
/// Assumes that output_folder is empty
pub fn unpack_archive(
reader: Box<dyn Read>,
output_folder: &Path,
question_policy: QuestionPolicy,
mut display_handle: impl Write,
) -> crate::Result<Vec<PathBuf>> {
assert!(output_folder.read_dir().expect("dir exists").count() == 0);
let mut archive = tar::Archive::new(reader);
let mut files_unpacked = vec![];
@ -31,18 +32,13 @@ pub fn unpack_archive(
let mut file = file?;
let file_path = output_folder.join(file.path()?);
if !utils::clear_path(&file_path, question_policy)? {
// User doesn't want to overwrite
continue;
}
file.unpack_in(output_folder)?;
// This is printed for every file in the archive and has little
// importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "{:?} extracted. ({})", output_folder.join(file.path()?), Bytes::new(file.size()));
info!(@display_handle, inaccessible, "{:?} extracted. ({})", output_folder.join(file.path()?), Bytes::new(file.size()));
files_unpacked.push(file_path);
}
@ -68,9 +64,10 @@ pub fn list_archive(reader: Box<dyn Read>) -> crate::Result<Vec<FileInArchive>>
}
/// Compresses the archives given by `input_filenames` into the file given previously to `writer`.
pub fn build_archive_from_paths<W>(input_filenames: &[PathBuf], writer: W) -> crate::Result<W>
pub fn build_archive_from_paths<W, D>(input_filenames: &[PathBuf], writer: W, mut display_handle: D) -> crate::Result<W>
where
W: Write,
D: Write,
{
let mut builder = tar::Builder::new(writer);
@ -88,7 +85,7 @@ where
// little importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "Compressing '{}'.", utils::to_utf(path));
info!(@display_handle, inaccessible, "Compressing '{}'.", utils::to_utf(path));
if path.is_dir() {
builder.append_dir(path, path)?;

31
src/archive/zip.rs
View File

@ -15,21 +15,23 @@ use crate::{
info,
list::FileInArchive,
utils::{
cd_into_same_dir_as, clear_path, concatenate_os_str_list, dir_is_empty, get_invalid_utf8_paths, strip_cur_dir,
to_utf, Bytes,
cd_into_same_dir_as, concatenate_os_str_list, dir_is_empty, get_invalid_utf8_paths, strip_cur_dir, to_utf,
Bytes,
},
QuestionPolicy,
};
/// Unpacks the archive given by `archive` into the folder given by `into`.
pub fn unpack_archive<R>(
/// Unpacks the archive given by `archive` into the folder given by `output_folder`.
/// Assumes that output_folder is empty
pub fn unpack_archive<R, D>(
mut archive: ZipArchive<R>,
into: &Path,
question_policy: QuestionPolicy,
output_folder: &Path,
mut display_handle: D,
) -> crate::Result<Vec<PathBuf>>
where
R: Read + Seek,
D: Write,
{
assert!(output_folder.read_dir().expect("dir exists").count() == 0);
let mut unpacked_files = vec![];
for idx in 0..archive.len() {
let mut file = archive.by_index(idx)?;
@ -38,11 +40,7 @@ where
None => continue,
};
let file_path = into.join(file_path);
if !clear_path(&file_path, question_policy)? {
// User doesn't want to overwrite
continue;
}
let file_path = output_folder.join(file_path);
check_for_comments(&file);
@ -52,7 +50,7 @@ where
// importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "File {} extracted to \"{}\"", idx, file_path.display());
info!(@display_handle, inaccessible, "File {} extracted to \"{}\"", idx, file_path.display());
fs::create_dir_all(&file_path)?;
}
_is_file @ false => {
@ -64,7 +62,7 @@ where
let file_path = strip_cur_dir(file_path.as_path());
// same reason is in _is_dir: long, often not needed text
info!(inaccessible, "{:?} extracted. ({})", file_path.display(), Bytes::new(file.size()));
info!(@display_handle, inaccessible, "{:?} extracted. ({})", file_path.display(), Bytes::new(file.size()));
let mut output_file = fs::File::create(&file_path)?;
io::copy(&mut file, &mut output_file)?;
@ -102,9 +100,10 @@ where
}
/// Compresses the archives given by `input_filenames` into the file given previously to `writer`.
pub fn build_archive_from_paths<W>(input_filenames: &[PathBuf], writer: W) -> crate::Result<W>
pub fn build_archive_from_paths<W, D>(input_filenames: &[PathBuf], writer: W, mut display_handle: D) -> crate::Result<W>
where
W: Write + Seek,
D: Write,
{
let mut writer = zip::ZipWriter::new(writer);
let options = zip::write::FileOptions::default();
@ -134,7 +133,7 @@ where
// little importance for most users, but would generate lots of
// spoken text for users using screen readers, braille displays
// and so on
info!(inaccessible, "Compressing '{}'.", to_utf(path));
info!(@display_handle, inaccessible, "Compressing '{}'.", to_utf(path));
if path.is_dir() {
if dir_is_empty(path) {

3
src/cli.rs
View File

@ -13,7 +13,8 @@ use once_cell::sync::OnceCell;
use crate::{Opts, QuestionPolicy, Subcommand};
/// Whether to enable accessible output (removes info output and reduces other
/// output, removes visual markers like '[' and ']')
/// output, removes visual markers like '[' and ']').
/// Removes th progress bar as well
pub static ACCESSIBLE: OnceCell<bool> = OnceCell::new();
impl Opts {

85
src/commands.rs
View File

@ -21,6 +21,7 @@ use crate::{
},
info,
list::{self, ListOptions},
progress::Progress,
utils::{
self, concatenate_os_str_list, dir_is_empty, nice_directory_display, to_utf, try_infer_extension,
user_wants_to_continue_decompressing,
@ -280,6 +281,18 @@ pub fn run(args: Opts, question_policy: QuestionPolicy) -> crate::Result<()> {
// formats contains each format necessary for compression, example: [Tar, Gz] (in compression order)
// output_file is the resulting compressed file name, example: "compressed.tar.gz"
fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs::File) -> crate::Result<()> {
// The next lines are for displaying the progress bar
// If the input files contain a directory, then the total size will be underestimated
let (total_input_size, precise) = files
.iter()
.map(|f| (f.metadata().expect("file exists").len(), f.is_file()))
.fold((0, true), |(total_size, and_precise), (size, precise)| (total_size + size, and_precise & precise));
//NOTE: canonicalize is here to avoid a weird bug:
// > If output_file_path is a nested path and it exists and the user overwrite it
// >> output_file_path.exists() will always return false (somehow)
// - canonicalize seems to fix this
let output_file_path = output_file.path().canonicalize()?;
let file_writer = BufWriter::with_capacity(BUFFER_CAPACITY, output_file);
let mut writer: Box<dyn Write> = Box::new(file_writer);
@ -309,12 +322,28 @@ fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs:
match formats[0].compression_formats[0] {
Gzip | Bzip | Lz4 | Lzma | Zstd => {
let _progress = Progress::new_accessible_aware(
total_input_size,
precise,
Some(Box::new(move || output_file_path.metadata().expect("file exists").len())),
);
writer = chain_writer_encoder(&formats[0].compression_formats[0], writer)?;
let mut reader = fs::File::open(&files[0]).unwrap();
io::copy(&mut reader, &mut writer)?;
}
Tar => {
let mut writer = archive::tar::build_archive_from_paths(&files, writer)?;
let mut progress = Progress::new_accessible_aware(
total_input_size,
precise,
Some(Box::new(move || output_file_path.metadata().expect("file exists").len())),
);
archive::tar::build_archive_from_paths(
&files,
&mut writer,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)?;
writer.flush()?;
}
Zip => {
@ -328,7 +357,27 @@ fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs:
eprintln!("\tThe design of .zip makes it impossible to compress via stream.");
let mut vec_buffer = io::Cursor::new(vec![]);
archive::zip::build_archive_from_paths(&files, &mut vec_buffer)?;
let current_position_fn = {
let vec_buffer_ptr = {
struct FlyPtr(*const io::Cursor<Vec<u8>>);
unsafe impl Send for FlyPtr {}
FlyPtr(&vec_buffer as *const _)
};
Box::new(move || {
let vec_buffer_ptr = &vec_buffer_ptr;
// Safety: ptr is valid and vec_buffer is still alive
unsafe { &*vec_buffer_ptr.0 }.position()
})
};
let mut progress = Progress::new_accessible_aware(total_input_size, precise, Some(current_position_fn));
archive::zip::build_archive_from_paths(
&files,
&mut vec_buffer,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)?;
let vec_buffer = vec_buffer.into_inner();
io::copy(&mut vec_buffer.as_slice(), &mut writer)?;
}
@ -351,6 +400,7 @@ fn decompress_file(
question_policy: QuestionPolicy,
) -> crate::Result<()> {
assert!(output_dir.exists());
let total_input_size = input_file_path.metadata().expect("file exists").len();
let reader = fs::File::open(&input_file_path)?;
// Zip archives are special, because they require io::Seek, so it requires it's logic separated
// from decoder chaining.
@ -362,7 +412,14 @@ fn decompress_file(
if formats.len() == 1 && *formats[0].compression_formats == [Zip] {
let zip_archive = zip::ZipArchive::new(reader)?;
let files = if let ControlFlow::Continue(files) = smart_unpack(
Box::new(move |output_dir| crate::archive::zip::unpack_archive(zip_archive, output_dir, question_policy)),
Box::new(move |output_dir| {
let mut progress = Progress::new_accessible_aware(total_input_size, true, None);
crate::archive::zip::unpack_archive(
zip_archive,
output_dir,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)
}),
output_dir,
&output_file_path,
question_policy,
@ -419,12 +476,25 @@ fn decompress_file(
}
let mut writer = writer.unwrap();
let current_position_fn = Box::new({
let output_file_path = output_file_path.clone();
move || output_file_path.clone().metadata().expect("file exists").len()
});
let _progress = Progress::new_accessible_aware(total_input_size, true, Some(current_position_fn));
io::copy(&mut reader, &mut writer)?;
files_unpacked = vec![output_file_path];
}
Tar => {
files_unpacked = if let ControlFlow::Continue(files) = smart_unpack(
Box::new(move |output_dir| crate::archive::tar::unpack_archive(reader, output_dir, question_policy)),
Box::new(move |output_dir| {
let mut progress = Progress::new_accessible_aware(total_input_size, true, None);
crate::archive::tar::unpack_archive(
reader,
output_dir,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)
}),
output_dir,
&output_file_path,
question_policy,
@ -448,7 +518,12 @@ fn decompress_file(
files_unpacked = if let ControlFlow::Continue(files) = smart_unpack(
Box::new(move |output_dir| {
crate::archive::zip::unpack_archive(zip_archive, output_dir, question_policy)
let mut progress = Progress::new_accessible_aware(total_input_size, true, None);
crate::archive::zip::unpack_archive(
zip_archive,
output_dir,
progress.as_mut().map(Progress::display_handle).unwrap_or(&mut io::stdout()),
)
}),
output_dir,
&output_file_path,

1
src/lib.rs
View File

@ -15,6 +15,7 @@ pub mod commands;
pub mod error;
pub mod extension;
pub mod list;
pub mod progress;
pub mod utils;
/// CLI argparsing definitions, using `clap`.

29
src/macros.rs
View File

@ -14,32 +14,45 @@
/// while it would generate long and hard to navigate text for blind people
/// who have to have each line of output read to them aloud, whithout to
/// ability to skip some lines deemed not important like a seeing person would.
///
/// By default `info` outputs to Stdout, if you want to specify the output you can use
/// `@display_handle` modifier
#[macro_export]
macro_rules! info {
// Accessible (short/important) info message.
// Show info message even in ACCESSIBLE mode
(accessible, $($arg:tt)*) => {
info!(@::std::io::stdout(), accessible, $($arg)*);
};
(@$display_handle: expr, accessible, $($arg:tt)*) => {
let display_handle = &mut $display_handle;
// if in ACCESSIBLE mode, suppress the "[INFO]" and just print the message
if (!$crate::cli::ACCESSIBLE.get().unwrap()) {
$crate::macros::_info_helper();
if !(*$crate::cli::ACCESSIBLE.get().unwrap()) {
$crate::macros::_info_helper(display_handle);
}
println!($($arg)*);
writeln!(display_handle, $($arg)*).unwrap();
};
// Inccessible (long/no important) info message.
// Print info message if ACCESSIBLE is not turned on
(inaccessible, $($arg:tt)*) => {
if (!$crate::cli::ACCESSIBLE.get().unwrap()) {
$crate::macros::_info_helper();
println!($($arg)*);
info!(@::std::io::stdout(), inaccessible, $($arg)*);
};
(@$display_handle: expr, inaccessible, $($arg:tt)*) => {
if (!$crate::cli::ACCESSIBLE.get().unwrap())
{
let display_handle = &mut $display_handle;
$crate::macros::_info_helper(display_handle);
writeln!(display_handle, $($arg)*).unwrap();
}
};
}
/// Helper to display "\[INFO\]", colored yellow
pub fn _info_helper() {
pub fn _info_helper(handle: &mut impl std::io::Write) {
use crate::utils::colors::{RESET, YELLOW};
print!("{}[INFO]{} ", *YELLOW, *RESET);
write!(handle, "{}[INFO]{} ", *YELLOW, *RESET).unwrap();
}
/// Macro that prints \[WARNING\] messages, wraps [`println`].

115
src/progress.rs Normal file
View File

@ -0,0 +1,115 @@
//! Module that provides functions to display progress bars for compressing and decompressing files.
use std::{
io,
sync::mpsc::{self, Receiver, Sender},
thread,
time::Duration,
};
use indicatif::{ProgressBar, ProgressStyle};
/// Draw a ProgressBar using a function that checks periodically for the progress
pub struct Progress {
draw_stop: Sender<()>,
clean_done: Receiver<()>,
display_handle: DisplayHandle,
}
/// Writes to this struct will be displayed on the progress bar or stdout depending on the
/// ProgressBarPolicy
struct DisplayHandle {
buf: Vec<u8>,
sender: Sender<String>,
}
impl io::Write for DisplayHandle {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.buf.extend(buf);
// Newline is the signal to flush
if matches!(buf.last(), Some(&b'\n')) {
self.buf.pop();
self.flush()?;
}
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
fn io_error<X>(_: X) -> io::Error {
io::Error::new(io::ErrorKind::Other, "failed to flush buffer")
}
self.sender.send(String::from_utf8(self.buf.drain(..).collect()).map_err(io_error)?).map_err(io_error)
}
}
impl Progress {
/// Create a ProgressBar using a function that checks periodically for the progress
/// If precise is true, the total_input_size will be displayed as the total_bytes size
/// If ACCESSIBLE is set, this function returns None
pub fn new_accessible_aware(
total_input_size: u64,
precise: bool,
current_position_fn: Option<Box<dyn Fn() -> u64 + Send>>,
) -> Option<Self> {
if *crate::cli::ACCESSIBLE.get().unwrap() {
return None;
}
Some(Self::new(total_input_size, precise, current_position_fn))
}
fn new(total_input_size: u64, precise: bool, current_position_fn: Option<Box<dyn Fn() -> u64 + Send>>) -> Self {
let (draw_tx, draw_rx) = mpsc::channel();
let (clean_tx, clean_rx) = mpsc::channel();
let (msg_tx, msg_rx) = mpsc::channel();
thread::spawn(move || {
let template = {
let mut t = String::new();
t += "{prefix} [{elapsed_precise}] ";
if precise && current_position_fn.is_some() {
t += "[{wide_bar:.cyan/blue}] ";
} else {
t += "{spinner:.green} ";
}
if current_position_fn.is_some() {
t += "{bytes}/ ";
}
if precise {
t += "{total_bytes} ";
}
t += "({bytes_per_sec}, {eta}) {path}";
t
};
let pb = ProgressBar::new(total_input_size);
pb.set_style(ProgressStyle::default_bar().template(&template).progress_chars("#>-"));
while draw_rx.try_recv().is_err() {
if let Some(ref pos_fn) = current_position_fn {
pb.set_position(pos_fn());
} else {
pb.tick();
}
if let Ok(msg) = msg_rx.try_recv() {
pb.set_prefix(msg);
}
thread::sleep(Duration::from_millis(100));
}
pb.finish();
let _ = clean_tx.send(());
});
Progress {
draw_stop: draw_tx,
clean_done: clean_rx,
display_handle: DisplayHandle { buf: Vec::new(), sender: msg_tx },
}
}
pub(crate) fn display_handle(&mut self) -> &mut dyn io::Write {
&mut self.display_handle
}
}
impl Drop for Progress {
fn drop(&mut self) {
let _ = self.draw_stop.send(());
let _ = self.clean_done.recv();
}
}

2
src/utils/fs.rs
View File

@ -3,7 +3,7 @@
use std::{
env,
fs::ReadDir,
io::Read,
io::{Read, Write},
path::{Path, PathBuf},
};