From 0121a0c0dfb18c5284d614186edd0710e5ebae36 Mon Sep 17 00:00:00 2001 From: oxalica Date: Wed, 2 Jul 2025 01:15:17 -0400 Subject: [PATCH] feat: add basic compressing support for squashfs --- src/archive/squashfs.rs | 220 +++++++++++++++++++++++++++++++++++++-- src/commands/compress.rs | 34 ++++-- 2 files changed, 237 insertions(+), 17 deletions(-) diff --git a/src/archive/squashfs.rs b/src/archive/squashfs.rs index c2b3465..d8953c0 100644 --- a/src/archive/squashfs.rs +++ b/src/archive/squashfs.rs @@ -1,17 +1,23 @@ use std::{ - fs, - io::{self, BufWriter, Write}, - path::Path, + env, fs, + io::{self, BufWriter, Seek, Write}, + path::{Path, PathBuf}, + time::SystemTime, }; -use backhand::{FilesystemReader, InnerNode, SquashfsFileReader}; -use filetime_creation::{set_file_handle_times, set_file_mtime, FileTime}; +use backhand::{ + compression::Compressor, FilesystemCompressor, FilesystemReader, FilesystemWriter, InnerNode, NodeHeader, + SquashfsFileReader, +}; +use filetime_creation::{set_file_handle_times, FileTime}; +use same_file::Handle; use crate::{ + error::FinalError, list::FileInArchive, utils::{ logger::{info, warning}, - Bytes, + Bytes, FileVisibilityPolicy, }, }; @@ -131,3 +137,205 @@ pub fn unpack_archive(archive: FilesystemReader<'_>, output_folder: &Path, quiet Ok(unpacked_files) } + +pub fn build_archive_from_paths( + input_filenames: &[PathBuf], + output_path: &Path, + mut writer: W, + file_visibility_policy: FileVisibilityPolicy, + quiet: bool, + follow_symlinks: bool, +) -> crate::Result +where + W: Write + Seek, +{ + let root_dir = match input_filenames { + [path] if path.is_dir() => path, + _ => { + let error = FinalError::with_title("Cannot build squashfs") + .detail("Squashfs requires a single directory input for root directory") + .detail(if input_filenames.len() != 1 { + "Multiple paths are provided".into() + } else { + format!("Not a directory: {:?}", input_filenames[0]) + }); + return Err(error.into()); + } + }; + + let output_handle = Handle::from_path(output_path); + + let mut fs_writer = FilesystemWriter::default(); + // Set the default compression to Gzip with default level, matching mksquashfs's default. + // The default choice of `backhand` is Xz which is not enabled by us. + // TODO: We do not support customization argument for archive formats. + fs_writer.set_compressor(FilesystemCompressor::new(Compressor::Gzip, None).expect("gzip is supported")); + + // cd *into* the source directory, using it as the archive root. + let previous_cwd = env::current_dir()?; + env::set_current_dir(root_dir)?; + + for entry in file_visibility_policy.build_walker(".") { + let entry = entry?; + let path = entry.path(); + + if let Ok(handle) = &output_handle { + if matches!(Handle::from_path(path), Ok(x) if &x == handle) { + warning(format!( + "Cannot compress `{}` into itself, skipping", + output_path.display() + )); + } + } + + if !quiet { + // `fs_writer.push_*` only maintains metadata. We do not want to give + // users a false information that we are compressing during the + // traversal. So this is not "compressing". + // File reading, compression and writing are done in + // `fs_writer.write` below, after the hierarchy tree gets finalized. + info(format!("Found {path:?}")); + } + + let metadata = entry.metadata()?; + let file_type = metadata.file_type(); + + let mut header = NodeHeader::default(); + header.mtime = match metadata.modified() { + // Not available. + Err(_) => 0, + Ok(mtime) => { + let mtime = mtime + .duration_since(SystemTime::UNIX_EPOCH) + .ok() + .and_then(|dur| u32::try_from(dur.as_secs()).ok()); + if mtime.is_none() { + warning(format!( + "Modification time of {path:?} exceeds the representable range (1970-01-01 ~ 2106-02-07) \ + of squashfs. Recorded as 1970-01-01." + )); + } + mtime.unwrap_or(0) + } + }; + + #[cfg(not(unix))] + { + header.permissions = 0o777; + } + + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + // Only permission bits, not file type bits. + header.permissions = metadata.permissions().mode() as u16 & !(libc::S_IFMT as u16); + header.uid = metadata.uid(); + header.gid = metadata.gid(); + } + + // Root directory is special cased. + if path == Path::new(".") { + fs_writer.set_root_mode(header.permissions); + fs_writer.set_root_uid(header.uid); + fs_writer.set_root_gid(header.gid); + continue; + } + + if file_type.is_dir() { + fs_writer.push_dir(path, header)?; + } else if !follow_symlinks && file_type.is_symlink() { + let target = fs::read_link(path)?; + fs_writer.push_symlink(target, path, header)?; + } else if maybe_push_unix_special_inode(&mut fs_writer, path, &metadata, header)? { + // Already handled. + } else { + // Fallback case: read as a regular file. + // See comments of `LazyFile` for why not `File::open` here. + let reader = LazyFile::Path(path.to_path_buf()); + fs_writer.push_file(reader, path, header)?; + } + } + + if !quiet { + info(format!("Compressing data")); + } + + // Finalize the superblock and write data. This should be done before + // resetting current directory, because `LazyFile`s store relative paths. + fs_writer.write(&mut writer)?; + + env::set_current_dir(previous_cwd)?; + + Ok(writer) +} + +#[cfg(not(unix))] +fn maybe_push_unix_special_inode( + _writer: &mut FilesystemWriter, + _path: &Path, + _metadata: &fs::Metadata, + _header: NodeHeader, +) -> io::Result { + Ok(false) +} + +#[cfg(unix)] +fn maybe_push_unix_special_inode( + writer: &mut FilesystemWriter, + path: &Path, + metadata: &fs::Metadata, + header: NodeHeader, +) -> io::Result { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + + let file_type = metadata.file_type(); + if file_type.is_fifo() { + writer.push_fifo(path, header)?; + } else if file_type.is_socket() { + writer.push_socket(path, header)?; + } else if file_type.is_block_device() { + let dev = metadata.rdev() as u32; + writer.push_block_device(dev, path, header)?; + } else if file_type.is_char_device() { + let dev = metadata.rdev() as u32; + writer.push_char_device(dev, path, header)?; + } else { + return Ok(false); + } + Ok(true) +} + +/// Delay file opening until the first read and close it as soon as the EOF is encountered. +/// +/// Due to design of `backhand`, we need to store all `impl Read` into the +/// builder during traversal and write out the squashfs later. But we cannot +/// open and store all file handles during traversal or it will exhaust file +/// descriptors on *NIX if there are thousands of files (a pretty low limit!). +/// +/// Upstream discussion: https://github.com/wcampbell0x2a/backhand/discussions/614 +enum LazyFile { + Path(PathBuf), + Opened(fs::File), + Closed, +} + +impl io::Read for LazyFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + LazyFile::Path(path) => { + let file = fs::File::open(path)?; + *self = Self::Opened(file); + self.read(buf) + } + LazyFile::Opened(file) => { + let cnt = file.read(buf)?; + if buf.len() != 0 && cnt == 0 { + *self = Self::Closed; + } + Ok(cnt) + } + LazyFile::Closed => Ok(0), + } + } +} diff --git a/src/commands/compress.rs b/src/commands/compress.rs index 1737359..0a0c488 100644 --- a/src/commands/compress.rs +++ b/src/commands/compress.rs @@ -124,29 +124,41 @@ pub fn compress_files( )?; writer.flush()?; } - Squashfs => todo!(), - Zip => { + Zip | Squashfs => { + let is_zip = matches!(first_format, Zip); + if !formats.is_empty() { // Locking necessary to guarantee that warning and question // messages stay adjacent let _locks = lock_and_flush_output_stdio(); - warn_user_about_loading_in_memory(".zip"); + warn_user_about_loading_in_memory(if is_zip { ".zip" } else { ".sqfs" }); if !user_wants_to_continue(output_path, question_policy, QuestionAction::Compression)? { return Ok(false); } } let mut vec_buffer = Cursor::new(vec![]); + if is_zip { + archive::zip::build_archive_from_paths( + &files, + output_path, + &mut vec_buffer, + file_visibility_policy, + quiet, + follow_symlinks, + )?; + } else { + archive::squashfs::build_archive_from_paths( + &files, + output_path, + &mut vec_buffer, + file_visibility_policy, + quiet, + follow_symlinks, + )?; + } - archive::zip::build_archive_from_paths( - &files, - output_path, - &mut vec_buffer, - file_visibility_policy, - quiet, - follow_symlinks, - )?; vec_buffer.rewind()?; io::copy(&mut vec_buffer, &mut writer)?; }