From 4ffb55f46a713623b2e309c1157ab213a05840cf Mon Sep 17 00:00:00 2001 From: liushuyu Date: Sat, 3 Apr 2021 05:22:01 -0600 Subject: [PATCH 1/4] builder: add support for archiving special files on Unix Added support for archiving character, block and fifo files on Unix, this is useful when you want to take a whole system backup or re-archiving an extracted system image --- src/builder.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests/all.rs | 32 ++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/src/builder.rs b/src/builder.rs index a38fe81e..6bc4efb4 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -409,6 +409,39 @@ fn append_path_with_name( mode, Some(&link_name), ) + } else { + #[cfg(unix)] + { + append_special(dst, path, &stat, mode) + } + #[cfg(not(unix))] + { + Err(other(&format!("{} has unknown file type", path.display()))) + } + } +} + +#[cfg(unix)] +fn append_special( + dst: &mut dyn Write, + path: &Path, + stat: &fs::Metadata, + mode: HeaderMode, +) -> io::Result<()> { + use ::std::os::unix::fs::FileTypeExt; + let file_type = stat.file_type(); + if file_type.is_socket() { + // sockets can't be archived + Err(other(&format!( + "{}: socket can not be archived", + path.display() + ))) + } else if file_type.is_fifo() { + append_fs_special(dst, path, &stat, EntryType::Fifo, mode) + } else if file_type.is_char_device() { + append_fs_special(dst, path, &stat, EntryType::Char, mode) + } else if file_type.is_block_device() { + append_fs_special(dst, path, &stat, EntryType::Block, mode) } else { Err(other(&format!("{} has unknown file type", path.display()))) } @@ -518,6 +551,32 @@ fn append_fs( append(dst, &header, read) } +#[cfg(unix)] +fn append_fs_special( + dst: &mut dyn Write, + path: &Path, + meta: &fs::Metadata, + entry_type: EntryType, + mode: HeaderMode, +) -> io::Result<()> { + use ::std::os::unix::fs::MetadataExt; + let mut header = Header::new_gnu(); + header.set_metadata_in_mode(meta, mode); + prepare_header_path(dst, &mut header, path)?; + + header.set_entry_type(entry_type); + let dev_id = meta.rdev(); + let dev_major = ((dev_id >> 32) & 0xffff_f000) | ((dev_id >> 8) & 0x0000_0fff); + let dev_minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); + header.set_device_major(dev_major as u32)?; + header.set_device_minor(dev_minor as u32)?; + + header.set_cksum(); + dst.write_all(header.as_bytes())?; + + Ok(()) +} + fn append_dir_all( dst: &mut dyn Write, path: &Path, @@ -543,6 +602,14 @@ fn append_dir_all( let link_name = fs::read_link(&src)?; append_fs(dst, &dest, &stat, &mut io::empty(), mode, Some(&link_name))?; } else { + #[cfg(unix)] + { + let stat = fs::metadata(&src)?; + if !stat.is_file() { + append_special(dst, &dest, &stat, mode)?; + continue; + } + } append_file(dst, &dest, &mut fs::File::open(src)?, mode)?; } } diff --git a/tests/all.rs b/tests/all.rs index 1abb9161..a527321d 100644 --- a/tests/all.rs +++ b/tests/all.rs @@ -1212,3 +1212,35 @@ fn read_only_directory_containing_files() { let mut ar = Archive::new(&contents[..]); assert!(ar.unpack(td.path()).is_ok()); } + +#[test] +#[cfg(unix)] +fn tar_directory_containing_special_files() { + use std::env; + use std::ffi::CString; + + let td = t!(TempBuilder::new().prefix("tar-rs").tempdir()); + let cdev = td.path().join("chr"); + let fifo = td.path().join("fifo"); + + unsafe { + let cdev_path = t!(CString::new(cdev.to_str().unwrap())); + let ret = libc::mknod(cdev_path.as_ptr(), libc::S_IFCHR | 0o644, 0); + assert_eq!(ret, 0); + let fifo_path = t!(CString::new(fifo.to_str().unwrap())); + let ret = libc::mknod(fifo_path.as_ptr(), libc::S_IFIFO | 0o644, 0); + assert_eq!(ret, 0); + } + + t!(env::set_current_dir(td.path())); + let mut ar = Builder::new(Vec::new()); + // append_path has a different logic for processing files, so we need to test it as well + t!(ar.append_path("chr")); + t!(ar.append_path("fifo")); + t!(ar.append_dir_all("special", td.path())); + // unfortunately, block device file cannot be created by non-root users + // as a substitute, just test the file that exists on most Unix systems + t!(env::set_current_dir("/dev/")); + t!(ar.append_path("loop0")); + t!(ar.finish()); +} From 2cb25880962a6aebc9bdce1250eca45bbdba72f2 Mon Sep 17 00:00:00 2001 From: liushuyu Date: Sat, 3 Apr 2021 05:39:55 -0600 Subject: [PATCH 2/4] tests: display system error for the special files test --- tests/all.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/all.rs b/tests/all.rs index a527321d..69712e1a 100644 --- a/tests/all.rs +++ b/tests/all.rs @@ -1220,27 +1220,27 @@ fn tar_directory_containing_special_files() { use std::ffi::CString; let td = t!(TempBuilder::new().prefix("tar-rs").tempdir()); - let cdev = td.path().join("chr"); let fifo = td.path().join("fifo"); unsafe { - let cdev_path = t!(CString::new(cdev.to_str().unwrap())); - let ret = libc::mknod(cdev_path.as_ptr(), libc::S_IFCHR | 0o644, 0); - assert_eq!(ret, 0); let fifo_path = t!(CString::new(fifo.to_str().unwrap())); let ret = libc::mknod(fifo_path.as_ptr(), libc::S_IFIFO | 0o644, 0); - assert_eq!(ret, 0); + if ret != 0 { + libc::perror(fifo_path.as_ptr()); + panic!("Failed to create a FIFO file"); + } } t!(env::set_current_dir(td.path())); let mut ar = Builder::new(Vec::new()); // append_path has a different logic for processing files, so we need to test it as well - t!(ar.append_path("chr")); t!(ar.append_path("fifo")); t!(ar.append_dir_all("special", td.path())); // unfortunately, block device file cannot be created by non-root users // as a substitute, just test the file that exists on most Unix systems t!(env::set_current_dir("/dev/")); t!(ar.append_path("loop0")); + // CI systems seem to have issues with creating a chr device + t!(ar.append_path("null")); t!(ar.finish()); } From 994033c26990e19cc6c3b5b69b6bef30382ae2e6 Mon Sep 17 00:00:00 2001 From: liushuyu Date: Thu, 22 Apr 2021 03:32:28 -0600 Subject: [PATCH 3/4] builder: address comments ... * attempt to reduce `stat` calls --- src/builder.rs | 23 ++++++++++++++++------- tests/all.rs | 5 +++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 6bc4efb4..39f56f50 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -584,15 +584,25 @@ fn append_dir_all( mode: HeaderMode, follow: bool, ) -> io::Result<()> { - let mut stack = vec![(src_path.to_path_buf(), true, false)]; - while let Some((src, is_dir, is_symlink)) = stack.pop() { + let metadata = fs::metadata(src_path)?; + if !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a directory", src_path.display()), + )); + } + let mut stack = vec![(src_path.to_path_buf(), metadata)]; + while let Some((src, metadata)) = stack.pop() { let dest = path.join(src.strip_prefix(&src_path).unwrap()); + let file_type = metadata.file_type(); + let is_dir = file_type.is_dir(); + let is_symlink = file_type.is_symlink(); // In case of a symlink pointing to a directory, is_dir is false, but src.is_dir() will return true if is_dir || (is_symlink && follow && src.is_dir()) { for entry in fs::read_dir(&src)? { let entry = entry?; - let file_type = entry.file_type()?; - stack.push((entry.path(), file_type.is_dir(), file_type.is_symlink())); + let metadata = entry.metadata()?; + stack.push((entry.path(), metadata)); } if dest != Path::new("") { append_dir(dst, &dest, &src, mode)?; @@ -604,9 +614,8 @@ fn append_dir_all( } else { #[cfg(unix)] { - let stat = fs::metadata(&src)?; - if !stat.is_file() { - append_special(dst, &dest, &stat, mode)?; + if !file_type.is_file() { + append_special(dst, &dest, &metadata, mode)?; continue; } } diff --git a/tests/all.rs b/tests/all.rs index 69712e1a..082c32cb 100644 --- a/tests/all.rs +++ b/tests/all.rs @@ -1236,6 +1236,11 @@ fn tar_directory_containing_special_files() { // append_path has a different logic for processing files, so we need to test it as well t!(ar.append_path("fifo")); t!(ar.append_dir_all("special", td.path())); + // unfortunately, macOS CI can't handle `set_current_dir` correctly + if !cfg!(target_os = "linux") { + t!(ar.finish()); + return; + } // unfortunately, block device file cannot be created by non-root users // as a substitute, just test the file that exists on most Unix systems t!(env::set_current_dir("/dev/")); From a511442ff2e58c795a001ac54d8fb68c62b308d9 Mon Sep 17 00:00:00 2001 From: liushuyu Date: Fri, 23 Apr 2021 16:43:54 -0600 Subject: [PATCH 4/4] builder: merge `append_fs_special` into `append_special` --- src/builder.rs | 81 ++++++++++++++++++++------------------------------ tests/all.rs | 8 ++--- 2 files changed, 34 insertions(+), 55 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 39f56f50..0c904474 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -428,23 +428,41 @@ fn append_special( stat: &fs::Metadata, mode: HeaderMode, ) -> io::Result<()> { - use ::std::os::unix::fs::FileTypeExt; + use ::std::os::unix::fs::{FileTypeExt, MetadataExt}; + let file_type = stat.file_type(); + let entry_type; if file_type.is_socket() { // sockets can't be archived - Err(other(&format!( + return Err(other(&format!( "{}: socket can not be archived", path.display() - ))) + ))); } else if file_type.is_fifo() { - append_fs_special(dst, path, &stat, EntryType::Fifo, mode) + entry_type = EntryType::Fifo; } else if file_type.is_char_device() { - append_fs_special(dst, path, &stat, EntryType::Char, mode) + entry_type = EntryType::Char; } else if file_type.is_block_device() { - append_fs_special(dst, path, &stat, EntryType::Block, mode) + entry_type = EntryType::Block; } else { - Err(other(&format!("{} has unknown file type", path.display()))) + return Err(other(&format!("{} has unknown file type", path.display()))); } + + let mut header = Header::new_gnu(); + header.set_metadata_in_mode(stat, mode); + prepare_header_path(dst, &mut header, path)?; + + header.set_entry_type(entry_type); + let dev_id = stat.rdev(); + let dev_major = ((dev_id >> 32) & 0xffff_f000) | ((dev_id >> 8) & 0x0000_0fff); + let dev_minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); + header.set_device_major(dev_major as u32)?; + header.set_device_minor(dev_minor as u32)?; + + header.set_cksum(); + dst.write_all(header.as_bytes())?; + + Ok(()) } fn append_file( @@ -551,32 +569,6 @@ fn append_fs( append(dst, &header, read) } -#[cfg(unix)] -fn append_fs_special( - dst: &mut dyn Write, - path: &Path, - meta: &fs::Metadata, - entry_type: EntryType, - mode: HeaderMode, -) -> io::Result<()> { - use ::std::os::unix::fs::MetadataExt; - let mut header = Header::new_gnu(); - header.set_metadata_in_mode(meta, mode); - prepare_header_path(dst, &mut header, path)?; - - header.set_entry_type(entry_type); - let dev_id = meta.rdev(); - let dev_major = ((dev_id >> 32) & 0xffff_f000) | ((dev_id >> 8) & 0x0000_0fff); - let dev_minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); - header.set_device_major(dev_major as u32)?; - header.set_device_minor(dev_minor as u32)?; - - header.set_cksum(); - dst.write_all(header.as_bytes())?; - - Ok(()) -} - fn append_dir_all( dst: &mut dyn Write, path: &Path, @@ -584,25 +576,15 @@ fn append_dir_all( mode: HeaderMode, follow: bool, ) -> io::Result<()> { - let metadata = fs::metadata(src_path)?; - if !metadata.is_dir() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("{} is not a directory", src_path.display()), - )); - } - let mut stack = vec![(src_path.to_path_buf(), metadata)]; - while let Some((src, metadata)) = stack.pop() { + let mut stack = vec![(src_path.to_path_buf(), true, false)]; + while let Some((src, is_dir, is_symlink)) = stack.pop() { let dest = path.join(src.strip_prefix(&src_path).unwrap()); - let file_type = metadata.file_type(); - let is_dir = file_type.is_dir(); - let is_symlink = file_type.is_symlink(); // In case of a symlink pointing to a directory, is_dir is false, but src.is_dir() will return true if is_dir || (is_symlink && follow && src.is_dir()) { for entry in fs::read_dir(&src)? { let entry = entry?; - let metadata = entry.metadata()?; - stack.push((entry.path(), metadata)); + let file_type = entry.file_type()?; + stack.push((entry.path(), file_type.is_dir(), file_type.is_symlink())); } if dest != Path::new("") { append_dir(dst, &dest, &src, mode)?; @@ -614,8 +596,9 @@ fn append_dir_all( } else { #[cfg(unix)] { - if !file_type.is_file() { - append_special(dst, &dest, &metadata, mode)?; + let stat = fs::metadata(&src)?; + if !stat.is_file() { + append_special(dst, &dest, &stat, mode)?; continue; } } diff --git a/tests/all.rs b/tests/all.rs index 082c32cb..d29a5190 100644 --- a/tests/all.rs +++ b/tests/all.rs @@ -1213,8 +1213,9 @@ fn read_only_directory_containing_files() { assert!(ar.unpack(td.path()).is_ok()); } +// This test was marked linux only due to macOS CI can't handle `set_current_dir` correctly #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] fn tar_directory_containing_special_files() { use std::env; use std::ffi::CString; @@ -1236,11 +1237,6 @@ fn tar_directory_containing_special_files() { // append_path has a different logic for processing files, so we need to test it as well t!(ar.append_path("fifo")); t!(ar.append_dir_all("special", td.path())); - // unfortunately, macOS CI can't handle `set_current_dir` correctly - if !cfg!(target_os = "linux") { - t!(ar.finish()); - return; - } // unfortunately, block device file cannot be created by non-root users // as a substitute, just test the file that exists on most Unix systems t!(env::set_current_dir("/dev/"));