From cde5d5ed11a91fe3217ee27ad1ac9135326ce20d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 14:33:10 -0700 Subject: [PATCH 001/281] Implement shallow copy from within the file being written --- Cargo.toml | 7 +-- README.md | 4 +- benches/read_entry.rs | 4 +- benches/read_metadata.rs | 4 +- examples/extract.rs | 2 +- examples/extract_lorem.rs | 2 +- examples/file_info.rs | 2 +- examples/stdin_info.rs | 2 +- examples/write_dir.rs | 28 +++++------ examples/write_sample.rs | 8 +-- fuzz/fuzz_targets/fuzz_read.rs | 2 +- src/lib.rs | 2 +- src/read.rs | 22 ++++---- src/result.rs | 4 +- src/write.rs | 92 ++++++++++++++++++++++++++++------ tests/aes_encryption.rs | 2 +- tests/end_to_end.rs | 28 +++++------ tests/invalid_date.rs | 2 +- tests/issue_234.rs | 4 +- tests/zip64_large.rs | 2 +- tests/zip_comment_garbage.rs | 2 +- tests/zip_crypto.rs | 8 +-- 22 files changed, 148 insertions(+), 85 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5468919a..474d09cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [package] -name = "zip" -version = "0.6.4" -authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick "] +name = "zip_next" +version = "0.6.5" +authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", +"Chris Hennick "] license = "MIT" repository = "https://github.com/zip-rs/zip.git" keywords = ["zip", "archive"] diff --git a/README.md b/README.md index d8f91b78..257ed5cd 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip = "0.6.4" +zip-next = "0.6.5" ``` Without the default features: ```toml [dependencies] -zip = { version = "0.6.4", default-features = false } +zip-next = { version = "0.6.5", default-features = false } ``` The features available are: diff --git a/benches/read_entry.rs b/benches/read_entry.rs index af9affe3..03b94fe4 100644 --- a/benches/read_entry.rs +++ b/benches/read_entry.rs @@ -4,13 +4,13 @@ use std::io::{Cursor, Read, Write}; use bencher::Bencher; use getrandom::getrandom; -use zip::{ZipArchive, ZipWriter}; +use zip_next::{ZipArchive, ZipWriter}; fn generate_random_archive(size: usize) -> Vec { let data = Vec::new(); let mut writer = ZipWriter::new(Cursor::new(data)); let options = - zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); + zip_next::write::FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); writer.start_file("random.dat", options).unwrap(); let mut bytes = vec![0u8; size]; diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index 95334b1c..295cd050 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -3,7 +3,7 @@ use bencher::{benchmark_group, benchmark_main}; use std::io::{Cursor, Write}; use bencher::Bencher; -use zip::{ZipArchive, ZipWriter}; +use zip_next::{ZipArchive, ZipWriter}; const FILE_COUNT: usize = 15_000; const FILE_SIZE: usize = 1024; @@ -12,7 +12,7 @@ fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { let data = Vec::new(); let mut writer = ZipWriter::new(Cursor::new(data)); let options = - zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); + zip_next::write::FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); let bytes = vec![0u8; file_size]; diff --git a/examples/extract.rs b/examples/extract.rs index 30807162..e5bad04c 100644 --- a/examples/extract.rs +++ b/examples/extract.rs @@ -14,7 +14,7 @@ fn real_main() -> i32 { let fname = std::path::Path::new(&*args[1]); let file = fs::File::open(fname).unwrap(); - let mut archive = zip::ZipArchive::new(file).unwrap(); + let mut archive = zip_next::ZipArchive::new(file).unwrap(); for i in 0..archive.len() { let mut file = archive.by_index(i).unwrap(); diff --git a/examples/extract_lorem.rs b/examples/extract_lorem.rs index bc50abe1..0d1705c5 100644 --- a/examples/extract_lorem.rs +++ b/examples/extract_lorem.rs @@ -13,7 +13,7 @@ fn real_main() -> i32 { let fname = std::path::Path::new(&*args[1]); let zipfile = std::fs::File::open(fname).unwrap(); - let mut archive = zip::ZipArchive::new(zipfile).unwrap(); + let mut archive = zip_next::ZipArchive::new(zipfile).unwrap(); let mut file = match archive.by_name("test/lorem_ipsum.txt") { Ok(file) => file, diff --git a/examples/file_info.rs b/examples/file_info.rs index 6a2adc58..1730eb8d 100644 --- a/examples/file_info.rs +++ b/examples/file_info.rs @@ -15,7 +15,7 @@ fn real_main() -> i32 { let file = fs::File::open(fname).unwrap(); let reader = BufReader::new(file); - let mut archive = zip::ZipArchive::new(reader).unwrap(); + let mut archive = zip_next::ZipArchive::new(reader).unwrap(); for i in 0..archive.len() { let file = archive.by_index(i).unwrap(); diff --git a/examples/stdin_info.rs b/examples/stdin_info.rs index a609916a..f49d6e73 100644 --- a/examples/stdin_info.rs +++ b/examples/stdin_info.rs @@ -10,7 +10,7 @@ fn real_main() -> i32 { let mut buf = [0u8; 16]; loop { - match zip::read::read_zipfile_from_stream(&mut stdin_handle) { + match zip_next::read::read_zipfile_from_stream(&mut stdin_handle) { Ok(Some(mut file)) => { println!( "{}: {} bytes ({} bytes packed)", diff --git a/examples/write_dir.rs b/examples/write_dir.rs index 3b043528..429e136e 100644 --- a/examples/write_dir.rs +++ b/examples/write_dir.rs @@ -1,8 +1,8 @@ use std::io::prelude::*; use std::io::{Seek, Write}; use std::iter::Iterator; -use zip::result::ZipError; -use zip::write::FileOptions; +use zip_next::result::ZipError; +use zip_next::write::FileOptions; use std::fs::File; use std::path::Path; @@ -12,30 +12,30 @@ fn main() { std::process::exit(real_main()); } -const METHOD_STORED: Option = Some(zip::CompressionMethod::Stored); +const METHOD_STORED: Option = Some(zip_next::CompressionMethod::Stored); #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] -const METHOD_DEFLATED: Option = Some(zip::CompressionMethod::Deflated); +const METHOD_DEFLATED: Option = Some(zip_next::CompressionMethod::Deflated); #[cfg(not(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" )))] -const METHOD_DEFLATED: Option = None; +const METHOD_DEFLATED: Option = None; #[cfg(feature = "bzip2")] -const METHOD_BZIP2: Option = Some(zip::CompressionMethod::Bzip2); +const METHOD_BZIP2: Option = Some(zip_next::CompressionMethod::Bzip2); #[cfg(not(feature = "bzip2"))] -const METHOD_BZIP2: Option = None; +const METHOD_BZIP2: Option = None; #[cfg(feature = "zstd")] -const METHOD_ZSTD: Option = Some(zip::CompressionMethod::Zstd); +const METHOD_ZSTD: Option = Some(zip_next::CompressionMethod::Zstd); #[cfg(not(feature = "zstd"))] -const METHOD_ZSTD: Option = None; +const METHOD_ZSTD: Option = None; fn real_main() -> i32 { let args: Vec<_> = std::env::args().collect(); @@ -66,12 +66,12 @@ fn zip_dir( it: &mut dyn Iterator, prefix: &str, writer: T, - method: zip::CompressionMethod, -) -> zip::result::ZipResult<()> + method: zip_next::CompressionMethod, +) -> zip_next::result::ZipResult<()> where T: Write + Seek, { - let mut zip = zip::ZipWriter::new(writer); + let mut zip = zip_next::ZipWriter::new(writer); let options = FileOptions::default() .compression_method(method) .unix_permissions(0o755); @@ -107,8 +107,8 @@ where fn doit( src_dir: &str, dst_file: &str, - method: zip::CompressionMethod, -) -> zip::result::ZipResult<()> { + method: zip_next::CompressionMethod, +) -> zip_next::result::ZipResult<()> { if !Path::new(src_dir).is_dir() { return Err(ZipError::FileNotFound); } diff --git a/examples/write_sample.rs b/examples/write_sample.rs index 2e45cb1e..bb9739d0 100644 --- a/examples/write_sample.rs +++ b/examples/write_sample.rs @@ -1,5 +1,5 @@ use std::io::prelude::*; -use zip::write::FileOptions; +use zip_next::write::FileOptions; fn main() { std::process::exit(real_main()); @@ -21,16 +21,16 @@ fn real_main() -> i32 { 0 } -fn doit(filename: &str) -> zip::result::ZipResult<()> { +fn doit(filename: &str) -> zip_next::result::ZipResult<()> { let path = std::path::Path::new(filename); let file = std::fs::File::create(path).unwrap(); - let mut zip = zip::ZipWriter::new(file); + let mut zip = zip_next::ZipWriter::new(file); zip.add_directory("test/", Default::default())?; let options = FileOptions::default() - .compression_method(zip::CompressionMethod::Stored) + .compression_method(zip_next::CompressionMethod::Stored) .unix_permissions(0o755); zip.start_file("test/☃.txt", options)?; zip.write_all(b"Hello, World!\n")?; diff --git a/fuzz/fuzz_targets/fuzz_read.rs b/fuzz/fuzz_targets/fuzz_read.rs index 97fbdd3b..359cf227 100644 --- a/fuzz/fuzz_targets/fuzz_read.rs +++ b/fuzz/fuzz_targets/fuzz_read.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; fn decompress_all(data: &[u8]) -> Result<(), Box> { let reader = std::io::Cursor::new(data); - let mut zip = zip::ZipArchive::new(reader)?; + let mut zip = zip_next::ZipArchive::new(reader)?; for i in 0..zip.len() { let mut file = zip.by_index(i)?; diff --git a/src/lib.rs b/src/lib.rs index 7f3e7a01..8975837d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,7 @@ //! //! //! - +#![feature(read_buf)] #![warn(missing_docs)] pub use crate::compression::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; diff --git a/src/read.rs b/src/read.rs index b702b4f2..e7400ef3 100644 --- a/src/read.rs +++ b/src/read.rs @@ -51,8 +51,8 @@ pub(crate) mod zip_archive { /// /// ```no_run /// use std::io::prelude::*; - /// fn list_zip_contents(reader: impl Read + Seek) -> zip::result::ZipResult<()> { - /// let mut zip = zip::ZipArchive::new(reader)?; + /// fn list_zip_contents(reader: impl Read + Seek) -> zip_next::result::ZipResult<()> { + /// let mut zip = zip_next::ZipArchive::new(reader)?; /// /// for i in 0..zip.len() { /// let mut file = zip.by_index(i)?; @@ -72,7 +72,7 @@ pub(crate) mod zip_archive { pub use zip_archive::ZipArchive; #[allow(clippy::large_enum_variant)] -enum CryptoReader<'a> { +pub(crate) enum CryptoReader<'a> { Plaintext(io::Take<&'a mut dyn Read>), ZipCrypto(ZipCryptoReaderValid>), #[cfg(feature = "aes-crypto")] @@ -119,7 +119,7 @@ impl<'a> CryptoReader<'a> { } } -enum ZipFileReader<'a> { +pub(crate) enum ZipFileReader<'a> { NoReader, Raw(io::Take<&'a mut dyn io::Read>), Stored(Crc32Reader>), @@ -178,12 +178,12 @@ impl<'a> ZipFileReader<'a> { /// A struct for reading a zip file pub struct ZipFile<'a> { - data: Cow<'a, ZipFileData>, - crypto_reader: Option>, - reader: ZipFileReader<'a>, + pub(crate) data: Cow<'a, ZipFileData>, + pub(crate) crypto_reader: Option>, + pub(crate) reader: ZipFileReader<'a>, } -fn find_content<'a>( +pub(crate) fn find_content<'a>( data: &ZipFileData, reader: &'a mut (impl Read + Seek), ) -> ZipResult> { @@ -206,7 +206,7 @@ fn find_content<'a>( } #[allow(clippy::too_many_arguments)] -fn make_crypto_reader<'a>( +pub(crate) fn make_crypto_reader<'a>( compression_method: crate::compression::CompressionMethod, crc32: u32, last_modified_time: DateTime, @@ -257,7 +257,7 @@ fn make_crypto_reader<'a>( Ok(Ok(reader)) } -fn make_reader( +pub(crate) fn make_reader( compression_method: CompressionMethod, crc32: u32, reader: CryptoReader, @@ -991,7 +991,7 @@ impl<'a> Drop for ZipFile<'a> { // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. let mut reader: std::io::Take<&mut dyn std::io::Read> = match &mut self.reader { ZipFileReader::NoReader => { - let innerreader = ::std::mem::replace(&mut self.crypto_reader, None); + let innerreader = self.crypto_reader.take(); innerreader.expect("Invalid reader state").into_inner() } reader => { diff --git a/src/result.rs b/src/result.rs index 00d558cb..11b10ad0 100644 --- a/src/result.rs +++ b/src/result.rs @@ -65,8 +65,8 @@ impl ZipError { /// The text used as an error when a password is required and not supplied /// /// ```rust,no_run - /// # use zip::result::ZipError; - /// # let mut archive = zip::ZipArchive::new(std::io::Cursor::new(&[])).unwrap(); + /// # use zip_next::result::ZipError; + /// # let mut archive = zip_next::ZipArchive::new(std::io::Cursor::new(&[])).unwrap(); /// match archive.by_index(1) { /// Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) => eprintln!("a password is needed to unzip this file"), /// _ => (), diff --git a/src/write.rs b/src/write.rs index 3f41c4d6..14f73658 100644 --- a/src/write.rs +++ b/src/write.rs @@ -52,17 +52,17 @@ pub(crate) mod zip_writer { /// API to edit its contents. /// /// ``` - /// # fn doit() -> zip::result::ZipResult<()> + /// # fn doit() -> zip_next::result::ZipResult<()> /// # { - /// # use zip::ZipWriter; + /// # use zip_next::ZipWriter; /// use std::io::Write; - /// use zip::write::FileOptions; + /// use zip_next::write::FileOptions; /// /// // We use a buffer here, though you'd normally use a `File` /// let mut buf = [0; 65536]; - /// let mut zip = zip::ZipWriter::new(std::io::Cursor::new(&mut buf[..])); + /// let mut zip = zip_next::ZipWriter::new(std::io::Cursor::new(&mut buf[..])); /// - /// let options = zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); + /// let options = zip_next::write::FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); /// zip.start_file("hello_world.txt", options)?; /// zip.write(b"Hello, World!")?; /// @@ -82,7 +82,7 @@ pub(crate) mod zip_writer { pub(super) writing_to_extra_field: bool, pub(super) writing_to_central_extra_field_only: bool, pub(super) writing_raw: bool, - pub(super) comment: Vec, + pub(super) comment: Vec } } pub use zip_writer::ZipWriter; @@ -291,7 +291,7 @@ impl ZipWriter { writing_to_extra_field: false, writing_to_central_extra_field_only: false, comment: footer.zip_file_comment, - writing_raw: true, // avoid recomputing the last file's header + writing_raw: true // avoid recomputing the last file's header }) } } @@ -309,7 +309,7 @@ impl ZipWriter { writing_to_extra_field: false, writing_to_central_extra_field_only: false, writing_raw: false, - comment: Vec::new(), + comment: Vec::new() } } @@ -493,8 +493,8 @@ impl ZipWriter { /// /// ``` /// use byteorder::{LittleEndian, WriteBytesExt}; - /// use zip::{ZipArchive, ZipWriter, result::ZipResult}; - /// use zip::{write::FileOptions, CompressionMethod}; + /// use zip_next::{ZipArchive, ZipWriter, result::ZipResult}; + /// use zip_next::{write::FileOptions, CompressionMethod}; /// use std::io::{Write, Cursor}; /// /// # fn main() -> ZipResult<()> { @@ -623,12 +623,12 @@ impl ZipWriter { /// ```no_run /// use std::fs::File; /// use std::io::{Read, Seek, Write}; - /// use zip::{ZipArchive, ZipWriter}; + /// use zip_next::{ZipArchive, ZipWriter}; /// /// fn copy_rename( /// src: &mut ZipArchive, /// dst: &mut ZipWriter, - /// ) -> zip::result::ZipResult<()> + /// ) -> zip_next::result::ZipResult<()> /// where /// R: Read + Seek, /// W: Write + Seek, @@ -676,9 +676,9 @@ impl ZipWriter { /// ```no_run /// use std::fs::File; /// use std::io::{Read, Seek, Write}; - /// use zip::{ZipArchive, ZipWriter}; + /// use zip_next::{ZipArchive, ZipWriter}; /// - /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip_next::result::ZipResult<()> /// where /// R: Read + Seek, /// W: Write + Seek, @@ -841,6 +841,32 @@ impl ZipWriter { } } +impl ZipWriter { + fn data_by_name(&mut self, name: &str) -> ZipResult<&ZipFileData> { + self.finish_file()?; + for file in self.files.iter() { + if file.file_name == name { + return Ok(file); + } + } + Err(ZipError::FileNotFound) + } + + /// Adds another entry to the central directory referring to the same content as an existing + /// entry. The file's local-file header will still refer to it by its original name, so + /// unzipping the file will technically be unspecified behavior. However, both [ZipArchive] and + /// OpenJDK ignore the filename in the local-file header and treat the central directory as + /// authoritative. + pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + let src_data = self.data_by_name(src_name)?; + let mut dest_data = src_data.to_owned(); + dest_data.file_name = dest_name.into(); + self.files.push(dest_data); + Ok(()) + } +} + impl Drop for ZipWriter { fn drop(&mut self) { if !self.inner.is_closed() { @@ -1309,7 +1335,8 @@ mod test { use crate::compression::CompressionMethod; use crate::types::DateTime; use std::io; - use std::io::Write; + use std::io::{Read, Write}; + use crate::ZipArchive; #[test] fn write_empty_zip() { @@ -1441,6 +1468,41 @@ mod test { assert_eq!(result.get_ref(), &v); } + #[cfg(test)] + const RT_TEST_TEXT: &str = "And I can't stop thinking about the moments that I lost to you\ + And I can't stop thinking of things I used to do\ + And I can't stop making bad decisions\ + And I can't stop eating stuff you make me chew\ + I put on a smile like you wanna see\ + Another day goes by that I long to be like you"; + #[cfg(test)] const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; + #[cfg(test)] const SECOND_FILENAME: &str = "different_name.xyz"; + + #[test] + fn test_shallow_copy() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::Deflated, + compression_level: Some(9), + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write(RT_TEST_TEXT.as_ref()).unwrap(); + writer.shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME).unwrap(); + let zip = writer.finish().unwrap(); + let mut reader = ZipArchive::new(zip).unwrap(); + let file_names: Vec<&str> = reader.file_names().collect(); + assert_eq!(file_names, vec![RT_TEST_FILENAME, SECOND_FILENAME]); + let mut first_file_content = String::new(); + reader.by_name(RT_TEST_FILENAME).unwrap().read_to_string(&mut first_file_content).unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader.by_name(SECOND_FILENAME).unwrap().read_to_string(&mut second_file_content).unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + #[test] fn path_to_string() { let mut path = std::path::PathBuf::new(); diff --git a/tests/aes_encryption.rs b/tests/aes_encryption.rs index 4b393ebf..eaad6888 100644 --- a/tests/aes_encryption.rs +++ b/tests/aes_encryption.rs @@ -1,7 +1,7 @@ #![cfg(feature = "aes-crypto")] use std::io::{self, Read}; -use zip::ZipArchive; +use zip_next::ZipArchive; const SECRET_CONTENT: &str = "Lorem ipsum dolor sit amet"; diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 09e7ce47..b5dedf32 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -3,8 +3,8 @@ use std::collections::HashSet; use std::io::prelude::*; use std::io::{Cursor, Seek}; use std::iter::FromIterator; -use zip::write::FileOptions; -use zip::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; +use zip_next::write::FileOptions; +use zip_next::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; // This test asserts that after creating a zip file, then reading its contents back out, // the extracted data will *always* be exactly the same as the original data. @@ -32,8 +32,8 @@ fn copy() { let mut tgt_file = &mut Cursor::new(Vec::new()); { - let mut src_archive = zip::ZipArchive::new(src_file).unwrap(); - let mut zip = zip::ZipWriter::new(&mut tgt_file); + let mut src_archive = zip_next::ZipArchive::new(src_file).unwrap(); + let mut zip = zip_next::ZipWriter::new(&mut tgt_file); { let file = src_archive @@ -53,7 +53,7 @@ fn copy() { } } - let mut tgt_archive = zip::ZipArchive::new(tgt_file).unwrap(); + let mut tgt_archive = zip_next::ZipArchive::new(tgt_file).unwrap(); check_archive_file_contents(&mut tgt_archive, ENTRY_NAME, LOREM_IPSUM); check_archive_file_contents(&mut tgt_archive, COPY_ENTRY_NAME, LOREM_IPSUM); @@ -69,7 +69,7 @@ fn append() { write_test_archive(file, method).expect("Couldn't write to test file"); { - let mut zip = zip::ZipWriter::new_append(&mut file).unwrap(); + let mut zip = zip_next::ZipWriter::new_append(&mut file).unwrap(); zip.start_file( COPY_ENTRY_NAME, FileOptions::default().compression_method(method), @@ -79,7 +79,7 @@ fn append() { zip.finish().unwrap(); } - let mut zip = zip::ZipArchive::new(&mut file).unwrap(); + let mut zip = zip_next::ZipArchive::new(&mut file).unwrap(); check_archive_file_contents(&mut zip, ENTRY_NAME, LOREM_IPSUM); check_archive_file_contents(&mut zip, COPY_ENTRY_NAME, LOREM_IPSUM); } @@ -89,8 +89,8 @@ fn append() { fn write_test_archive( file: &mut Cursor>, method: CompressionMethod, -) -> zip::result::ZipResult<()> { - let mut zip = zip::ZipWriter::new(file); +) -> zip_next::result::ZipResult<()> { + let mut zip = zip_next::ZipWriter::new(file); zip.add_directory("test/", Default::default())?; @@ -116,8 +116,8 @@ fn write_test_archive( } // Load an archive from buffer and check for test data. -fn check_test_archive(zip_file: R) -> zip::result::ZipResult> { - let mut archive = zip::ZipArchive::new(zip_file).unwrap(); +fn check_test_archive(zip_file: R) -> zip_next::result::ZipResult> { + let mut archive = zip_next::ZipArchive::new(zip_file).unwrap(); // Check archive contains expected file names. { @@ -147,9 +147,9 @@ fn check_test_archive(zip_file: R) -> zip::result::ZipResult( - archive: &mut zip::ZipArchive, + archive: &mut zip_next::ZipArchive, name: &str, -) -> zip::result::ZipResult { +) -> zip_next::result::ZipResult { let mut file = archive.by_name(name)?; let mut contents = String::new(); @@ -183,7 +183,7 @@ fn check_archive_file( // Check a file in the archive contains the given data. fn check_archive_file_contents( - archive: &mut zip::ZipArchive, + archive: &mut zip_next::ZipArchive, name: &str, expected: &[u8], ) { diff --git a/tests/invalid_date.rs b/tests/invalid_date.rs index 3f24e251..b684bf6a 100644 --- a/tests/invalid_date.rs +++ b/tests/invalid_date.rs @@ -1,5 +1,5 @@ use std::io::Cursor; -use zip::read::ZipArchive; +use zip_next::read::ZipArchive; const BUF: &[u8] = &[ 0x50, 0x4b, 0x03, 0x04, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, diff --git a/tests/issue_234.rs b/tests/issue_234.rs index f8c1d2c8..48e5d668 100644 --- a/tests/issue_234.rs +++ b/tests/issue_234.rs @@ -1,4 +1,4 @@ -use zip::result::ZipError; +use zip_next::result::ZipError; const BUF: &[u8] = &[ 0, 80, 75, 1, 2, 127, 120, 0, 3, 3, 75, 80, 232, 3, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 7, 0, 0, 0, @@ -23,7 +23,7 @@ const BUF: &[u8] = &[ #[test] fn invalid_header() { let reader = std::io::Cursor::new(&BUF); - let archive = zip::ZipArchive::new(reader); + let archive = zip_next::ZipArchive::new(reader); match archive { Err(ZipError::InvalidArchive(_)) => {} value => panic!("Unexpected value: {value:?}"), diff --git a/tests/zip64_large.rs b/tests/zip64_large.rs index 468ef198..7cbdd1b5 100644 --- a/tests/zip64_large.rs +++ b/tests/zip64_large.rs @@ -190,7 +190,7 @@ impl Read for Zip64File { #[test] fn zip64_large() { let zipfile = Zip64File::new(); - let mut archive = zip::ZipArchive::new(zipfile).unwrap(); + let mut archive = zip_next::ZipArchive::new(zipfile).unwrap(); let mut buf = [0u8; 32]; for i in 0..archive.len() { diff --git a/tests/zip_comment_garbage.rs b/tests/zip_comment_garbage.rs index ef4d9750..73702a08 100644 --- a/tests/zip_comment_garbage.rs +++ b/tests/zip_comment_garbage.rs @@ -18,7 +18,7 @@ // 0000002e use std::io; -use zip::ZipArchive; +use zip_next::ZipArchive; #[test] fn correctly_handle_zip_with_garbage_after_comment() { diff --git a/tests/zip_crypto.rs b/tests/zip_crypto.rs index 6c4d6b81..3251e4a6 100644 --- a/tests/zip_crypto.rs +++ b/tests/zip_crypto.rs @@ -39,7 +39,7 @@ fn encrypted_file() { 0x00, 0x00, ]); - let mut archive = zip::ZipArchive::new(zip_file_bytes).unwrap(); + let mut archive = zip_next::ZipArchive::new(zip_file_bytes).unwrap(); assert_eq!(archive.len(), 1); //Only one file inside archive: `test.txt` @@ -47,8 +47,8 @@ fn encrypted_file() { // No password let file = archive.by_index(0); match file { - Err(zip::result::ZipError::UnsupportedArchive( - zip::result::ZipError::PASSWORD_REQUIRED, + Err(zip_next::result::ZipError::UnsupportedArchive( + zip_next::result::ZipError::PASSWORD_REQUIRED, )) => (), Err(_) => panic!( "Expected PasswordRequired error when opening encrypted file without password" @@ -61,7 +61,7 @@ fn encrypted_file() { // Wrong password let file = archive.by_index_decrypt(0, b"wrong password"); match file { - Ok(Err(zip::result::InvalidPassword)) => (), + Ok(Err(zip_next::result::InvalidPassword)) => (), Err(_) => panic!( "Expected InvalidPassword error when opening encrypted file with wrong password" ), From 3af27ad353fb92ba472731ac3c83a745a2d2a881 Mon Sep 17 00:00:00 2001 From: "mend-bolt-for-github[bot]" <42819689+mend-bolt-for-github[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 21:33:22 +0000 Subject: [PATCH 002/281] Add .whitesource configuration file --- .whitesource | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .whitesource diff --git a/.whitesource b/.whitesource new file mode 100644 index 00000000..9c7ae90b --- /dev/null +++ b/.whitesource @@ -0,0 +1,14 @@ +{ + "scanSettings": { + "baseBranches": [] + }, + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure", + "displayMode": "diff", + "useMendCheckNames": true + }, + "issueSettings": { + "minSeverityLevel": "LOW", + "issueType": "DEPENDENCY" + } +} \ No newline at end of file From d1740afc4e99e07f62a76902ff2928f5a9845b0e Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Sun, 23 Apr 2023 14:43:17 -0700 Subject: [PATCH 003/281] Update links and Cargo instructions in README --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 257ed5cd..9a8c8927 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ -zip-rs -====== +zip_next +======== -[![Build Status](https://img.shields.io/github/workflow/status/zip-rs/zip/CI)](https://github.com/zip-rs/zip/actions?query=branch%3Amaster+workflow%3ACI) -[![Crates.io version](https://img.shields.io/crates/v/zip.svg)](https://crates.io/crates/zip) -[![Discord](https://badgen.net/badge/icon/discord?icon=discord&label)](https://discord.gg/rQ7H9cSsF4) +[![Build Status](https://img.shields.io/github/workflow/status/Pr0methean/zip-next/CI)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) +[![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip/0.6.3/zip/) +[Documentation](https://docs.rs/zip_next/0.6.5/zip_next/) Info ---- @@ -32,14 +31,14 @@ With all default features: ```toml [dependencies] -zip-next = "0.6.5" +zip_next = "0.6.5" ``` Without the default features: ```toml [dependencies] -zip-next = { version = "0.6.5", default-features = false } +zip_next = { version = "0.6.5", default-features = false } ``` The features available are: From 2aa4665f42bb68ff9a2b84597000841089c90d16 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Sun, 23 Apr 2023 14:46:39 -0700 Subject: [PATCH 004/281] Create rust-clippy.yml --- .github/workflows/rust-clippy.yml | 55 +++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/rust-clippy.yml diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml new file mode 100644 index 00000000..754b2566 --- /dev/null +++ b/.github/workflows/rust-clippy.yml @@ -0,0 +1,55 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# rust-clippy is a tool that runs a bunch of lints to catch common +# mistakes in your Rust code and help improve your Rust code. +# More details at https://github.com/rust-lang/rust-clippy +# and https://rust-lang.github.io/rust-clippy/ + +name: rust-clippy analyze + +on: + push: + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + schedule: + - cron: '26 1 * * 0' + +jobs: + rust-clippy-analyze: + name: Run rust-clippy analyzing + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af #@v1 + with: + profile: minimal + toolchain: stable + components: clippy + override: true + + - name: Install required cargo + run: cargo install clippy-sarif sarif-fmt + + - name: Run rust-clippy + run: + cargo clippy + --all-features + --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt + continue-on-error: true + + - name: Upload analysis results to GitHub + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: rust-clippy-results.sarif + wait-for-processing: true From de638786f18597272a2d81f25df632462fb7bd61 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Sun, 23 Apr 2023 14:48:06 -0700 Subject: [PATCH 005/281] Create dependabot_automation.yml --- .github/workflows/dependabot_automation.yml | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/dependabot_automation.yml diff --git a/.github/workflows/dependabot_automation.yml b/.github/workflows/dependabot_automation.yml new file mode 100644 index 00000000..4104bafd --- /dev/null +++ b/.github/workflows/dependabot_automation.yml @@ -0,0 +1,27 @@ +name: Dependabot auto-approve and auto-merge +on: pull_request + +permissions: + contents: write + pull-requests: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1.4.0 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Approve + run: gh pr review --approve "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + - name: Enable auto-merge + run: gh pr merge --auto --merge "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} From 360a7de003fcdee1d9d15d78f6e9f60f430ee164 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 14:51:17 -0700 Subject: [PATCH 006/281] Fix clippy warning: use write_all instead of write --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 14f73658..a6c201a6 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1489,7 +1489,7 @@ mod test { large_file: false, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); - writer.write(RT_TEST_TEXT.as_ref()).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); writer.shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME).unwrap(); let zip = writer.finish().unwrap(); let mut reader = ZipArchive::new(zip).unwrap(); From 7e7483ef5c9a569f4ee048a67a344d1f10cd3eac Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 14:52:14 -0700 Subject: [PATCH 007/281] Remove unnecessary package qualifiers --- src/write.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/write.rs b/src/write.rs index a6c201a6..9375773c 100644 --- a/src/write.rs +++ b/src/write.rs @@ -29,7 +29,7 @@ use time::OffsetDateTime; #[cfg(feature = "zstd")] use zstd::stream::write::Encoder as ZstdEncoder; -enum GenericZipWriter { +enum GenericZipWriter { Closed, Storer(W), #[cfg(any( @@ -60,9 +60,9 @@ pub(crate) mod zip_writer { /// /// // We use a buffer here, though you'd normally use a `File` /// let mut buf = [0; 65536]; - /// let mut zip = zip_next::ZipWriter::new(std::io::Cursor::new(&mut buf[..])); + /// let mut zip = ZipWriter::new(std::io::Cursor::new(&mut buf[..])); /// - /// let options = zip_next::write::FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); + /// let options = FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); /// zip.start_file("hello_world.txt", options)?; /// zip.write(b"Hello, World!")?; /// @@ -74,7 +74,7 @@ pub(crate) mod zip_writer { /// # } /// # doit().unwrap(); /// ``` - pub struct ZipWriter { + pub struct ZipWriter { pub(super) inner: GenericZipWriter, pub(super) files: Vec, pub(super) stats: ZipWriterStats, @@ -200,7 +200,7 @@ impl Default for FileOptions { } } -impl Write for ZipWriter { +impl Write for ZipWriter { fn write(&mut self, buf: &[u8]) -> io::Result { if !self.writing_to_file { return Err(io::Error::new( @@ -254,7 +254,7 @@ impl ZipWriterStats { } } -impl ZipWriter { +impl ZipWriter { /// Initializes the archive from an existing ZIP archive, making it ready for append. pub fn new_append(mut readwriter: A) -> ZipResult> { let (footer, cde_start_pos) = spec::CentralDirectoryEnd::find_and_parse(&mut readwriter)?; @@ -296,7 +296,7 @@ impl ZipWriter { } } -impl ZipWriter { +impl ZipWriter { /// Initializes the archive. /// /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. @@ -419,7 +419,7 @@ impl ZipWriter { /// Create a file in the archive and start writing its' contents. /// - /// The data should be written using the [`io::Write`] implementation on this [`ZipWriter`] + /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] pub fn start_file(&mut self, name: S, mut options: FileOptions) -> ZipResult<()> where S: Into, @@ -455,7 +455,7 @@ impl ZipWriter { /// /// Returns the number of padding bytes required to align the file. /// - /// The data should be written using the [`io::Write`] implementation on this [`ZipWriter`] + /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] pub fn start_file_aligned( &mut self, name: S, @@ -489,7 +489,7 @@ impl ZipWriter { /// Returns the preliminary starting offset of the file data without any extra data allowing to /// align the file data by calculating a pad length to be prepended as part of the extra data. /// - /// The data should be written using the [`io::Write`] implementation on this [`ZipWriter`] + /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] /// /// ``` /// use byteorder::{LittleEndian, WriteBytesExt}; @@ -841,7 +841,7 @@ impl ZipWriter { } } -impl ZipWriter { +impl ZipWriter { fn data_by_name(&mut self, name: &str) -> ZipResult<&ZipFileData> { self.finish_file()?; for file in self.files.iter() { @@ -867,7 +867,7 @@ impl ZipWriter { } } -impl Drop for ZipWriter { +impl Drop for ZipWriter { fn drop(&mut self) { if !self.inner.is_closed() { if let Err(e) = self.finalize() { @@ -877,7 +877,7 @@ impl Drop for ZipWriter { } } -impl GenericZipWriter { +impl GenericZipWriter { fn switch_to( &mut self, compression: CompressionMethod, @@ -1117,7 +1117,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR Ok(()) } -fn update_local_file_header( +fn update_local_file_header( writer: &mut T, file: &ZipFileData, ) -> ZipResult<()> { @@ -1265,7 +1265,7 @@ fn write_local_zip64_extra_field(writer: &mut T, file: &ZipFileData) - Ok(()) } -fn update_local_zip64_extra_field( +fn update_local_zip64_extra_field( writer: &mut T, file: &ZipFileData, ) -> ZipResult<()> { From 5e5f5ebe54b14290cd3df9e5976c87f0e5fbdb22 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 14:54:44 -0700 Subject: [PATCH 008/281] Bump minimum version to 1.66.0 --- .github/workflows/ci.yaml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 35d4a6e9..5312ab85 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - rust: [stable, 1.59.0] + rust: [stable, 1.66.0] steps: - uses: actions/checkout@master diff --git a/README.md b/README.md index 9a8c8927..65938642 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ All of these are enabled by default. MSRV ---- -Our current Minimum Supported Rust Version is **1.59.0**. When adding features, +Our current Minimum Supported Rust Version is **1.66.0**. When adding features, we will follow these guidelines: - We will always support the latest four minor Rust versions. This gives you a 6 From d3400509bcb26113a13ddf502ec9359c299b73b6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 14:58:10 -0700 Subject: [PATCH 009/281] Fix formatting issues from `cargo fmt` --- benches/read_entry.rs | 4 ++-- benches/read_metadata.rs | 3 ++- examples/write_dir.rs | 8 +++++--- src/write.rs | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/benches/read_entry.rs b/benches/read_entry.rs index 03b94fe4..4ee20b02 100644 --- a/benches/read_entry.rs +++ b/benches/read_entry.rs @@ -9,8 +9,8 @@ use zip_next::{ZipArchive, ZipWriter}; fn generate_random_archive(size: usize) -> Vec { let data = Vec::new(); let mut writer = ZipWriter::new(Cursor::new(data)); - let options = - zip_next::write::FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); + let options = zip_next::write::FileOptions::default() + .compression_method(zip_next::CompressionMethod::Stored); writer.start_file("random.dat", options).unwrap(); let mut bytes = vec![0u8; size]; diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index 295cd050..7325d852 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -12,7 +12,8 @@ fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { let data = Vec::new(); let mut writer = ZipWriter::new(Cursor::new(data)); let options = - zip_next::write::FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); + zip_next::write::FileOptions::default() + .compression_method(zip_next::CompressionMethod::Stored); let bytes = vec![0u8; file_size]; diff --git a/examples/write_dir.rs b/examples/write_dir.rs index 429e136e..77136703 100644 --- a/examples/write_dir.rs +++ b/examples/write_dir.rs @@ -12,14 +12,16 @@ fn main() { std::process::exit(real_main()); } -const METHOD_STORED: Option = Some(zip_next::CompressionMethod::Stored); +const METHOD_STORED: Option = + Some(zip_next::CompressionMethod::Stored); #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] -const METHOD_DEFLATED: Option = Some(zip_next::CompressionMethod::Deflated); +const METHOD_DEFLATED: Option = + Some(zip_next::CompressionMethod::Deflated); #[cfg(not(any( feature = "deflate", feature = "deflate-miniz", @@ -101,7 +103,7 @@ where } } zip.finish()?; - Result::Ok(()) + Ok(()) } fn doit( diff --git a/src/write.rs b/src/write.rs index 9375773c..fa06bf17 100644 --- a/src/write.rs +++ b/src/write.rs @@ -82,7 +82,7 @@ pub(crate) mod zip_writer { pub(super) writing_to_extra_field: bool, pub(super) writing_to_central_extra_field_only: bool, pub(super) writing_raw: bool, - pub(super) comment: Vec + pub(super) comment: Vec, } } pub use zip_writer::ZipWriter; @@ -291,7 +291,7 @@ impl ZipWriter { writing_to_extra_field: false, writing_to_central_extra_field_only: false, comment: footer.zip_file_comment, - writing_raw: true // avoid recomputing the last file's header + writing_raw: true, // avoid recomputing the last file's header }) } } @@ -309,7 +309,7 @@ impl ZipWriter { writing_to_extra_field: false, writing_to_central_extra_field_only: false, writing_raw: false, - comment: Vec::new() + comment: Vec::new(), } } From 2f1d73851c2bae49580cc2fb2492e613087e6f0c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:03:17 -0700 Subject: [PATCH 010/281] Fix fuzz dependency --- fuzz/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index bfdb764c..4349eb08 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -[dependencies.zip] +[dependencies.zip_next] path = ".." # Prevent this from interfering with workspaces From 14c61e0b974b2524c5ca9fdede5871d07fc856ce Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:04:08 -0700 Subject: [PATCH 011/281] Revert "Create rust-clippy.yml" This reverts commit 2aa4665f42bb68ff9a2b84597000841089c90d16. --- .github/workflows/rust-clippy.yml | 55 ------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 .github/workflows/rust-clippy.yml diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml deleted file mode 100644 index 754b2566..00000000 --- a/.github/workflows/rust-clippy.yml +++ /dev/null @@ -1,55 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# rust-clippy is a tool that runs a bunch of lints to catch common -# mistakes in your Rust code and help improve your Rust code. -# More details at https://github.com/rust-lang/rust-clippy -# and https://rust-lang.github.io/rust-clippy/ - -name: rust-clippy analyze - -on: - push: - branches: [ "master" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "master" ] - schedule: - - cron: '26 1 * * 0' - -jobs: - rust-clippy-analyze: - name: Run rust-clippy analyzing - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Install Rust toolchain - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af #@v1 - with: - profile: minimal - toolchain: stable - components: clippy - override: true - - - name: Install required cargo - run: cargo install clippy-sarif sarif-fmt - - - name: Run rust-clippy - run: - cargo clippy - --all-features - --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt - continue-on-error: true - - - name: Upload analysis results to GitHub - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: rust-clippy-results.sarif - wait-for-processing: true From 85689cd2e63a7ea25d11bbec57028b8536c47375 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:05:53 -0700 Subject: [PATCH 012/281] Remove unused `#![feature(read_buf)]` --- src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 8975837d..9a50fea5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,6 @@ //! //! //! -#![feature(read_buf)] #![warn(missing_docs)] pub use crate::compression::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; From 9330bdb7b7e044e36c7f20e2c0f536a6350ebbe2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:09:22 -0700 Subject: [PATCH 013/281] Don't need Read for shallow copy --- src/write.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index fa06bf17..f91bd946 100644 --- a/src/write.rs +++ b/src/write.rs @@ -839,9 +839,7 @@ impl ZipWriter { Ok(()) } -} -impl ZipWriter { fn data_by_name(&mut self, name: &str) -> ZipResult<&ZipFileData> { self.finish_file()?; for file in self.files.iter() { From 6dc099d12883c9b51596efeceef85aaefd166171 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:12:56 -0700 Subject: [PATCH 014/281] Fix more formatting issues --- benches/read_metadata.rs | 6 +++--- src/write.rs | 29 +++++++++++++++++++---------- tests/end_to_end.rs | 15 ++++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index 7325d852..f7080390 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -4,6 +4,7 @@ use std::io::{Cursor, Write}; use bencher::Bencher; use zip_next::{ZipArchive, ZipWriter}; +use zip_next::write::FileOptions; const FILE_COUNT: usize = 15_000; const FILE_SIZE: usize = 1024; @@ -11,9 +12,8 @@ const FILE_SIZE: usize = 1024; fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { let data = Vec::new(); let mut writer = ZipWriter::new(Cursor::new(data)); - let options = - zip_next::write::FileOptions::default() - .compression_method(zip_next::CompressionMethod::Stored); + let options = FileOptions::default() + .compression_method(zip_next::CompressionMethod::Stored); let bytes = vec![0u8; file_size]; diff --git a/src/write.rs b/src/write.rs index f91bd946..da1cc86f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1115,10 +1115,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR Ok(()) } -fn update_local_file_header( - writer: &mut T, - file: &ZipFileData, -) -> ZipResult<()> { +fn update_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { const CRC32_OFFSET: u64 = 14; writer.seek(io::SeekFrom::Start(file.header_start + CRC32_OFFSET))?; writer.write_u32::(file.crc32)?; @@ -1332,9 +1329,9 @@ mod test { use super::{FileOptions, ZipWriter}; use crate::compression::CompressionMethod; use crate::types::DateTime; + use crate::ZipArchive; use std::io; use std::io::{Read, Write}; - use crate::ZipArchive; #[test] fn write_empty_zip() { @@ -1473,8 +1470,10 @@ mod test { And I can't stop eating stuff you make me chew\ I put on a smile like you wanna see\ Another day goes by that I long to be like you"; - #[cfg(test)] const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; - #[cfg(test)] const SECOND_FILENAME: &str = "different_name.xyz"; + #[cfg(test)] + const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; + #[cfg(test)] + const SECOND_FILENAME: &str = "different_name.xyz"; #[test] fn test_shallow_copy() { @@ -1488,16 +1487,26 @@ mod test { }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); - writer.shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME).unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); let zip = writer.finish().unwrap(); let mut reader = ZipArchive::new(zip).unwrap(); let file_names: Vec<&str> = reader.file_names().collect(); assert_eq!(file_names, vec![RT_TEST_FILENAME, SECOND_FILENAME]); let mut first_file_content = String::new(); - reader.by_name(RT_TEST_FILENAME).unwrap().read_to_string(&mut first_file_content).unwrap(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); assert_eq!(first_file_content, RT_TEST_TEXT); let mut second_file_content = String::new(); - reader.by_name(SECOND_FILENAME).unwrap().read_to_string(&mut second_file_content).unwrap(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); assert_eq!(second_file_content, RT_TEST_TEXT); } diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index b5dedf32..6f2b26c7 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -4,7 +4,8 @@ use std::io::prelude::*; use std::io::{Cursor, Seek}; use std::iter::FromIterator; use zip_next::write::FileOptions; -use zip_next::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; +use zip_next::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS, ZipWriter}; +use zip_next::result::ZipResult; // This test asserts that after creating a zip file, then reading its contents back out, // the extracted data will *always* be exactly the same as the original data. @@ -33,7 +34,7 @@ fn copy() { { let mut src_archive = zip_next::ZipArchive::new(src_file).unwrap(); - let mut zip = zip_next::ZipWriter::new(&mut tgt_file); + let mut zip = ZipWriter::new(&mut tgt_file); { let file = src_archive @@ -69,7 +70,7 @@ fn append() { write_test_archive(file, method).expect("Couldn't write to test file"); { - let mut zip = zip_next::ZipWriter::new_append(&mut file).unwrap(); + let mut zip = ZipWriter::new_append(&mut file).unwrap(); zip.start_file( COPY_ENTRY_NAME, FileOptions::default().compression_method(method), @@ -89,8 +90,8 @@ fn append() { fn write_test_archive( file: &mut Cursor>, method: CompressionMethod, -) -> zip_next::result::ZipResult<()> { - let mut zip = zip_next::ZipWriter::new(file); +) -> ZipResult<()> { + let mut zip = ZipWriter::new(file); zip.add_directory("test/", Default::default())?; @@ -116,7 +117,7 @@ fn write_test_archive( } // Load an archive from buffer and check for test data. -fn check_test_archive(zip_file: R) -> zip_next::result::ZipResult> { +fn check_test_archive(zip_file: R) -> ZipResult> { let mut archive = zip_next::ZipArchive::new(zip_file).unwrap(); // Check archive contains expected file names. @@ -149,7 +150,7 @@ fn check_test_archive(zip_file: R) -> zip_next::result::ZipResul fn read_archive_file( archive: &mut zip_next::ZipArchive, name: &str, -) -> zip_next::result::ZipResult { +) -> ZipResult { let mut file = archive.by_name(name)?; let mut contents = String::new(); From 3b3b63cef5444b8991c1cb906bb8fbb2fe539f05 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:14:44 -0700 Subject: [PATCH 015/281] Fix a flaky test --- src/write.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index da1cc86f..ac07a49c 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1492,8 +1492,11 @@ mod test { .unwrap(); let zip = writer.finish().unwrap(); let mut reader = ZipArchive::new(zip).unwrap(); - let file_names: Vec<&str> = reader.file_names().collect(); - assert_eq!(file_names, vec![RT_TEST_FILENAME, SECOND_FILENAME]); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); let mut first_file_content = String::new(); reader .by_name(RT_TEST_FILENAME) From 911d67a795daef2483b2ea141973234892b8c868 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:15:45 -0700 Subject: [PATCH 016/281] Fix another formatting issue --- tests/end_to_end.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 6f2b26c7..4f4b8744 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -87,10 +87,7 @@ fn append() { } // Write a test zip archive to buffer. -fn write_test_archive( - file: &mut Cursor>, - method: CompressionMethod, -) -> ZipResult<()> { +fn write_test_archive(file: &mut Cursor>, method: CompressionMethod) -> ZipResult<()> { let mut zip = ZipWriter::new(file); zip.add_directory("test/", Default::default())?; From 06b5ceaef9e50d23d6b766d03d45d0669a76f1fa Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:19:19 -0700 Subject: [PATCH 017/281] Fix another formatting issue --- benches/read_metadata.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index f7080390..f7664772 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -3,7 +3,7 @@ use bencher::{benchmark_group, benchmark_main}; use std::io::{Cursor, Write}; use bencher::Bencher; -use zip_next::{ZipArchive, ZipWriter}; +use zip_next::{CompressionMethod, ZipArchive, ZipWriter}; use zip_next::write::FileOptions; const FILE_COUNT: usize = 15_000; @@ -12,8 +12,7 @@ const FILE_SIZE: usize = 1024; fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { let data = Vec::new(); let mut writer = ZipWriter::new(Cursor::new(data)); - let options = FileOptions::default() - .compression_method(zip_next::CompressionMethod::Stored); + let options = FileOptions::default().compression_method(CompressionMethod::Stored); let bytes = vec![0u8; file_size]; From 98d37c8b777456ff1441cd1c197d46227c9d2415 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 15:26:00 -0700 Subject: [PATCH 018/281] Fix more formatting issues (sort imports) --- benches/read_metadata.rs | 2 +- tests/end_to_end.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index f7664772..b4d6bb36 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -3,8 +3,8 @@ use bencher::{benchmark_group, benchmark_main}; use std::io::{Cursor, Write}; use bencher::Bencher; -use zip_next::{CompressionMethod, ZipArchive, ZipWriter}; use zip_next::write::FileOptions; +use zip_next::{CompressionMethod, ZipArchive, ZipWriter}; const FILE_COUNT: usize = 15_000; const FILE_SIZE: usize = 1024; diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 4f4b8744..fd31570b 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -3,9 +3,9 @@ use std::collections::HashSet; use std::io::prelude::*; use std::io::{Cursor, Seek}; use std::iter::FromIterator; -use zip_next::write::FileOptions; -use zip_next::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS, ZipWriter}; use zip_next::result::ZipResult; +use zip_next::write::FileOptions; +use zip_next::{CompressionMethod, ZipWriter, SUPPORTED_COMPRESSION_METHODS}; // This test asserts that after creating a zip file, then reading its contents back out, // the extracted data will *always* be exactly the same as the original data. From 02a44087aef4086f664f967d0a7b634122400cfd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:32:47 +0000 Subject: [PATCH 019/281] chore(deps): update pbkdf2 requirement from 0.11.0 to 0.12.1 Updates the requirements on [pbkdf2](https://github.com/RustCrypto/password-hashes) to permit the latest version. - [Release notes](https://github.com/RustCrypto/password-hashes/releases) - [Commits](https://github.com/RustCrypto/password-hashes/compare/scrypt-v0.11.0...pbkdf2-v0.12.1) --- updated-dependencies: - dependency-name: pbkdf2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 474d09cd..f7a9ed84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ constant_time_eq = { version = "0.1.5", optional = true } crc32fast = "1.3.2" flate2 = { version = "1.0.23", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } -pbkdf2 = {version = "0.11.0", optional = true } +pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.1", optional = true } time = { version = "0.3.7", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.11.2", optional = true } From 74747c37835d3052381384179145218c88301640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:33:02 +0000 Subject: [PATCH 020/281] chore(deps): update zstd requirement from 0.11.2 to 0.12.3 Updates the requirements on [zstd](https://github.com/gyscos/zstd-rs) to permit the latest version. - [Release notes](https://github.com/gyscos/zstd-rs/releases) - [Commits](https://github.com/gyscos/zstd-rs/commits) --- updated-dependencies: - dependency-name: zstd dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 474d09cd..91882154 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ hmac = { version = "0.12.1", optional = true, features = ["reset"] } pbkdf2 = {version = "0.11.0", optional = true } sha1 = {version = "0.10.1", optional = true } time = { version = "0.3.7", optional = true, default-features = false, features = ["std"] } -zstd = { version = "0.11.2", optional = true } +zstd = { version = "0.12.3", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.8" From 82071e101a5388990657fd25e6dd1f7cc2054a42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:33:12 +0000 Subject: [PATCH 021/281] chore(deps): update constant_time_eq requirement from 0.1.5 to 0.2.5 Updates the requirements on [constant_time_eq](https://github.com/cesarb/constant_time_eq) to permit the latest version. - [Release notes](https://github.com/cesarb/constant_time_eq/releases) - [Changelog](https://github.com/cesarb/constant_time_eq/blob/master/CHANGES) - [Commits](https://github.com/cesarb/constant_time_eq/compare/0.1.5...0.2.5) --- updated-dependencies: - dependency-name: constant_time_eq dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 474d09cd..b3378e33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ edition = "2021" aes = { version = "0.7.5", optional = true } byteorder = "1.4.3" bzip2 = { version = "0.4.3", optional = true } -constant_time_eq = { version = "0.1.5", optional = true } +constant_time_eq = { version = "0.2.5", optional = true } crc32fast = "1.3.2" flate2 = { version = "1.0.23", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } From 491c512d6c3bb6f24d9f86d319e10582d4994312 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 16:00:27 -0700 Subject: [PATCH 022/281] Update AES library (API has changed) --- Cargo.toml | 2 +- src/aes.rs | 9 +++++---- src/aes_ctr.rs | 11 +++++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 44ccf407..69ad86c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ Library to support the reading and writing of zip files. edition = "2021" [dependencies] -aes = { version = "0.7.5", optional = true } +aes = { version = "0.8.2", optional = true } byteorder = "1.4.3" bzip2 = { version = "0.4.3", optional = true } constant_time_eq = { version = "0.2.5", optional = true } diff --git a/src/aes.rs b/src/aes.rs index 8997705c..c42a05ca 100644 --- a/src/aes.rs +++ b/src/aes.rs @@ -9,7 +9,7 @@ use crate::types::AesMode; use constant_time_eq::constant_time_eq; use hmac::{Hmac, Mac}; use sha1::Sha1; -use std::io::{self, Read}; +use std::io::{self, Error, ErrorKind, Read}; /// The length of the password verifcation value in bytes const PWD_VERIFY_LENGTH: usize = 2; @@ -84,7 +84,8 @@ impl AesReader { let mut derived_key: Vec = vec![0; derived_key_len]; // use PBKDF2 with HMAC-Sha1 to derive the key - pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key); + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; let decrypt_key = &derived_key[0..key_length]; let hmac_key = &derived_key[key_length..key_length * 2]; let pwd_verify = &derived_key[derived_key_len - 2..]; @@ -165,8 +166,8 @@ impl Read for AesReaderValid { // use constant time comparison to mitigate timing attacks if !constant_time_eq(computed_auth_code, &read_auth_code) { return Err( - io::Error::new( - io::ErrorKind::InvalidData, + Error::new( + ErrorKind::InvalidData, "Invalid authentication code, this could be due to an invalid password or errors in the data" ) ); diff --git a/src/aes_ctr.rs b/src/aes_ctr.rs index 0f34335c..1c252584 100644 --- a/src/aes_ctr.rs +++ b/src/aes_ctr.rs @@ -4,10 +4,12 @@ //! different byte order (little endian) than NIST (big endian). //! See [AesCtrZipKeyStream](./struct.AesCtrZipKeyStream.html) for more information. +use aes::cipher; +use aes::cipher::{BlockCipher, BlockEncrypt}; use aes::cipher::generic_array::GenericArray; -use aes::{BlockEncrypt, NewBlockCipher}; use byteorder::WriteBytesExt; use std::{any, fmt}; +use cipher::KeyInit; /// Internal block size of an AES cipher. const AES_BLOCK_SIZE: usize = 16; @@ -27,7 +29,7 @@ pub trait AesKind { /// Key type. type Key: AsRef<[u8]>; /// Cipher used to decrypt. - type Cipher; + type Cipher: KeyInit; } impl AesKind for Aes128 { @@ -82,7 +84,7 @@ where impl AesCtrZipKeyStream where C: AesKind, - C::Cipher: NewBlockCipher, + C::Cipher: BlockCipher, { /// Creates a new zip variant AES-CTR key stream. /// @@ -151,13 +153,14 @@ fn xor(dest: &mut [u8], src: &[u8]) { mod tests { use super::{Aes128, Aes192, Aes256, AesCipher, AesCtrZipKeyStream, AesKind}; use aes::{BlockEncrypt, NewBlockCipher}; + use aes::cipher::{BlockCipher, BlockEncrypt}; /// Checks whether `crypt_in_place` produces the correct plaintext after one use and yields the /// cipertext again after applying it again. fn roundtrip(key: &[u8], ciphertext: &mut [u8], expected_plaintext: &[u8]) where Aes: AesKind, - Aes::Cipher: NewBlockCipher + BlockEncrypt, + Aes::Cipher: BlockCipher + BlockEncrypt, { let mut key_stream = AesCtrZipKeyStream::::new(key); From 7d7325324f8ec432fa0788f554dc92ad8b3e0994 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 16:00:37 -0700 Subject: [PATCH 023/281] Enable fuzz testing during CI --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5312ab85..b761b9a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,3 +90,6 @@ jobs: - name: compile fuzz run: | cargo fuzz build fuzz_read + - name: run fuzz + run: | + cargo fuzz run fuzz_read -- -timeout=1s -runs=10000000 From 406abacd2c71d8822b9e0fe6ea410c9a716b94cf Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 16:03:21 -0700 Subject: [PATCH 024/281] Fix formatting --- src/aes_ctr.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aes_ctr.rs b/src/aes_ctr.rs index 1c252584..bf5ca9e5 100644 --- a/src/aes_ctr.rs +++ b/src/aes_ctr.rs @@ -5,11 +5,11 @@ //! See [AesCtrZipKeyStream](./struct.AesCtrZipKeyStream.html) for more information. use aes::cipher; -use aes::cipher::{BlockCipher, BlockEncrypt}; use aes::cipher::generic_array::GenericArray; +use aes::cipher::{BlockCipher, BlockEncrypt}; use byteorder::WriteBytesExt; -use std::{any, fmt}; use cipher::KeyInit; +use std::{any, fmt}; /// Internal block size of an AES cipher. const AES_BLOCK_SIZE: usize = 16; @@ -152,7 +152,6 @@ fn xor(dest: &mut [u8], src: &[u8]) { #[cfg(test)] mod tests { use super::{Aes128, Aes192, Aes256, AesCipher, AesCtrZipKeyStream, AesKind}; - use aes::{BlockEncrypt, NewBlockCipher}; use aes::cipher::{BlockCipher, BlockEncrypt}; /// Checks whether `crypt_in_place` produces the correct plaintext after one use and yields the From 5f9567e31327f431a947f91aa3cbc1a8fd6f3a3f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 16:08:45 -0700 Subject: [PATCH 025/281] Bump version to 0.6.6 --- Cargo.toml | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 69ad86c2..85dbe586 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.5" +version = "0.6.6" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 65938642..cd7af001 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://img.shields.io/github/workflow/status/Pr0methean/zip-next/CI)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.6.5/zip_next/) +[Documentation](https://docs.rs/zip_next/0.6.6/zip_next/) Info ---- @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.5" +zip_next = "0.6.6" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.5", default-features = false } +zip_next = { version = "0.6.6", default-features = false } ``` The features available are: From d0300bc6e6025132f509610da795504c9b86b364 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 23 Apr 2023 16:20:38 -0700 Subject: [PATCH 026/281] Update repository link --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 85dbe586..e1b5c67a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.6.6" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" -repository = "https://github.com/zip-rs/zip.git" +repository = "https://github.com/Pr0methean/zip-next.git" keywords = ["zip", "archive"] description = """ Library to support the reading and writing of zip files. From 36e7b19969d1941780a3b3d1f37cd313df08d066 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 24 Apr 2023 10:44:36 -0700 Subject: [PATCH 027/281] Add deep-copy method, and include copying in end-to-end tests --- src/write.rs | 103 ++++++++++++++++++++++++++++++++++++++------ tests/end_to_end.rs | 53 +++++++++++++++-------- 2 files changed, 124 insertions(+), 32 deletions(-) diff --git a/src/write.rs b/src/write.rs index ac07a49c..025fcbf4 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1,7 +1,7 @@ //! Types for creating ZIP archives use crate::compression::CompressionMethod; -use crate::read::{central_header_to_zip_file, ZipArchive, ZipFile}; +use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, ZipFileReader}; use crate::result::{ZipError, ZipResult}; use crate::spec; use crate::types::{AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; @@ -11,6 +11,7 @@ use std::convert::TryInto; use std::default::Default; use std::io; use std::io::prelude::*; +use std::io::{BufReader, SeekFrom}; use std::mem; #[cfg(any( @@ -268,10 +269,7 @@ impl ZipWriter { let (archive_offset, directory_start, number_of_files) = ZipArchive::get_directory_counts(&mut readwriter, &footer, cde_start_pos)?; - if readwriter - .seek(io::SeekFrom::Start(directory_start)) - .is_err() - { + if readwriter.seek(SeekFrom::Start(directory_start)).is_err() { return Err(ZipError::InvalidArchive( "Could not seek to start of central directory", )); @@ -281,7 +279,7 @@ impl ZipWriter { .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset)) .collect::, _>>()?; - let _ = readwriter.seek(io::SeekFrom::Start(directory_start)); // seek directory_start to overwrite it + let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it Ok(ZipWriter { inner: GenericZipWriter::Storer(readwriter), @@ -296,6 +294,45 @@ impl ZipWriter { } } +impl ZipWriter { + /// Adds another copy of a file already in this archive. This will produce a larger but more + /// widely-compatible archive compared to [shallow_copy_file]. + pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + let write_position = self.inner.get_plain().stream_position()?; + let src_data = self.data_by_name(src_name)?.to_owned(); + let data_start = src_data.data_start.load(); + let real_size = src_data.compressed_size.max(write_position - data_start); + let mut options = FileOptions::default() + .large_file(real_size > spec::ZIP64_BYTES_THR) + .last_modified_time(src_data.last_modified_time) + .compression_method(src_data.compression_method); + if let Some(perms) = src_data.unix_mode() { + options = options.unix_permissions(perms); + } + + let raw_values = ZipRawValues { + crc32: src_data.crc32, + compressed_size: real_size, + uncompressed_size: src_data.uncompressed_size, + }; + let reader = self.inner.get_plain(); + let mut reader = BufReader::new(ZipFileReader::Raw(find_content(&src_data, reader)?)); + let mut copy = Vec::with_capacity(real_size as usize); + reader.read_to_end(&mut copy)?; + drop(reader); + self.inner + .get_plain() + .seek(SeekFrom::Start(write_position))?; + self.start_entry(dest_name, options, Some(raw_values))?; + self.writing_raw = true; + self.writing_to_file = true; + self.write_all(©)?; + + Ok(()) + } +} + impl ZipWriter { /// Initializes the archive. /// @@ -409,7 +446,7 @@ impl ZipWriter { file.compressed_size = file_end - self.stats.start; update_local_file_header(writer, file)?; - writer.seek(io::SeekFrom::Start(file_end))?; + writer.seek(SeekFrom::Start(file_end))?; } self.writing_to_file = false; @@ -603,9 +640,9 @@ impl ZipWriter { // Update extra field length in local file header. let extra_field_length = if file.large_file { 20 } else { 0 } + file.extra_field.len() as u16; - writer.seek(io::SeekFrom::Start(file.header_start + 28))?; + writer.seek(SeekFrom::Start(file.header_start + 28))?; writer.write_u16::(extra_field_length)?; - writer.seek(io::SeekFrom::Start(header_end))?; + writer.seek(SeekFrom::Start(header_end))?; self.inner .switch_to(file.compression_method, file.compression_level)?; @@ -852,9 +889,9 @@ impl ZipWriter { /// Adds another entry to the central directory referring to the same content as an existing /// entry. The file's local-file header will still refer to it by its original name, so - /// unzipping the file will technically be unspecified behavior. However, both [ZipArchive] and - /// OpenJDK ignore the filename in the local-file header and treat the central directory as - /// authoritative. + /// unzipping the file will technically be unspecified behavior. [ZipArchive] ignores the + /// filename in the local-file header and treat the central directory as authoritative. However, + /// some other software (e.g. Minecraft) will refuse to extract a file copied this way. pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; let src_data = self.data_by_name(src_name)?; @@ -1117,7 +1154,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR fn update_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { const CRC32_OFFSET: u64 = 14; - writer.seek(io::SeekFrom::Start(file.header_start + CRC32_OFFSET))?; + writer.seek(SeekFrom::Start(file.header_start + CRC32_OFFSET))?; writer.write_u32::(file.crc32)?; if file.large_file { update_local_zip64_extra_field(writer, file)?; @@ -1265,7 +1302,7 @@ fn update_local_zip64_extra_field( file: &ZipFileData, ) -> ZipResult<()> { let zip64_extra_field = file.header_start + 30 + file.file_name.as_bytes().len() as u64; - writer.seek(io::SeekFrom::Start(zip64_extra_field + 4))?; + writer.seek(SeekFrom::Start(zip64_extra_field + 4))?; writer.write_u64::(file.uncompressed_size)?; writer.write_u64::(file.compressed_size)?; // Excluded fields: @@ -1513,6 +1550,44 @@ mod test { assert_eq!(second_file_content, RT_TEST_TEXT); } + #[test] + fn test_deep_copy() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::Deflated, + compression_level: Some(9), + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap(); + let mut reader = ZipArchive::new(zip).unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + #[test] fn path_to_string() { let mut path = std::path::PathBuf::new(); diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index fd31570b..f5c56f14 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -15,10 +15,11 @@ fn end_to_end() { let file = &mut Cursor::new(Vec::new()); println!("Writing file with {method} compression"); - write_test_archive(file, method).expect("Couldn't write test zip archive"); + write_test_archive(file, method, true).expect("Couldn't write test zip archive"); println!("Checking file contents"); check_archive_file(file, ENTRY_NAME, Some(method), LOREM_IPSUM); + check_archive_file(file, INTERNAL_COPY_ENTRY_NAME, Some(method), LOREM_IPSUM); } } @@ -28,7 +29,7 @@ fn end_to_end() { fn copy() { for &method in SUPPORTED_COMPRESSION_METHODS { let src_file = &mut Cursor::new(Vec::new()); - write_test_archive(src_file, method).expect("Couldn't write to test file"); + write_test_archive(src_file, method, false).expect("Couldn't write to test file"); let mut tgt_file = &mut Cursor::new(Vec::new()); @@ -66,28 +67,35 @@ fn copy() { #[test] fn append() { for &method in SUPPORTED_COMPRESSION_METHODS { - let mut file = &mut Cursor::new(Vec::new()); - write_test_archive(file, method).expect("Couldn't write to test file"); + for shallow_copy in vec![false, true] { + let mut file = &mut Cursor::new(Vec::new()); + write_test_archive(file, method, shallow_copy).expect("Couldn't write to test file"); - { - let mut zip = ZipWriter::new_append(&mut file).unwrap(); - zip.start_file( - COPY_ENTRY_NAME, - FileOptions::default().compression_method(method), - ) - .unwrap(); - zip.write_all(LOREM_IPSUM).unwrap(); - zip.finish().unwrap(); + { + let mut zip = ZipWriter::new_append(&mut file).unwrap(); + zip.start_file( + COPY_ENTRY_NAME, + FileOptions::default().compression_method(method), + ) + .unwrap(); + zip.write_all(LOREM_IPSUM).unwrap(); + zip.finish().unwrap(); + } + + let mut zip = zip_next::ZipArchive::new(&mut file).unwrap(); + check_archive_file_contents(&mut zip, ENTRY_NAME, LOREM_IPSUM); + check_archive_file_contents(&mut zip, COPY_ENTRY_NAME, LOREM_IPSUM); + check_archive_file_contents(&mut zip, INTERNAL_COPY_ENTRY_NAME, LOREM_IPSUM); } - - let mut zip = zip_next::ZipArchive::new(&mut file).unwrap(); - check_archive_file_contents(&mut zip, ENTRY_NAME, LOREM_IPSUM); - check_archive_file_contents(&mut zip, COPY_ENTRY_NAME, LOREM_IPSUM); } } // Write a test zip archive to buffer. -fn write_test_archive(file: &mut Cursor>, method: CompressionMethod) -> ZipResult<()> { +fn write_test_archive( + file: &mut Cursor>, + method: CompressionMethod, + shallow_copy: bool, +) -> ZipResult<()> { let mut zip = ZipWriter::new(file); zip.add_directory("test/", Default::default())?; @@ -109,6 +117,12 @@ fn write_test_archive(file: &mut Cursor>, method: CompressionMethod) -> zip.start_file(ENTRY_NAME, options)?; zip.write_all(LOREM_IPSUM)?; + if shallow_copy { + zip.shallow_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME)?; + } else { + zip.deep_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME)?; + } + zip.finish()?; Ok(()) } @@ -124,6 +138,7 @@ fn check_test_archive(zip_file: R) -> ZipResult>(); @@ -201,3 +216,5 @@ const EXTRA_DATA: &[u8] = b"Extra Data"; const ENTRY_NAME: &str = "test/lorem_ipsum.txt"; const COPY_ENTRY_NAME: &str = "test/lorem_ipsum_renamed.txt"; + +const INTERNAL_COPY_ENTRY_NAME: &str = "test/lorem_ipsum_copied.txt"; From 50b31c25af4eb55eb6778851b6805dcf53d20761 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 24 Apr 2023 10:49:12 -0700 Subject: [PATCH 028/281] Fix Clippy issue --- tests/end_to_end.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index f5c56f14..245ff6ba 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -67,9 +67,9 @@ fn copy() { #[test] fn append() { for &method in SUPPORTED_COMPRESSION_METHODS { - for shallow_copy in vec![false, true] { + for shallow_copy in &[false, true] { let mut file = &mut Cursor::new(Vec::new()); - write_test_archive(file, method, shallow_copy).expect("Couldn't write to test file"); + write_test_archive(file, method, *shallow_copy).expect("Couldn't write to test file"); { let mut zip = ZipWriter::new_append(&mut file).unwrap(); From 570992c2d537dd5e3de1d64ddb3bcc41971488e3 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 24 Apr 2023 11:05:35 -0700 Subject: [PATCH 029/281] Update CHANGELOG --- CHANGELOG.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd79e391..e80b8b57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,4 +6,19 @@ - [#333](https://github.com/zip-rs/zip/pull/333): disabled the default features of the `time` dependency, and also `formatting` and `macros`, as they were enabled by mistake. - Deprecated [`DateTime::from_time`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#method.from_time) in favor of [`DateTime::try_from`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#impl-TryFrom-for-DateTime) - \ No newline at end of file + +## [0.6.6] + +### Changed + + - Updated dependency versions. + +### Added + + - `shallow_copy_file` method: copy a file from within the ZipWriter + +## [0.6.7] + +### Added + +- `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter From 6b5d05869137ab02301232c35d13fb6bcbd41361 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 24 Apr 2023 11:05:54 -0700 Subject: [PATCH 030/281] Bump version to 0.6.7 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e1b5c67a..fbb0ece4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.6" +version = "0.6.7" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index cd7af001..7936e29c 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.6" +zip_next = "0.6.7" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.6", default-features = false } +zip_next = { version = "0.6.7", default-features = false } ``` The features available are: From eac20be7ea2feea91b1fa0f8e481099b4c1a7a4b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 24 Apr 2023 11:08:26 -0700 Subject: [PATCH 031/281] Corrections to CHANGELOG --- CHANGELOG.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e80b8b57..b79d255e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,19 +6,25 @@ - [#333](https://github.com/zip-rs/zip/pull/333): disabled the default features of the `time` dependency, and also `formatting` and `macros`, as they were enabled by mistake. - Deprecated [`DateTime::from_time`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#method.from_time) in favor of [`DateTime::try_from`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#impl-TryFrom-for-DateTime) + +## [0.6.5] + +### Added + + - `shallow_copy_file` method: copy a file from within the ZipWriter ## [0.6.6] +### Fixed + + - Unused flag `#![feature(read_buf)]` was breaking compatibility with stable compiler. + ### Changed - Updated dependency versions. -### Added - - - `shallow_copy_file` method: copy a file from within the ZipWriter - ## [0.6.7] ### Added -- `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter + - `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter From de16a07a1b6899ca8e30a634c8c2f90730075bed Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Tue, 25 Apr 2023 14:42:40 -0700 Subject: [PATCH 032/281] Fix build-status badge link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7936e29c..b88e18a2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ zip_next ======== -[![Build Status](https://img.shields.io/github/workflow/status/Pr0methean/zip-next/CI)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) +[![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) [Documentation](https://docs.rs/zip_next/0.6.6/zip_next/) From b7f4bd6faf3d229b6aa06a6a97df755a006aedf6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 25 Apr 2023 15:06:11 -0700 Subject: [PATCH 033/281] Minor refactor --- src/write.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/write.rs b/src/write.rs index 025fcbf4..49d03696 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1,7 +1,7 @@ //! Types for creating ZIP archives use crate::compression::CompressionMethod; -use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, ZipFileReader}; +use crate::read::{central_header_to_zip_file, find_content, make_crypto_reader, make_reader, ZipArchive, ZipFile, ZipFileReader}; use crate::result::{ZipError, ZipResult}; use crate::spec; use crate::types::{AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; @@ -295,12 +295,17 @@ impl ZipWriter { } impl ZipWriter { + fn reader_by_name<'a>(&'a mut self, name: &str) -> ZipResult> { + let data = self.data_by_name(name)?.to_owned(); + find_content(&data, self.inner.get_plain()) + } + /// Adds another copy of a file already in this archive. This will produce a larger but more /// widely-compatible archive compared to [shallow_copy_file]. pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; let write_position = self.inner.get_plain().stream_position()?; - let src_data = self.data_by_name(src_name)?.to_owned(); + let src_data = self.data_by_name(src_name)?; let data_start = src_data.data_start.load(); let real_size = src_data.compressed_size.max(write_position - data_start); let mut options = FileOptions::default() @@ -310,14 +315,13 @@ impl ZipWriter { if let Some(perms) = src_data.unix_mode() { options = options.unix_permissions(perms); } - let raw_values = ZipRawValues { crc32: src_data.crc32, compressed_size: real_size, uncompressed_size: src_data.uncompressed_size, }; - let reader = self.inner.get_plain(); - let mut reader = BufReader::new(ZipFileReader::Raw(find_content(&src_data, reader)?)); + let mut reader = + BufReader::new(ZipFileReader::Raw(self.reader_by_name(src_name)?)); let mut copy = Vec::with_capacity(real_size as usize); reader.read_to_end(&mut copy)?; drop(reader); @@ -877,8 +881,7 @@ impl ZipWriter { Ok(()) } - fn data_by_name(&mut self, name: &str) -> ZipResult<&ZipFileData> { - self.finish_file()?; + fn data_by_name(&self, name: &str) -> ZipResult<&ZipFileData> { for file in self.files.iter() { if file.file_name == name { return Ok(file); From aabdec98aa911a251f5399e96f4366334e03a062 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 25 Apr 2023 15:06:33 -0700 Subject: [PATCH 034/281] Reformat --- src/write.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/write.rs b/src/write.rs index 49d03696..24d25d56 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1,7 +1,10 @@ //! Types for creating ZIP archives use crate::compression::CompressionMethod; -use crate::read::{central_header_to_zip_file, find_content, make_crypto_reader, make_reader, ZipArchive, ZipFile, ZipFileReader}; +use crate::read::{ + central_header_to_zip_file, find_content, make_crypto_reader, make_reader, ZipArchive, ZipFile, + ZipFileReader, +}; use crate::result::{ZipError, ZipResult}; use crate::spec; use crate::types::{AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; @@ -320,8 +323,7 @@ impl ZipWriter { compressed_size: real_size, uncompressed_size: src_data.uncompressed_size, }; - let mut reader = - BufReader::new(ZipFileReader::Raw(self.reader_by_name(src_name)?)); + let mut reader = BufReader::new(ZipFileReader::Raw(self.reader_by_name(src_name)?)); let mut copy = Vec::with_capacity(real_size as usize); reader.read_to_end(&mut copy)?; drop(reader); From a86f87ff6f4c732d78d4efe817b7b7ce8ac7e097 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 25 Apr 2023 16:34:03 -0700 Subject: [PATCH 035/281] Remove unused imports --- src/write.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/write.rs b/src/write.rs index 24d25d56..12d26726 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1,10 +1,7 @@ //! Types for creating ZIP archives use crate::compression::CompressionMethod; -use crate::read::{ - central_header_to_zip_file, find_content, make_crypto_reader, make_reader, ZipArchive, ZipFile, - ZipFileReader, -}; +use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, ZipFileReader}; use crate::result::{ZipError, ZipResult}; use crate::spec; use crate::types::{AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; @@ -911,7 +908,7 @@ impl Drop for ZipWriter { fn drop(&mut self) { if !self.inner.is_closed() { if let Err(e) = self.finalize() { - let _ = write!(io::stderr(), "ZipWriter drop failed: {e:?}"); + let _ = write!(io::stderr(), "ZipWriter drop failed: {:?}", e); } } } From 7d891942989b4b544c58f7d7761686d239100ebb Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 12:48:54 -0700 Subject: [PATCH 036/281] Bug fix for permissions on deep-copied files --- src/types.rs | 32 +++++++++++++-------------- src/write.rs | 45 ++++++++++++++++++++------------------ tests/end_to_end.rs | 53 +++++++++++++++++++++++++-------------------- 3 files changed, 68 insertions(+), 62 deletions(-) diff --git a/src/types.rs b/src/types.rs index c333d8fa..85ce6427 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,4 +1,5 @@ //! Types that specify what is contained in a ZIP. +use path::{Component, Path, PathBuf}; use std::path; #[cfg(not(any( @@ -12,7 +13,7 @@ use std::time::SystemTime; #[cfg(doc)] use {crate::read::ZipFile, crate::write::FileOptions}; -mod ffi { +pub(crate) mod ffi { pub const S_IFDIR: u32 = 0o0040000; pub const S_IFREG: u32 = 0o0100000; } @@ -100,7 +101,7 @@ pub struct DateTime { second: u8, } -impl ::std::default::Default for DateTime { +impl Default for DateTime { /// Constructs an 'default' datetime of 1980-01-01 00:00:00 fn default() -> DateTime { DateTime { @@ -353,7 +354,7 @@ pub struct ZipFileData { } impl ZipFileData { - pub fn file_name_sanitized(&self) -> ::std::path::PathBuf { + pub fn file_name_sanitized(&self) -> PathBuf { let no_null_filename = match self.file_name.find('\0') { Some(index) => &self.file_name[0..index], None => &self.file_name, @@ -363,7 +364,7 @@ impl ZipFileData { // zip files can contain both / and \ as separators regardless of the OS // and as we want to return a sanitized PathBuf that only supports the // OS separator let's convert incompatible separators to compatible ones - let separator = ::std::path::MAIN_SEPARATOR; + let separator = path::MAIN_SEPARATOR; let opposite_separator = match separator { '/' => '\\', _ => '/', @@ -371,27 +372,27 @@ impl ZipFileData { let filename = no_null_filename.replace(&opposite_separator.to_string(), &separator.to_string()); - ::std::path::Path::new(&filename) + Path::new(&filename) .components() - .filter(|component| matches!(*component, ::std::path::Component::Normal(..))) - .fold(::std::path::PathBuf::new(), |mut path, ref cur| { + .filter(|component| matches!(*component, path::Component::Normal(..))) + .fold(PathBuf::new(), |mut path, ref cur| { path.push(cur.as_os_str()); path }) } - pub(crate) fn enclosed_name(&self) -> Option<&path::Path> { + pub(crate) fn enclosed_name(&self) -> Option<&Path> { if self.file_name.contains('\0') { return None; } - let path = path::Path::new(&self.file_name); + let path = Path::new(&self.file_name); let mut depth = 0usize; for component in path.components() { match component { - path::Component::Prefix(_) | path::Component::RootDir => return None, - path::Component::ParentDir => depth = depth.checked_sub(1)?, - path::Component::Normal(_) => depth += 1, - path::Component::CurDir => (), + Component::Prefix(_) | Component::RootDir => return None, + Component::ParentDir => depth = depth.checked_sub(1)?, + Component::Normal(_) => depth += 1, + Component::CurDir => (), } } Some(path) @@ -509,10 +510,7 @@ mod test { large_file: false, aes_mode: None, }; - assert_eq!( - data.file_name_sanitized(), - ::std::path::PathBuf::from("path/etc/passwd") - ); + assert_eq!(data.file_name_sanitized(), PathBuf::from("path/etc/passwd")); } #[test] diff --git a/src/write.rs b/src/write.rs index 12d26726..313f77d2 100644 --- a/src/write.rs +++ b/src/write.rs @@ -4,7 +4,7 @@ use crate::compression::CompressionMethod; use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, ZipFileReader}; use crate::result::{ZipError, ZipResult}; use crate::spec; -use crate::types::{AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; +use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crc32fast::Hasher; use std::convert::TryInto; @@ -307,32 +307,33 @@ impl ZipWriter { let write_position = self.inner.get_plain().stream_position()?; let src_data = self.data_by_name(src_name)?; let data_start = src_data.data_start.load(); - let real_size = src_data.compressed_size.max(write_position - data_start); + let compressed_size = src_data.compressed_size; + if compressed_size > write_position - data_start { + return Err(ZipError::InvalidArchive("Source file size too large")); + } + let uncompressed_size = src_data.uncompressed_size; let mut options = FileOptions::default() - .large_file(real_size > spec::ZIP64_BYTES_THR) + .large_file(compressed_size.max(uncompressed_size) > spec::ZIP64_BYTES_THR) .last_modified_time(src_data.last_modified_time) .compression_method(src_data.compression_method); if let Some(perms) = src_data.unix_mode() { options = options.unix_permissions(perms); } + Self::normalize_options(&mut options); let raw_values = ZipRawValues { crc32: src_data.crc32, - compressed_size: real_size, - uncompressed_size: src_data.uncompressed_size, + compressed_size, + uncompressed_size, }; let mut reader = BufReader::new(ZipFileReader::Raw(self.reader_by_name(src_name)?)); - let mut copy = Vec::with_capacity(real_size as usize); + let mut copy = Vec::with_capacity(compressed_size as usize); reader.read_to_end(&mut copy)?; drop(reader); - self.inner - .get_plain() - .seek(SeekFrom::Start(write_position))?; self.start_entry(dest_name, options, Some(raw_values))?; - self.writing_raw = true; + self.inner.switch_to(CompressionMethod::Stored, None)?; self.writing_to_file = true; - self.write_all(©)?; - - Ok(()) + self.writing_raw = true; + Ok(self.write_all(©)?) } } @@ -464,10 +465,7 @@ impl ZipWriter { where S: Into, { - if options.permissions.is_none() { - options.permissions = Some(0o644); - } - *options.permissions.as_mut().unwrap() |= 0o100000; + Self::normalize_options(&mut options); self.start_entry(name, options, None)?; self.inner .switch_to(options.compression_method, options.compression_level)?; @@ -475,6 +473,13 @@ impl ZipWriter { Ok(()) } + fn normalize_options(options: &mut FileOptions) { + if options.permissions.is_none() { + options.permissions = Some(0o644); + } + *options.permissions.as_mut().unwrap() |= ffi::S_IFREG; + } + /// Starts a file, taking a Path as argument. /// /// This function ensures that the '/' path separator is used. It also ignores all non 'Normal' @@ -591,10 +596,7 @@ impl ZipWriter { where S: Into, { - if options.permissions.is_none() { - options.permissions = Some(0o644); - } - *options.permissions.as_mut().unwrap() |= 0o100000; + Self::normalize_options(&mut options); self.start_entry(name, options, None)?; self.writing_to_file = true; self.writing_to_extra_field = true; @@ -693,6 +695,7 @@ impl ZipWriter { if let Some(perms) = file.unix_mode() { options = options.unix_permissions(perms); } + Self::normalize_options(&mut options); let raw_values = ZipRawValues { crc32: file.crc32(), diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 245ff6ba..57f7a8e7 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -15,7 +15,7 @@ fn end_to_end() { let file = &mut Cursor::new(Vec::new()); println!("Writing file with {method} compression"); - write_test_archive(file, method, true).expect("Couldn't write test zip archive"); + write_test_archive(file, method, true); println!("Checking file contents"); check_archive_file(file, ENTRY_NAME, Some(method), LOREM_IPSUM); @@ -29,7 +29,7 @@ fn end_to_end() { fn copy() { for &method in SUPPORTED_COMPRESSION_METHODS { let src_file = &mut Cursor::new(Vec::new()); - write_test_archive(src_file, method, false).expect("Couldn't write to test file"); + write_test_archive(src_file, method, false); let mut tgt_file = &mut Cursor::new(Vec::new()); @@ -68,14 +68,17 @@ fn copy() { fn append() { for &method in SUPPORTED_COMPRESSION_METHODS { for shallow_copy in &[false, true] { + println!("Writing file with {method} compression, shallow_copy {shallow_copy}"); let mut file = &mut Cursor::new(Vec::new()); - write_test_archive(file, method, *shallow_copy).expect("Couldn't write to test file"); + write_test_archive(file, method, *shallow_copy); { let mut zip = ZipWriter::new_append(&mut file).unwrap(); zip.start_file( COPY_ENTRY_NAME, - FileOptions::default().compression_method(method), + FileOptions::default() + .compression_method(method) + .unix_permissions(0o755), ) .unwrap(); zip.write_all(LOREM_IPSUM).unwrap(); @@ -91,40 +94,39 @@ fn append() { } // Write a test zip archive to buffer. -fn write_test_archive( - file: &mut Cursor>, - method: CompressionMethod, - shallow_copy: bool, -) -> ZipResult<()> { +fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, shallow_copy: bool) { let mut zip = ZipWriter::new(file); - zip.add_directory("test/", Default::default())?; + zip.add_directory("test/", Default::default()).unwrap(); let options = FileOptions::default() .compression_method(method) .unix_permissions(0o755); - zip.start_file("test/☃.txt", options)?; - zip.write_all(b"Hello, World!\n")?; + zip.start_file("test/☃.txt", options).unwrap(); + zip.write_all(b"Hello, World!\n").unwrap(); - zip.start_file_with_extra_data("test_with_extra_data/🐢.txt", options)?; - zip.write_u16::(0xbeef)?; - zip.write_u16::(EXTRA_DATA.len() as u16)?; - zip.write_all(EXTRA_DATA)?; - zip.end_extra_data()?; - zip.write_all(b"Hello, World! Again.\n")?; + zip.start_file_with_extra_data("test_with_extra_data/🐢.txt", options) + .unwrap(); + zip.write_u16::(0xbeef).unwrap(); + zip.write_u16::(EXTRA_DATA.len() as u16) + .unwrap(); + zip.write_all(EXTRA_DATA).unwrap(); + zip.end_extra_data().unwrap(); + zip.write_all(b"Hello, World! Again.\n").unwrap(); - zip.start_file(ENTRY_NAME, options)?; - zip.write_all(LOREM_IPSUM)?; + zip.start_file(ENTRY_NAME, options).unwrap(); + zip.write_all(LOREM_IPSUM).unwrap(); if shallow_copy { - zip.shallow_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME)?; + zip.shallow_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME) + .unwrap(); } else { - zip.deep_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME)?; + zip.deep_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME) + .unwrap(); } - zip.finish()?; - Ok(()) + zip.finish().unwrap(); } // Load an archive from buffer and check for test data. @@ -200,6 +202,9 @@ fn check_archive_file_contents( name: &str, expected: &[u8], ) { + let file_permissions: u32 = archive.by_name(name).unwrap().unix_mode().unwrap(); + assert_eq!(file_permissions, 0o100755); + let file_contents: String = read_archive_file(archive, name).unwrap(); assert_eq!(file_contents.as_bytes(), expected); } From d5681d9edeaece6e3a7a2318acf0da56e2073184 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 13:16:14 -0700 Subject: [PATCH 037/281] Refactor: don't search for src_data twice --- src/write.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/write.rs b/src/write.rs index 313f77d2..89c73f70 100644 --- a/src/write.rs +++ b/src/write.rs @@ -295,11 +295,6 @@ impl ZipWriter { } impl ZipWriter { - fn reader_by_name<'a>(&'a mut self, name: &str) -> ZipResult> { - let data = self.data_by_name(name)?.to_owned(); - find_content(&data, self.inner.get_plain()) - } - /// Adds another copy of a file already in this archive. This will produce a larger but more /// widely-compatible archive compared to [shallow_copy_file]. pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { @@ -325,7 +320,7 @@ impl ZipWriter { compressed_size, uncompressed_size, }; - let mut reader = BufReader::new(ZipFileReader::Raw(self.reader_by_name(src_name)?)); + let mut reader = BufReader::new(ZipFileReader::Raw(find_content(&data, self.inner.get_plain())?)); let mut copy = Vec::with_capacity(compressed_size as usize); reader.read_to_end(&mut copy)?; drop(reader); From 9a47bbb6c8efed22ea7c95cdb6d7752e7f87c034 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 13:19:55 -0700 Subject: [PATCH 038/281] Refactor: don't search for src_data twice --- src/write.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 89c73f70..e420e050 100644 --- a/src/write.rs +++ b/src/write.rs @@ -320,7 +320,10 @@ impl ZipWriter { compressed_size, uncompressed_size, }; - let mut reader = BufReader::new(ZipFileReader::Raw(find_content(&data, self.inner.get_plain())?)); + let mut reader = BufReader::new(ZipFileReader::Raw(find_content( + &src_data.to_owned(), + self.inner.get_plain(), + )?)); let mut copy = Vec::with_capacity(compressed_size as usize); reader.read_to_end(&mut copy)?; drop(reader); From 59b9279235923e31be2a00079f1a0c8ae05e0960 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 13:23:24 -0700 Subject: [PATCH 039/281] Update CHANGELOG --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b79d255e..1d1e4261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,3 +28,16 @@ ### Added - `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter + +## [0.6.8] + +### Fixed + + - `deep_copy_file` could set incorrect Unix permissions. + - `deep_copy_file` could handle files incorrectly if their compressed size was u32::MAX bytes or less but their + uncompressed size was not. + - Documented that `deep_copy_file` does not copy a directory's contents. + +### Changed + + - Improved performance of `deep_copy_file`: it no longer searches for the filename twice. \ No newline at end of file From 8c4ee1f42b2c22103353f9914755721cac2aa525 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 13:47:24 -0700 Subject: [PATCH 040/281] Wrap files in RefCell --- src/write.rs | 61 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/write.rs b/src/write.rs index e420e050..3962b80a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -7,6 +7,7 @@ use crate::spec; use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crc32fast::Hasher; +use std::cell::RefCell; use std::convert::TryInto; use std::default::Default; use std::io; @@ -47,6 +48,7 @@ enum GenericZipWriter { // Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely pub(crate) mod zip_writer { use super::*; + use std::cell::RefCell; /// ZIP archive generator /// /// Handles the bookkeeping involved in building an archive, and provides an @@ -77,7 +79,7 @@ pub(crate) mod zip_writer { /// ``` pub struct ZipWriter { pub(super) inner: GenericZipWriter, - pub(super) files: Vec, + pub(super) files: Vec>, pub(super) stats: ZipWriterStats, pub(super) writing_to_file: bool, pub(super) writing_to_extra_field: bool, @@ -212,13 +214,18 @@ impl Write for ZipWriter { match self.inner.ref_mut() { Some(ref mut w) => { if self.writing_to_extra_field { - self.files.last_mut().unwrap().extra_field.write(buf) + self.files + .last_mut() + .unwrap() + .borrow_mut() + .extra_field + .write(buf) } else { let write_result = w.write(buf); if let Ok(count) = write_result { self.stats.update(&buf[0..count]); if self.stats.bytes_written > spec::ZIP64_BYTES_THR - && !self.files.last_mut().unwrap().large_file + && !self.files.last_mut().unwrap().borrow().large_file { let _inner = mem::replace(&mut self.inner, GenericZipWriter::Closed); return Err(io::Error::new( @@ -276,7 +283,7 @@ impl ZipWriter { } let files = (0..number_of_files) - .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset)) + .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset).map(RefCell::new)) .collect::, _>>()?; let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it @@ -300,7 +307,8 @@ impl ZipWriter { pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; let write_position = self.inner.get_plain().stream_position()?; - let src_data = self.data_by_name(src_name)?; + let src_data_rc = self.data_by_name(src_name)?; + let src_data = src_data_rc.borrow(); let data_start = src_data.data_start.load(); let compressed_size = src_data.compressed_size; if compressed_size > write_position - data_start { @@ -320,8 +328,9 @@ impl ZipWriter { compressed_size, uncompressed_size, }; + drop(src_data); let mut reader = BufReader::new(ZipFileReader::Raw(find_content( - &src_data.to_owned(), + &src_data_rc.clone().borrow(), self.inner.get_plain(), )?)); let mut copy = Vec::with_capacity(compressed_size as usize); @@ -422,7 +431,7 @@ impl ZipWriter { self.stats.bytes_written = 0; self.stats.hasher = Hasher::new(); - self.files.push(file); + self.files.push(RefCell::new(file)); } Ok(()) @@ -437,9 +446,9 @@ impl ZipWriter { let writer = self.inner.get_plain(); if !self.writing_raw { - let file = match self.files.last_mut() { + let mut file = match self.files.last_mut() { None => return Ok(()), - Some(f) => f, + Some(f) => f.borrow_mut(), }; file.crc32 = self.stats.hasher.clone().finalize(); file.uncompressed_size = self.stats.bytes_written; @@ -447,7 +456,7 @@ impl ZipWriter { let file_end = writer.stream_position()?; file.compressed_size = file_end - self.stats.start; - update_local_file_header(writer, file)?; + update_local_file_header(writer, &file)?; writer.seek(SeekFrom::Start(file_end))?; } @@ -598,7 +607,7 @@ impl ZipWriter { self.start_entry(name, options, None)?; self.writing_to_file = true; self.writing_to_extra_field = true; - Ok(self.files.last().unwrap().data_start.load()) + Ok(self.files.last().unwrap().borrow().data_start.load()) } /// End local and start central extra data. Requires [`ZipWriter::start_file_with_extra_data`]. @@ -606,7 +615,12 @@ impl ZipWriter { /// Returns the final starting offset of the file data. pub fn end_local_start_central_extra_data(&mut self) -> ZipResult { let data_start = self.end_extra_data()?; - self.files.last_mut().unwrap().extra_field.clear(); + self.files + .last_mut() + .unwrap() + .borrow_mut() + .extra_field + .clear(); self.writing_to_extra_field = true; self.writing_to_central_extra_field_only = true; Ok(data_start) @@ -625,9 +639,10 @@ impl ZipWriter { } let file = self.files.last_mut().unwrap(); - validate_extra_data(file)?; + validate_extra_data(&file.borrow())?; - let data_start = file.data_start.get_mut(); + let mut file = file.borrow_mut(); + let mut data_start_result = file.data_start.load(); if !self.writing_to_central_extra_field_only { let writer = self.inner.get_plain(); @@ -636,9 +651,12 @@ impl ZipWriter { writer.write_all(&file.extra_field)?; // Update final `data_start`. - let header_end = *data_start + file.extra_field.len() as u64; + let length = file.extra_field.len(); + let data_start = file.data_start.get_mut(); + let header_end = *data_start + length as u64; self.stats.start = header_end; *data_start = header_end; + data_start_result = header_end; // Update extra field length in local file header. let extra_field_length = @@ -653,7 +671,7 @@ impl ZipWriter { self.writing_to_extra_field = false; self.writing_to_central_extra_field_only = false; - Ok(*data_start) + Ok(data_start_result) } /// Add a new file using the already compressed data from a ZIP file being read and renames it, this @@ -835,7 +853,7 @@ impl ZipWriter { let central_start = writer.stream_position()?; for file in self.files.iter() { - write_central_directory_header(writer, file)?; + write_central_directory_header(writer, &file.borrow())?; } let central_size = writer.stream_position()? - central_start; @@ -881,9 +899,9 @@ impl ZipWriter { Ok(()) } - fn data_by_name(&self, name: &str) -> ZipResult<&ZipFileData> { + fn data_by_name(&self, name: &str) -> ZipResult<&RefCell> { for file in self.files.iter() { - if file.file_name == name { + if file.borrow().file_name == name { return Ok(file); } } @@ -897,10 +915,11 @@ impl ZipWriter { /// some other software (e.g. Minecraft) will refuse to extract a file copied this way. pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; - let src_data = self.data_by_name(src_name)?; + let src_data = self.data_by_name(src_name)?.borrow(); let mut dest_data = src_data.to_owned(); + drop(src_data); dest_data.file_name = dest_name.into(); - self.files.push(dest_data); + self.files.push(RefCell::new(dest_data)); Ok(()) } } From adaacd90f074d85ddfc96e1bc93efa1a6f6b0f4a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 13:58:17 -0700 Subject: [PATCH 041/281] Wrap files in Rc> --- src/write.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index 3962b80a..f9d49fa7 100644 --- a/src/write.rs +++ b/src/write.rs @@ -14,6 +14,7 @@ use std::io; use std::io::prelude::*; use std::io::{BufReader, SeekFrom}; use std::mem; +use std::rc::Rc; #[cfg(any( feature = "deflate", @@ -49,6 +50,8 @@ enum GenericZipWriter { pub(crate) mod zip_writer { use super::*; use std::cell::RefCell; + use std::rc::Rc; + /// ZIP archive generator /// /// Handles the bookkeeping involved in building an archive, and provides an @@ -79,7 +82,7 @@ pub(crate) mod zip_writer { /// ``` pub struct ZipWriter { pub(super) inner: GenericZipWriter, - pub(super) files: Vec>, + pub(super) files: Vec>>, pub(super) stats: ZipWriterStats, pub(super) writing_to_file: bool, pub(super) writing_to_extra_field: bool, @@ -283,7 +286,9 @@ impl ZipWriter { } let files = (0..number_of_files) - .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset).map(RefCell::new)) + .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset) + .map(RefCell::new) + .map(Rc::new)) .collect::, _>>()?; let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it @@ -431,7 +436,7 @@ impl ZipWriter { self.stats.bytes_written = 0; self.stats.hasher = Hasher::new(); - self.files.push(RefCell::new(file)); + self.files.push(Rc::new(RefCell::new(file))); } Ok(()) @@ -919,7 +924,7 @@ impl ZipWriter { let mut dest_data = src_data.to_owned(); drop(src_data); dest_data.file_name = dest_name.into(); - self.files.push(RefCell::new(dest_data)); + self.files.push(Rc::new(RefCell::new(dest_data))); Ok(()) } } From 29b3052e1875cf12dc36bc91832bfc7f75afb1b3 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 14:05:55 -0700 Subject: [PATCH 042/281] Use BTreeMap to look up files by name --- src/write.rs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/write.rs b/src/write.rs index f9d49fa7..462b9d14 100644 --- a/src/write.rs +++ b/src/write.rs @@ -8,6 +8,7 @@ use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSIO use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crc32fast::Hasher; use std::cell::RefCell; +use std::collections::BTreeMap; use std::convert::TryInto; use std::default::Default; use std::io; @@ -50,6 +51,7 @@ enum GenericZipWriter { pub(crate) mod zip_writer { use super::*; use std::cell::RefCell; + use std::collections::BTreeMap; use std::rc::Rc; /// ZIP archive generator @@ -83,6 +85,7 @@ pub(crate) mod zip_writer { pub struct ZipWriter { pub(super) inner: GenericZipWriter, pub(super) files: Vec>>, + pub(super) files_by_name: BTreeMap>>, pub(super) stats: ZipWriterStats, pub(super) writing_to_file: bool, pub(super) writing_to_extra_field: bool, @@ -291,11 +294,17 @@ impl ZipWriter { .map(Rc::new)) .collect::, _>>()?; + let mut files_by_name = BTreeMap::new(); + for file in files.iter() { + files_by_name.insert(file.borrow().file_name.to_owned(), file.to_owned()); + } + let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it Ok(ZipWriter { inner: GenericZipWriter::Storer(readwriter), files, + files_by_name, stats: Default::default(), writing_to_file: false, writing_to_extra_field: false, @@ -357,6 +366,7 @@ impl ZipWriter { ZipWriter { inner: GenericZipWriter::Storer(inner), files: Vec::new(), + files_by_name: BTreeMap::new(), stats: Default::default(), writing_to_file: false, writing_to_extra_field: false, @@ -403,6 +413,7 @@ impl ZipWriter { { let writer = self.inner.get_plain(); let header_start = writer.stream_position()?; + let name = name.into(); let permissions = options.permissions.unwrap_or(0o100644); let mut file = ZipFileData { @@ -416,7 +427,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: name.into(), + file_name: name.to_owned(), file_name_raw: Vec::new(), // Never used for saving extra_field: Vec::new(), file_comment: String::new(), @@ -436,7 +447,9 @@ impl ZipWriter { self.stats.bytes_written = 0; self.stats.hasher = Hasher::new(); - self.files.push(Rc::new(RefCell::new(file))); + let file = Rc::new(RefCell::new(file)); + self.files.push(file.to_owned()); + self.files_by_name.insert(name, file); } Ok(()) @@ -904,13 +917,8 @@ impl ZipWriter { Ok(()) } - fn data_by_name(&self, name: &str) -> ZipResult<&RefCell> { - for file in self.files.iter() { - if file.borrow().file_name == name { - return Ok(file); - } - } - Err(ZipError::FileNotFound) + fn data_by_name(&self, name: &str) -> ZipResult<&Rc>> { + self.files_by_name.get(name).ok_or(ZipError::FileNotFound) } /// Adds another entry to the central directory referring to the same content as an existing @@ -924,7 +932,9 @@ impl ZipWriter { let mut dest_data = src_data.to_owned(); drop(src_data); dest_data.file_name = dest_name.into(); - self.files.push(Rc::new(RefCell::new(dest_data))); + let dest_data = Rc::new(RefCell::new(dest_data)); + self.files.push(dest_data.to_owned()); + self.files_by_name.insert(dest_name.into(), dest_data); Ok(()) } } From eaa2d26ca9e26a8b18506a5413afd51a9a5b5f19 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 15:01:56 -0700 Subject: [PATCH 043/281] Use HashMap rather than BTreeMap, and check for duplicate file names --- src/write.rs | 55 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/src/write.rs b/src/write.rs index 462b9d14..581cf1f7 100644 --- a/src/write.rs +++ b/src/write.rs @@ -8,7 +8,7 @@ use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSIO use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crc32fast::Hasher; use std::cell::RefCell; -use std::collections::BTreeMap; +use std::collections::HashMap; use std::convert::TryInto; use std::default::Default; use std::io; @@ -47,13 +47,13 @@ enum GenericZipWriter { #[cfg(feature = "zstd")] Zstd(ZstdEncoder<'static, W>), } + +type FileRef = Rc>; + // Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely pub(crate) mod zip_writer { use super::*; - use std::cell::RefCell; - use std::collections::BTreeMap; - use std::rc::Rc; - + use std::collections::HashMap; /// ZIP archive generator /// /// Handles the bookkeeping involved in building an archive, and provides an @@ -84,8 +84,8 @@ pub(crate) mod zip_writer { /// ``` pub struct ZipWriter { pub(super) inner: GenericZipWriter, - pub(super) files: Vec>>, - pub(super) files_by_name: BTreeMap>>, + pub(super) files: Vec, + pub(super) files_by_name: HashMap, pub(super) stats: ZipWriterStats, pub(super) writing_to_file: bool, pub(super) writing_to_extra_field: bool, @@ -289,12 +289,14 @@ impl ZipWriter { } let files = (0..number_of_files) - .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset) - .map(RefCell::new) - .map(Rc::new)) + .map(|_| { + central_header_to_zip_file(&mut readwriter, archive_offset) + .map(RefCell::new) + .map(Rc::new) + }) .collect::, _>>()?; - let mut files_by_name = BTreeMap::new(); + let mut files_by_name = HashMap::new(); for file in files.iter() { files_by_name.insert(file.borrow().file_name.to_owned(), file.to_owned()); } @@ -366,7 +368,7 @@ impl ZipWriter { ZipWriter { inner: GenericZipWriter::Storer(inner), files: Vec::new(), - files_by_name: BTreeMap::new(), + files_by_name: HashMap::new(), stats: Default::default(), writing_to_file: false, writing_to_extra_field: false, @@ -411,12 +413,11 @@ impl ZipWriter { }); { - let writer = self.inner.get_plain(); - let header_start = writer.stream_position()?; + let header_start = self.inner.get_plain().stream_position()?; let name = name.into(); let permissions = options.permissions.unwrap_or(0o100644); - let mut file = ZipFileData { + let file = ZipFileData { system: System::Unix, version_made_by: DEFAULT_VERSION, encrypted: false, @@ -427,7 +428,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: name.to_owned(), + file_name: name, file_name_raw: Vec::new(), // Never used for saving extra_field: Vec::new(), file_comment: String::new(), @@ -438,23 +439,33 @@ impl ZipWriter { large_file: options.large_file, aes_mode: None, }; - write_local_file_header(writer, &file)?; + let file = self.insert_file_data(file)?; + let writer = self.inner.get_plain(); + write_local_file_header(writer, &file.borrow())?; let header_end = writer.stream_position()?; self.stats.start = header_end; - *file.data_start.get_mut() = header_end; + *file.borrow_mut().data_start.get_mut() = header_end; self.stats.bytes_written = 0; self.stats.hasher = Hasher::new(); - - let file = Rc::new(RefCell::new(file)); - self.files.push(file.to_owned()); - self.files_by_name.insert(name, file); } Ok(()) } + fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { + let name = &file.file_name; + if self.files_by_name.contains_key(name) { + return Err(ZipError::InvalidArchive("Duplicate filename")); + } + let name = name.to_owned(); + let file = Rc::new(RefCell::new(file)); + self.files.push(file.to_owned()); + self.files_by_name.insert(name, file.to_owned()); + Ok(file) + } + fn finish_file(&mut self) -> ZipResult<()> { if self.writing_to_extra_field { // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. From db9866719bbe4552920966552c0b9a1f57d74ca6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 15:05:53 -0700 Subject: [PATCH 044/281] Add unit test for duplicate file names --- src/write.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/write.rs b/src/write.rs index 581cf1f7..7b069c33 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1636,6 +1636,20 @@ mod test { assert_eq!(second_file_content, RT_TEST_TEXT); } + #[test] + fn duplicate_filenames() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file("foo/bar/test", FileOptions::default()) + .unwrap(); + writer + .write("The quick brown 🦊 jumps over the lazy 🐕".as_bytes()) + .unwrap(); + writer + .start_file("foo/bar/test", FileOptions::default()) + .expect_err("Expected duplicate filename not to be allowed"); + } + #[test] fn path_to_string() { let mut path = std::path::PathBuf::new(); From f9bd7f8c085dbb620acdc36cfcae3d61e916f5e2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 15:07:40 -0700 Subject: [PATCH 045/281] Update CHANGELOG and doc comment --- CHANGELOG.md | 6 +++++- src/write.rs | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d1e4261..9bf7f9eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ ## [0.6.8] +### Added + + - Detects duplicate filenames. + ### Fixed - `deep_copy_file` could set incorrect Unix permissions. @@ -40,4 +44,4 @@ ### Changed - - Improved performance of `deep_copy_file`: it no longer searches for the filename twice. \ No newline at end of file + - Improved performance of `deep_copy_file` by using a HashMap and eliminating a redundant search. \ No newline at end of file diff --git a/src/write.rs b/src/write.rs index 7b069c33..62d763c4 100644 --- a/src/write.rs +++ b/src/write.rs @@ -494,7 +494,8 @@ impl ZipWriter { Ok(()) } - /// Create a file in the archive and start writing its' contents. + /// Create a file in the archive and start writing its' contents. The file must not have the + /// same name as a file already in the archive. /// /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] pub fn start_file(&mut self, name: S, mut options: FileOptions) -> ZipResult<()> From d26abaa297b62f49b44ad07fc5fd601bd0e49058 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 15:42:26 -0700 Subject: [PATCH 046/281] Fix Clippy warning --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 62d763c4..caac7a52 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1644,7 +1644,7 @@ mod test { .start_file("foo/bar/test", FileOptions::default()) .unwrap(); writer - .write("The quick brown 🦊 jumps over the lazy 🐕".as_bytes()) + .write_all("The quick brown 🦊 jumps over the lazy 🐕".as_bytes()) .unwrap(); writer .start_file("foo/bar/test", FileOptions::default()) From 81de6887cc0f7bbcfde8a519523e79f2214a9a40 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 15:44:22 -0700 Subject: [PATCH 047/281] Bump version to 0.6.8 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fbb0ece4..4ec3d3a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.7" +version = "0.6.8" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index b88e18a2..9604c48b 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.7" +zip_next = "0.6.8" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.7", default-features = false } +zip_next = { version = "0.6.8", default-features = false } ``` The features available are: From 6290a028a1a99a0a2e52c30bfadf8a189b13167a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 16:39:48 -0700 Subject: [PATCH 048/281] Refactor: store index rather than Rc --- src/write.rs | 80 ++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 50 deletions(-) diff --git a/src/write.rs b/src/write.rs index caac7a52..65f0bd92 100644 --- a/src/write.rs +++ b/src/write.rs @@ -7,7 +7,6 @@ use crate::spec; use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crc32fast::Hasher; -use std::cell::RefCell; use std::collections::HashMap; use std::convert::TryInto; use std::default::Default; @@ -15,7 +14,6 @@ use std::io; use std::io::prelude::*; use std::io::{BufReader, SeekFrom}; use std::mem; -use std::rc::Rc; #[cfg(any( feature = "deflate", @@ -48,7 +46,7 @@ enum GenericZipWriter { Zstd(ZstdEncoder<'static, W>), } -type FileRef = Rc>; +type FileRef = ZipFileData; // Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely pub(crate) mod zip_writer { @@ -85,7 +83,7 @@ pub(crate) mod zip_writer { pub struct ZipWriter { pub(super) inner: GenericZipWriter, pub(super) files: Vec, - pub(super) files_by_name: HashMap, + pub(super) files_by_name: HashMap, pub(super) stats: ZipWriterStats, pub(super) writing_to_file: bool, pub(super) writing_to_extra_field: bool, @@ -220,18 +218,13 @@ impl Write for ZipWriter { match self.inner.ref_mut() { Some(ref mut w) => { if self.writing_to_extra_field { - self.files - .last_mut() - .unwrap() - .borrow_mut() - .extra_field - .write(buf) + self.files.last_mut().unwrap().extra_field.write(buf) } else { let write_result = w.write(buf); if let Ok(count) = write_result { self.stats.update(&buf[0..count]); if self.stats.bytes_written > spec::ZIP64_BYTES_THR - && !self.files.last_mut().unwrap().borrow().large_file + && !self.files.last_mut().unwrap().large_file { let _inner = mem::replace(&mut self.inner, GenericZipWriter::Closed); return Err(io::Error::new( @@ -289,16 +282,12 @@ impl ZipWriter { } let files = (0..number_of_files) - .map(|_| { - central_header_to_zip_file(&mut readwriter, archive_offset) - .map(RefCell::new) - .map(Rc::new) - }) + .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset)) .collect::, _>>()?; let mut files_by_name = HashMap::new(); - for file in files.iter() { - files_by_name.insert(file.borrow().file_name.to_owned(), file.to_owned()); + for (index, file) in files.iter().enumerate() { + files_by_name.insert(file.file_name.to_owned(), index); } let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it @@ -323,8 +312,8 @@ impl ZipWriter { pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; let write_position = self.inner.get_plain().stream_position()?; - let src_data_rc = self.data_by_name(src_name)?; - let src_data = src_data_rc.borrow(); + let src_index = self.index_by_name(src_name)?; + let src_data = &self.files[src_index]; let data_start = src_data.data_start.load(); let compressed_size = src_data.compressed_size; if compressed_size > write_position - data_start { @@ -344,9 +333,8 @@ impl ZipWriter { compressed_size, uncompressed_size, }; - drop(src_data); let mut reader = BufReader::new(ZipFileReader::Raw(find_content( - &src_data_rc.clone().borrow(), + src_data, self.inner.get_plain(), )?)); let mut copy = Vec::with_capacity(compressed_size as usize); @@ -439,13 +427,14 @@ impl ZipWriter { large_file: options.large_file, aes_mode: None, }; - let file = self.insert_file_data(file)?; + let index = self.insert_file_data(file)?; + let file = &mut self.files[index]; let writer = self.inner.get_plain(); - write_local_file_header(writer, &file.borrow())?; + write_local_file_header(writer, file)?; let header_end = writer.stream_position()?; self.stats.start = header_end; - *file.borrow_mut().data_start.get_mut() = header_end; + *file.data_start.get_mut() = header_end; self.stats.bytes_written = 0; self.stats.hasher = Hasher::new(); @@ -454,16 +443,16 @@ impl ZipWriter { Ok(()) } - fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { + fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { let name = &file.file_name; if self.files_by_name.contains_key(name) { return Err(ZipError::InvalidArchive("Duplicate filename")); } let name = name.to_owned(); - let file = Rc::new(RefCell::new(file)); - self.files.push(file.to_owned()); - self.files_by_name.insert(name, file.to_owned()); - Ok(file) + self.files.push(file); + let index = self.files.len() - 1; + self.files_by_name.insert(name, index); + Ok(index) } fn finish_file(&mut self) -> ZipResult<()> { @@ -477,7 +466,7 @@ impl ZipWriter { if !self.writing_raw { let mut file = match self.files.last_mut() { None => return Ok(()), - Some(f) => f.borrow_mut(), + Some(f) => f, }; file.crc32 = self.stats.hasher.clone().finalize(); file.uncompressed_size = self.stats.bytes_written; @@ -485,7 +474,7 @@ impl ZipWriter { let file_end = writer.stream_position()?; file.compressed_size = file_end - self.stats.start; - update_local_file_header(writer, &file)?; + update_local_file_header(writer, file)?; writer.seek(SeekFrom::Start(file_end))?; } @@ -637,7 +626,7 @@ impl ZipWriter { self.start_entry(name, options, None)?; self.writing_to_file = true; self.writing_to_extra_field = true; - Ok(self.files.last().unwrap().borrow().data_start.load()) + Ok(self.files.last().unwrap().data_start.load()) } /// End local and start central extra data. Requires [`ZipWriter::start_file_with_extra_data`]. @@ -645,12 +634,7 @@ impl ZipWriter { /// Returns the final starting offset of the file data. pub fn end_local_start_central_extra_data(&mut self) -> ZipResult { let data_start = self.end_extra_data()?; - self.files - .last_mut() - .unwrap() - .borrow_mut() - .extra_field - .clear(); + self.files.last_mut().unwrap().extra_field.clear(); self.writing_to_extra_field = true; self.writing_to_central_extra_field_only = true; Ok(data_start) @@ -669,9 +653,8 @@ impl ZipWriter { } let file = self.files.last_mut().unwrap(); - validate_extra_data(&file.borrow())?; + validate_extra_data(file)?; - let mut file = file.borrow_mut(); let mut data_start_result = file.data_start.load(); if !self.writing_to_central_extra_field_only { @@ -883,7 +866,7 @@ impl ZipWriter { let central_start = writer.stream_position()?; for file in self.files.iter() { - write_central_directory_header(writer, &file.borrow())?; + write_central_directory_header(writer, file)?; } let central_size = writer.stream_position()? - central_start; @@ -929,8 +912,8 @@ impl ZipWriter { Ok(()) } - fn data_by_name(&self, name: &str) -> ZipResult<&Rc>> { - self.files_by_name.get(name).ok_or(ZipError::FileNotFound) + fn index_by_name(&self, name: &str) -> ZipResult { + Ok(*self.files_by_name.get(name).ok_or(ZipError::FileNotFound)?) } /// Adds another entry to the central directory referring to the same content as an existing @@ -940,13 +923,10 @@ impl ZipWriter { /// some other software (e.g. Minecraft) will refuse to extract a file copied this way. pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; - let src_data = self.data_by_name(src_name)?.borrow(); - let mut dest_data = src_data.to_owned(); - drop(src_data); + let src_index = self.index_by_name(src_name)?; + let mut dest_data = self.files[src_index].to_owned(); dest_data.file_name = dest_name.into(); - let dest_data = Rc::new(RefCell::new(dest_data)); - self.files.push(dest_data.to_owned()); - self.files_by_name.insert(dest_name.into(), dest_data); + self.insert_file_data(dest_data)?; Ok(()) } } From 7bad785f47c765061123ac1e85588192d4c54c73 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 16:45:03 -0700 Subject: [PATCH 049/281] Update CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf7f9eb..dc57cb06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,4 +44,10 @@ ### Changed - - Improved performance of `deep_copy_file` by using a HashMap and eliminating a redundant search. \ No newline at end of file + - Improved performance of `deep_copy_file` by using a HashMap and eliminating a redundant search. + +## [0.6.9] + +### Fixed + + - Fixed an issue that prevented `ZipWriter` from implementing `Send`. From 23a1b3d9f79b6cf7e50358f956367e91e7cf43d8 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 16:45:41 -0700 Subject: [PATCH 050/281] Bump version to 0.6.9 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4ec3d3a9..9af77752 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.8" +version = "0.6.9" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 9604c48b..65f95eb3 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.8" +zip_next = "0.6.9" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.8", default-features = false } +zip_next = { version = "0.6.9", default-features = false } ``` The features available are: From 389bd8575416cab21f62ee58a96c005407b60995 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 16:53:37 -0700 Subject: [PATCH 051/281] Update dependencies --- CHANGELOG.md | 6 ++++++ Cargo.toml | 16 ++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc57cb06..77e8b779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,3 +51,9 @@ ### Fixed - Fixed an issue that prevented `ZipWriter` from implementing `Send`. + +## [0.6.10] + +### Changed + + - Updated dependency versions. diff --git a/Cargo.toml b/Cargo.toml index 9af77752..01db9e4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,24 +14,24 @@ edition = "2021" [dependencies] aes = { version = "0.8.2", optional = true } byteorder = "1.4.3" -bzip2 = { version = "0.4.3", optional = true } +bzip2 = { version = "0.4.4", optional = true } constant_time_eq = { version = "0.2.5", optional = true } crc32fast = "1.3.2" -flate2 = { version = "1.0.23", default-features = false, optional = true } +flate2 = { version = "1.0.26", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } pbkdf2 = {version = "0.12.1", optional = true } -sha1 = {version = "0.10.1", optional = true } -time = { version = "0.3.7", optional = true, default-features = false, features = ["std"] } +sha1 = {version = "0.10.5", optional = true } +time = { version = "0.3.20", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] -crossbeam-utils = "0.8.8" +crossbeam-utils = "0.8.15" [dev-dependencies] bencher = "0.1.5" -getrandom = "0.2.5" -walkdir = "2.3.2" -time = { version = "0.3.7", features = ["formatting", "macros"] } +getrandom = "0.2.9" +walkdir = "2.3.3" +time = { version = "0.3.20", features = ["formatting", "macros"] } [features] aes-crypto = [ "aes", "constant_time_eq", "hmac", "pbkdf2", "sha1" ] From b52464cf250cf298e4c851bfb3bc89a17ae22753 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 17:00:30 -0700 Subject: [PATCH 052/281] Bump version to 0.6.10 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 01db9e4e..1fa7e8f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.9" +version = "0.6.10" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 65f95eb3..f3fb31b4 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.9" +zip_next = "0.6.10" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.9", default-features = false } +zip_next = { version = "0.6.10", default-features = false } ``` The features available are: From 6b4a07f2a7399a3ec53ca03d7f6c5f975bff428b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 17:46:48 -0700 Subject: [PATCH 053/281] Bug fix: don't switch_to when deep copying --- src/write.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 65f0bd92..35374aa6 100644 --- a/src/write.rs +++ b/src/write.rs @@ -341,7 +341,6 @@ impl ZipWriter { reader.read_to_end(&mut copy)?; drop(reader); self.start_entry(dest_name, options, Some(raw_values))?; - self.inner.switch_to(CompressionMethod::Stored, None)?; self.writing_to_file = true; self.writing_raw = true; Ok(self.write_all(©)?) From ed5c76d2e87b8649df481d4e1ba4df87f4ec84bb Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 17:54:31 -0700 Subject: [PATCH 054/281] Perform copy earlier, to catch recent bug --- tests/end_to_end.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 57f7a8e7..047b5955 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -103,6 +103,17 @@ fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, sha .compression_method(method) .unix_permissions(0o755); + zip.start_file(ENTRY_NAME, options).unwrap(); + zip.write_all(LOREM_IPSUM).unwrap(); + + if shallow_copy { + zip.shallow_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME) + .unwrap(); + } else { + zip.deep_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME) + .unwrap(); + } + zip.start_file("test/☃.txt", options).unwrap(); zip.write_all(b"Hello, World!\n").unwrap(); @@ -115,17 +126,6 @@ fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, sha zip.end_extra_data().unwrap(); zip.write_all(b"Hello, World! Again.\n").unwrap(); - zip.start_file(ENTRY_NAME, options).unwrap(); - zip.write_all(LOREM_IPSUM).unwrap(); - - if shallow_copy { - zip.shallow_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME) - .unwrap(); - } else { - zip.deep_copy_file(ENTRY_NAME, INTERNAL_COPY_ENTRY_NAME) - .unwrap(); - } - zip.finish().unwrap(); } From 5d9296244cd235a430e9878c89b4e94d5020981b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 18:03:11 -0700 Subject: [PATCH 055/281] Add debug_assert! that file end isn't before start --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index 35374aa6..d0f5165a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -471,6 +471,7 @@ impl ZipWriter { file.uncompressed_size = self.stats.bytes_written; let file_end = writer.stream_position()?; + debug_assert!(file_end >= self.stats.start); file.compressed_size = file_end - self.stats.start; update_local_file_header(writer, file)?; From 4ae76f6e0723599585ea61e76e43d6ea4d0bd4ce Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 20:39:11 -0700 Subject: [PATCH 056/281] Bump version to 0.6.11 and update CHANGELOG --- CHANGELOG.md | 4 ++++ Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77e8b779..7d5d12ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,3 +57,7 @@ ### Changed - Updated dependency versions. + +## [0.6.11] + + - Fixed a bug that could cause later writes to fail after a `deep_copy_file` call. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 1fa7e8f0..730382e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.10" +version = "0.6.11" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index f3fb31b4..082a4883 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.10" +zip_next = "0.6.11" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.10", default-features = false } +zip_next = { version = "0.6.11", default-features = false } ``` The features available are: From 16af9ae253dbb0a4201b5429c047d14dfb5e8aa6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 20:44:27 -0700 Subject: [PATCH 057/281] Fix Clippy warning and eliminate type alias that no longer simplifies --- src/write.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index d0f5165a..aebb1724 100644 --- a/src/write.rs +++ b/src/write.rs @@ -46,8 +46,6 @@ enum GenericZipWriter { Zstd(ZstdEncoder<'static, W>), } -type FileRef = ZipFileData; - // Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely pub(crate) mod zip_writer { use super::*; @@ -82,7 +80,7 @@ pub(crate) mod zip_writer { /// ``` pub struct ZipWriter { pub(super) inner: GenericZipWriter, - pub(super) files: Vec, + pub(super) files: Vec, pub(super) files_by_name: HashMap, pub(super) stats: ZipWriterStats, pub(super) writing_to_file: bool, @@ -463,7 +461,7 @@ impl ZipWriter { let writer = self.inner.get_plain(); if !self.writing_raw { - let mut file = match self.files.last_mut() { + let file = match self.files.last_mut() { None => return Ok(()), Some(f) => f, }; From ccd591ab14a24b37418817f22928d623dd2f7647 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 20:46:16 -0700 Subject: [PATCH 058/281] Update CHANGELOG and bump version to 0.6.12 --- CHANGELOG.md | 10 +++++++++- Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5d12ce..12f865db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,4 +60,12 @@ ## [0.6.11] - - Fixed a bug that could cause later writes to fail after a `deep_copy_file` call. \ No newline at end of file +### Fixed + + - Fixed a bug that could cause later writes to fail after a `deep_copy_file` call. + +## [0.6.12] + +### Fixed + + - Fixed a Clippy warning that was missed during the last release. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 730382e4..1aa26571 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.11" +version = "0.6.12" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 082a4883..baa64d00 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.11" +zip_next = "0.6.12" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.11", default-features = false } +zip_next = { version = "0.6.12", default-features = false } ``` The features available are: From cdfe103f94112c5d51155c8d01fe4c2899127940 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 21:05:42 -0700 Subject: [PATCH 059/281] Bug fix? --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index aebb1724..4e1714ea 100644 --- a/src/write.rs +++ b/src/write.rs @@ -338,6 +338,7 @@ impl ZipWriter { let mut copy = Vec::with_capacity(compressed_size as usize); reader.read_to_end(&mut copy)?; drop(reader); + self.inner.get_plain().seek(SeekFrom::Start(write_position))?; self.start_entry(dest_name, options, Some(raw_values))?; self.writing_to_file = true; self.writing_raw = true; From db4afdb36c6f235cfeb5e97e00460789c2cd8ecc Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 21:07:33 -0700 Subject: [PATCH 060/281] Cargo fmt --- src/write.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 4e1714ea..f469612e 100644 --- a/src/write.rs +++ b/src/write.rs @@ -338,7 +338,9 @@ impl ZipWriter { let mut copy = Vec::with_capacity(compressed_size as usize); reader.read_to_end(&mut copy)?; drop(reader); - self.inner.get_plain().seek(SeekFrom::Start(write_position))?; + self.inner + .get_plain() + .seek(SeekFrom::Start(write_position))?; self.start_entry(dest_name, options, Some(raw_values))?; self.writing_to_file = true; self.writing_raw = true; From 90b89b546054d55ec74723f7a8ff6a8147f1e34f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 21:19:31 -0700 Subject: [PATCH 061/281] WIP: Write fuzzing --- .github/workflows/ci.yaml | 21 ++++- Cargo.toml | 4 + fuzz/Cargo.toml | 7 ++ fuzz/fuzz_targets/fuzz_write.rs | 67 ++++++++++++++ src/compression.rs | 1 + src/types.rs | 6 ++ src/write.rs | 152 ++++++++++++++++++-------------- 7 files changed, 189 insertions(+), 69 deletions(-) create mode 100644 fuzz/fuzz_targets/fuzz_write.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b761b9a4..8ede65df 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,7 +75,7 @@ jobs: - name: Docs run: cargo doc - fuzz: + fuzz_read: runs-on: ubuntu-latest steps: @@ -93,3 +93,22 @@ jobs: - name: run fuzz run: | cargo fuzz run fuzz_read -- -timeout=1s -runs=10000000 + + fuzz_write: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + + - run: cargo install cargo-fuzz + - name: compile fuzz + run: | + cargo fuzz build fuzz_write + - name: run fuzz + run: | + cargo fuzz run fuzz_write -- -timeout=1s -runs=10000000 diff --git a/Cargo.toml b/Cargo.toml index 1aa26571..a165a8e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,10 +23,14 @@ pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.20", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } +replace_with = "0.1.7" [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" +[target.'cfg(fuzzing)'.dependencies] +arbitrary = { version = "1.3.0", features = ["derive"] } + [dev-dependencies] bencher = "0.1.5" getrandom = "0.2.9" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 4349eb08..f63ff859 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,6 +10,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" +arbitrary = { version = "1.3.0", features = ["derive"] } [dependencies.zip_next] path = ".." @@ -23,3 +24,9 @@ name = "fuzz_read" path = "fuzz_targets/fuzz_read.rs" test = false doc = false + +[[bin]] +name = "fuzz_write" +path = "fuzz_targets/fuzz_write.rs" +test = false +doc = false diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs new file mode 100644 index 00000000..3b2a73d6 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -0,0 +1,67 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use arbitrary::Arbitrary; +use std::io::{Cursor, Read, Seek, Write}; + +#[derive(Arbitrary,Debug)] +pub struct File { + pub name: String, + pub contents: Vec +} + +#[derive(Arbitrary,Debug)] +pub enum FileOperation { + Write { + file: File, + options: zip_next::write::FileOptions + }, + ShallowCopy { + base: Box, + new_name: String + }, + DeepCopy { + base: Box, + new_name: String + }, +} + +impl FileOperation { + pub fn get_name(&self) -> String { + match self { + FileOperation::Write {file, ..} => &file.name, + FileOperation::ShallowCopy {new_name, ..} => new_name, + FileOperation::DeepCopy {new_name, ..} => new_name + }.to_owned() + } +} + +fn do_operation(writer: &mut zip_next::ZipWriter, + operation: &FileOperation) -> Result<(), Box> + where T: Read + Write + Seek { + match operation { + FileOperation::Write {file, mut options} => { + if (*file).contents.len() >= u32::MAX as usize { + options = options.large_file(true); + } + writer.start_file(file.name.to_owned(), options)?; + writer.write_all(file.contents.as_slice())?; + } + FileOperation::ShallowCopy {base, new_name} => { + do_operation(writer, base)?; + writer.shallow_copy_file(&base.get_name(), new_name)?; + } + FileOperation::DeepCopy {base, new_name} => { + do_operation(writer, base)?; + writer.deep_copy_file(&base.get_name(), new_name)?; + } + } + Ok(()) +} + +fuzz_target!(|data: Vec| { + let mut writer = zip_next::ZipWriter::new(Cursor::new(Vec::new())); + for operation in data { + let _ = do_operation(&mut writer, &operation); + } + writer.finish().unwrap(); +}); \ No newline at end of file diff --git a/src/compression.rs b/src/compression.rs index baec9399..43366ed1 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -11,6 +11,7 @@ use std::fmt; /// When creating ZIP files, you may choose the method to use with /// [`crate::write::FileOptions::compression_method`] #[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] #[non_exhaustive] pub enum CompressionMethod { /// Store the file as is diff --git a/src/types.rs b/src/types.rs index 85ce6427..23e28ba1 100644 --- a/src/types.rs +++ b/src/types.rs @@ -92,6 +92,7 @@ impl System { /// Modern zip files store more precise timestamps, which are ignored by [`crate::read::ZipArchive`], /// so keep in mind that these timestamps are unreliable. [We're working on this](https://github.com/zip-rs/zip/issues/156#issuecomment-652981904). #[derive(Debug, Clone, Copy)] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] pub struct DateTime { year: u16, month: u8, @@ -173,6 +174,11 @@ impl DateTime { } } + /// Indicates whether this date and time can be written to a zip archive. + pub fn is_valid(&self) -> bool { + DateTime::from_date_and_time(self.year, self.month, self.day, self.hour, self.minute, self.second).is_ok() + } + #[cfg(feature = "time")] /// Converts a OffsetDateTime object to a DateTime /// diff --git a/src/write.rs b/src/write.rs index f469612e..62e5711e 100644 --- a/src/write.rs +++ b/src/write.rs @@ -24,6 +24,7 @@ use flate2::write::DeflateEncoder; #[cfg(feature = "bzip2")] use bzip2::write::BzEncoder; +use replace_with::{replace_with, replace_with_and_return}; #[cfg(feature = "time")] use time::OffsetDateTime; @@ -91,6 +92,7 @@ pub(crate) mod zip_writer { } } pub use zip_writer::ZipWriter; +use crate::write::GenericZipWriter::{Closed, Storer}; #[derive(Default)] struct ZipWriterStats { @@ -106,13 +108,14 @@ struct ZipRawValues { } /// Metadata for a file to be written -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] pub struct FileOptions { - compression_method: CompressionMethod, - compression_level: Option, - last_modified_time: DateTime, - permissions: Option, - large_file: bool, + pub(crate) compression_method: CompressionMethod, + pub(crate) compression_level: Option, + pub(crate) last_modified_time: DateTime, + pub(crate) permissions: Option, + pub(crate) large_file: bool, } impl FileOptions { @@ -224,7 +227,8 @@ impl Write for ZipWriter { if self.stats.bytes_written > spec::ZIP64_BYTES_THR && !self.files.last_mut().unwrap().large_file { - let _inner = mem::replace(&mut self.inner, GenericZipWriter::Closed); + self.finish_file()?; + self.files.pop(); return Err(io::Error::new( io::ErrorKind::Other, "Large file option has not been set", @@ -236,7 +240,7 @@ impl Write for ZipWriter { } None => Err(io::Error::new( io::ErrorKind::BrokenPipe, - "ZipWriter was already closed", + "write(): ZipWriter was already closed", )), } } @@ -246,7 +250,7 @@ impl Write for ZipWriter { Some(ref mut w) => w.flush(), None => Err(io::Error::new( io::ErrorKind::BrokenPipe, - "ZipWriter was already closed", + "flush(): ZipWriter was already closed", )), } } @@ -504,6 +508,9 @@ impl ZipWriter { if options.permissions.is_none() { options.permissions = Some(0o644); } + if !options.last_modified_time.is_valid() { + options.last_modified_time = FileOptions::default().last_modified_time; + } *options.permissions.as_mut().unwrap() |= ffi::S_IFREG; } @@ -950,7 +957,11 @@ impl GenericZipWriter { ) -> ZipResult<()> { match self.current_compression() { Some(method) if method == compression => return Ok(()), - None => { + _ => {} + } + + match self { + GenericZipWriter::Closed => { return Err(io::Error::new( io::ErrorKind::BrokenPipe, "ZipWriter was already closed", @@ -959,20 +970,19 @@ impl GenericZipWriter { } _ => {} } - - let bare = match mem::replace(self, GenericZipWriter::Closed) { - GenericZipWriter::Storer(w) => w, + let bare: &mut W = match self { + Storer(w) => &mut unsafe { mem::transmute_copy::(w) }, #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] - GenericZipWriter::Deflater(w) => w.finish()?, + GenericZipWriter::Deflater(w) => &mut w.finish()?, #[cfg(feature = "bzip2")] - GenericZipWriter::Bzip2(w) => w.finish()?, + GenericZipWriter::Bzip2(w) => &mut w.finish()?, #[cfg(feature = "zstd")] - GenericZipWriter::Zstd(w) => w.finish()?, - GenericZipWriter::Closed => { + GenericZipWriter::Zstd(w) => &mut w.finish()?, + Closed => { return Err(io::Error::new( io::ErrorKind::BrokenPipe, "ZipWriter was already closed", @@ -981,81 +991,87 @@ impl GenericZipWriter { } }; - *self = { - #[allow(deprecated)] - match compression { - CompressionMethod::Stored => { - if compression_level.is_some() { - return Err(ZipError::UnsupportedArchive( - "Unsupported compression level", - )); - } - - GenericZipWriter::Storer(bare) + #[allow(deprecated)] + match compression { + CompressionMethod::Stored => { + if compression_level.is_some() { + return Err(ZipError::UnsupportedArchive( + "Unsupported compression level", + )); } - #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" - ))] - CompressionMethod::Deflated => GenericZipWriter::Deflater(DeflateEncoder::new( - bare, + + let _ = mem::replace(self, Storer(unsafe { mem::transmute_copy::(bare) })); + Ok(()) + } + #[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" + ))] + CompressionMethod::Deflated => { + let _ = mem::replace(self, GenericZipWriter::Deflater(DeflateEncoder::new( + unsafe { mem::transmute_copy::(bare) }, flate2::Compression::new( clamp_opt( compression_level .unwrap_or(flate2::Compression::default().level() as i32), deflate_compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32, + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32, ), - )), - #[cfg(feature = "bzip2")] - CompressionMethod::Bzip2 => GenericZipWriter::Bzip2(BzEncoder::new( - bare, + ))); + Ok(()) + }, + #[cfg(feature = "bzip2")] + CompressionMethod::Bzip2 => { + let _ = mem::replace(self, GenericZipWriter::Bzip2(BzEncoder::new( + unsafe { mem::transmute_copy::(bare) }, bzip2::Compression::new( clamp_opt( compression_level .unwrap_or(bzip2::Compression::default().level() as i32), bzip2_compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32, + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32, ), - )), - CompressionMethod::AES => { - return Err(ZipError::UnsupportedArchive( - "AES compression is not supported for writing", - )) - } - #[cfg(feature = "zstd")] - CompressionMethod::Zstd => GenericZipWriter::Zstd( + ))); + Ok(()) + }, + CompressionMethod::AES => { + return Err(ZipError::UnsupportedArchive( + "AES compression is not supported for writing", + )) + } + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => { + let _ = mem::replace(self, GenericZipWriter::Zstd( ZstdEncoder::new( - bare, + unsafe { mem::transmute_copy::(bare) }, clamp_opt( compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL), zstd::compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))?, + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))?, ) - .unwrap(), - ), - CompressionMethod::Unsupported(..) => { - return Err(ZipError::UnsupportedArchive("Unsupported compression")) - } + .unwrap(), + )); + Ok(()) + }, + CompressionMethod::Unsupported(..) => { + return Err(ZipError::UnsupportedArchive("Unsupported compression")); } - }; - - Ok(()) + } } fn ref_mut(&mut self) -> Option<&mut dyn Write> { match *self { - GenericZipWriter::Storer(ref mut w) => Some(w as &mut dyn Write), + Storer(ref mut w) => Some(w as &mut dyn Write), #[cfg(any( feature = "deflate", feature = "deflate-miniz", @@ -1066,7 +1082,7 @@ impl GenericZipWriter { GenericZipWriter::Bzip2(ref mut w) => Some(w as &mut dyn Write), #[cfg(feature = "zstd")] GenericZipWriter::Zstd(ref mut w) => Some(w as &mut dyn Write), - GenericZipWriter::Closed => None, + Closed => None, } } @@ -1076,7 +1092,7 @@ impl GenericZipWriter { fn get_plain(&mut self) -> &mut W { match *self { - GenericZipWriter::Storer(ref mut w) => w, + Storer(ref mut w) => w, _ => panic!("Should have switched to stored beforehand"), } } From 8c7192a750188e557032b0d82269c67e3ad0fb4e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 21:25:01 -0700 Subject: [PATCH 062/281] Run CI on push to any branch --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b761b9a4..1acbb636 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,8 +3,6 @@ name: CI on: pull_request: push: - branches: - - master env: RUSTFLAGS: -Dwarnings From c284de1a2c134472563839bcad2d7bac2737065a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 29 Apr 2023 21:26:34 -0700 Subject: [PATCH 063/281] Bump version to 0.6.13 and update CHANGELOG --- CHANGELOG.md | 8 +++++++- Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f865db..ab5785f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,4 +68,10 @@ ### Fixed - - Fixed a Clippy warning that was missed during the last release. \ No newline at end of file + - Fixed a Clippy warning that was missed during the last release. + +## [0.6.13] + +### Fixed + +- Fixed a possible bug in deep_copy_file. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 1aa26571..3e9dbd2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.12" +version = "0.6.13" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index baa64d00..8e886b4e 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.12" +zip_next = "0.6.13" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.12", default-features = false } +zip_next = { version = "0.6.13", default-features = false } ``` The features available are: From 43a9db888680d96f976452c3b6f68b7004b4e7f9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:11:07 -0700 Subject: [PATCH 064/281] Fix bugs where calling start_file with incorrect parameters would close the ZipWriter --- Cargo.toml | 1 - src/write.rs | 180 ++++++++++++++++++++++++--------------------------- 2 files changed, 85 insertions(+), 96 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bc1e12cf..6825e867 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.20", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } -replace_with = "0.1.7" [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" diff --git a/src/write.rs b/src/write.rs index 62e5711e..595a3e7c 100644 --- a/src/write.rs +++ b/src/write.rs @@ -24,7 +24,6 @@ use flate2::write::DeflateEncoder; #[cfg(feature = "bzip2")] use bzip2::write::BzEncoder; -use replace_with::{replace_with, replace_with_and_return}; #[cfg(feature = "time")] use time::OffsetDateTime; @@ -295,7 +294,7 @@ impl ZipWriter { let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it Ok(ZipWriter { - inner: GenericZipWriter::Storer(readwriter), + inner: Storer(readwriter), files, files_by_name, stats: Default::default(), @@ -358,7 +357,7 @@ impl ZipWriter { /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. pub fn new(inner: W) -> ZipWriter { ZipWriter { - inner: GenericZipWriter::Storer(inner), + inner: Storer(inner), files: Vec::new(), files_by_name: HashMap::new(), stats: Default::default(), @@ -824,7 +823,7 @@ impl ZipWriter { /// Note that the zipfile will also be finished on drop. pub fn finish(&mut self) -> ZipResult { self.finalize()?; - let inner = mem::replace(&mut self.inner, GenericZipWriter::Closed); + let inner = mem::replace(&mut self.inner, Closed); Ok(inner.unwrap()) } @@ -961,7 +960,7 @@ impl GenericZipWriter { } match self { - GenericZipWriter::Closed => { + Closed => { return Err(io::Error::new( io::ErrorKind::BrokenPipe, "ZipWriter was already closed", @@ -970,103 +969,94 @@ impl GenericZipWriter { } _ => {} } - let bare: &mut W = match self { - Storer(w) => &mut unsafe { mem::transmute_copy::(w) }, - #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" - ))] - GenericZipWriter::Deflater(w) => &mut w.finish()?, - #[cfg(feature = "bzip2")] - GenericZipWriter::Bzip2(w) => &mut w.finish()?, - #[cfg(feature = "zstd")] - GenericZipWriter::Zstd(w) => &mut w.finish()?, - Closed => { - return Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "ZipWriter was already closed", - ) - .into()) + + let make_new_self: Box GenericZipWriter> = { + #[allow(deprecated)] + match compression { + CompressionMethod::Stored => { + if compression_level.is_some() { + return Err(ZipError::UnsupportedArchive( + "Unsupported compression level", + )); + } + + Box::new(|bare| Storer(bare)) + } + #[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" + ))] + CompressionMethod::Deflated => { + let level = clamp_opt( + compression_level.unwrap_or(flate2::Compression::default().level() as i32), + deflate_compression_level_range() + ).ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; + Box::new(move |bare| + GenericZipWriter::Deflater(DeflateEncoder::new(bare, flate2::Compression::new(level))) + ) + } + #[cfg(feature = "bzip2")] + CompressionMethod::Bzip2 => { + let level = clamp_opt( + compression_level + .unwrap_or(bzip2::Compression::default().level() as i32), + bzip2_compression_level_range(), + ).ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; + Box::new(move |bare| + GenericZipWriter::Bzip2(BzEncoder::new(bare, bzip2::Compression::new(level))) + ) + }, + CompressionMethod::AES => { + return Err(ZipError::UnsupportedArchive( + "AES compression is not supported for writing", + )) + } + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => { + let level = clamp_opt( + compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL), + zstd::compression_level_range(), + ) + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))?; + Box::new(move |bare| + GenericZipWriter::Zstd(ZstdEncoder::new(bare, level).unwrap() + )) + }, + CompressionMethod::Unsupported(..) => { + return Err(ZipError::UnsupportedArchive("Unsupported compression")) + } } }; - #[allow(deprecated)] - match compression { - CompressionMethod::Stored => { - if compression_level.is_some() { - return Err(ZipError::UnsupportedArchive( - "Unsupported compression level", - )); - } - - let _ = mem::replace(self, Storer(unsafe { mem::transmute_copy::(bare) })); - Ok(()) - } + let bare = match mem::replace(self, Closed) { + Storer(w) => w, #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] - CompressionMethod::Deflated => { - let _ = mem::replace(self, GenericZipWriter::Deflater(DeflateEncoder::new( - unsafe { mem::transmute_copy::(bare) }, - flate2::Compression::new( - clamp_opt( - compression_level - .unwrap_or(flate2::Compression::default().level() as i32), - deflate_compression_level_range(), - ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32, - ), - ))); - Ok(()) - }, + GenericZipWriter::Deflater(w) => w.finish()?, #[cfg(feature = "bzip2")] - CompressionMethod::Bzip2 => { - let _ = mem::replace(self, GenericZipWriter::Bzip2(BzEncoder::new( - unsafe { mem::transmute_copy::(bare) }, - bzip2::Compression::new( - clamp_opt( - compression_level - .unwrap_or(bzip2::Compression::default().level() as i32), - bzip2_compression_level_range(), - ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32, - ), - ))); - Ok(()) - }, - CompressionMethod::AES => { - return Err(ZipError::UnsupportedArchive( - "AES compression is not supported for writing", - )) - } + GenericZipWriter::Bzip2(w) => w.finish()?, #[cfg(feature = "zstd")] - CompressionMethod::Zstd => { - let _ = mem::replace(self, GenericZipWriter::Zstd( - ZstdEncoder::new( - unsafe { mem::transmute_copy::(bare) }, - clamp_opt( - compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL), - zstd::compression_level_range(), - ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))?, - ) - .unwrap(), - )); - Ok(()) - }, - CompressionMethod::Unsupported(..) => { - return Err(ZipError::UnsupportedArchive("Unsupported compression")); + GenericZipWriter::Zstd(w) => w.finish()?, + Closed => { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "ZipWriter was already closed", + ) + .into()) } - } + }; + *self = (make_new_self)(bare); + Ok(()) } fn ref_mut(&mut self) -> Option<&mut dyn Write> { @@ -1099,7 +1089,7 @@ impl GenericZipWriter { fn current_compression(&self) -> Option { match *self { - GenericZipWriter::Storer(..) => Some(CompressionMethod::Stored), + Storer(..) => Some(CompressionMethod::Stored), #[cfg(any( feature = "deflate", feature = "deflate-miniz", @@ -1110,13 +1100,13 @@ impl GenericZipWriter { GenericZipWriter::Bzip2(..) => Some(CompressionMethod::Bzip2), #[cfg(feature = "zstd")] GenericZipWriter::Zstd(..) => Some(CompressionMethod::Zstd), - GenericZipWriter::Closed => None, + Closed => None, } } fn unwrap(self) -> W { match self { - GenericZipWriter::Storer(w) => w, + Storer(w) => w, _ => panic!("Should have switched to stored beforehand"), } } @@ -1135,7 +1125,7 @@ fn deflate_compression_level_range() -> std::ops::RangeInclusive { #[cfg(feature = "bzip2")] fn bzip2_compression_level_range() -> std::ops::RangeInclusive { - let min = bzip2::Compression::none().level() as i32; + let min = bzip2::Compression::fast().level() as i32; let max = bzip2::Compression::best().level() as i32; min..=max } From ae6d98dec2ee47ffed50cef3c24c44d663d4cc6e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:13:53 -0700 Subject: [PATCH 065/281] Fix formatting and a Clippy issue --- src/types.rs | 10 +++++++- src/write.rs | 72 +++++++++++++++++++++++++++------------------------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/types.rs b/src/types.rs index 23e28ba1..d09ee716 100644 --- a/src/types.rs +++ b/src/types.rs @@ -176,7 +176,15 @@ impl DateTime { /// Indicates whether this date and time can be written to a zip archive. pub fn is_valid(&self) -> bool { - DateTime::from_date_and_time(self.year, self.month, self.day, self.hour, self.minute, self.second).is_ok() + DateTime::from_date_and_time( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + ) + .is_ok() } #[cfg(feature = "time")] diff --git a/src/write.rs b/src/write.rs index 595a3e7c..7f0c0984 100644 --- a/src/write.rs +++ b/src/write.rs @@ -90,8 +90,8 @@ pub(crate) mod zip_writer { pub(super) comment: Vec, } } -pub use zip_writer::ZipWriter; use crate::write::GenericZipWriter::{Closed, Storer}; +pub use zip_writer::ZipWriter; #[derive(Default)] struct ZipWriterStats { @@ -959,15 +959,10 @@ impl GenericZipWriter { _ => {} } - match self { - Closed => { - return Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "ZipWriter was already closed", - ) - .into()) - } - _ => {} + if let Closed = self { + return Err( + io::Error::new(io::ErrorKind::BrokenPipe, "ZipWriter was already closed").into(), + ); } let make_new_self: Box GenericZipWriter> = { @@ -990,27 +985,34 @@ impl GenericZipWriter { CompressionMethod::Deflated => { let level = clamp_opt( compression_level.unwrap_or(flate2::Compression::default().level() as i32), - deflate_compression_level_range() - ).ok_or(ZipError::UnsupportedArchive( + deflate_compression_level_range(), + ) + .ok_or(ZipError::UnsupportedArchive( "Unsupported compression level", ))? as u32; - Box::new(move |bare| - GenericZipWriter::Deflater(DeflateEncoder::new(bare, flate2::Compression::new(level))) - ) + Box::new(move |bare| { + GenericZipWriter::Deflater(DeflateEncoder::new( + bare, + flate2::Compression::new(level), + )) + }) } #[cfg(feature = "bzip2")] CompressionMethod::Bzip2 => { let level = clamp_opt( - compression_level - .unwrap_or(bzip2::Compression::default().level() as i32), - bzip2_compression_level_range(), - ).ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32; - Box::new(move |bare| - GenericZipWriter::Bzip2(BzEncoder::new(bare, bzip2::Compression::new(level))) + compression_level.unwrap_or(bzip2::Compression::default().level() as i32), + bzip2_compression_level_range(), ) - }, + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; + Box::new(move |bare| { + GenericZipWriter::Bzip2(BzEncoder::new( + bare, + bzip2::Compression::new(level), + )) + }) + } CompressionMethod::AES => { return Err(ZipError::UnsupportedArchive( "AES compression is not supported for writing", @@ -1022,13 +1024,13 @@ impl GenericZipWriter { compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL), zstd::compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))?; - Box::new(move |bare| - GenericZipWriter::Zstd(ZstdEncoder::new(bare, level).unwrap() - )) - }, + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))?; + Box::new(move |bare| { + GenericZipWriter::Zstd(ZstdEncoder::new(bare, level).unwrap()) + }) + } CompressionMethod::Unsupported(..) => { return Err(ZipError::UnsupportedArchive("Unsupported compression")) } @@ -1038,9 +1040,9 @@ impl GenericZipWriter { let bare = match mem::replace(self, Closed) { Storer(w) => w, #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" ))] GenericZipWriter::Deflater(w) => w.finish()?, #[cfg(feature = "bzip2")] @@ -1052,7 +1054,7 @@ impl GenericZipWriter { io::ErrorKind::BrokenPipe, "ZipWriter was already closed", ) - .into()) + .into()) } }; *self = (make_new_self)(bare); From 1543b64f309b7c3c95add7037fd8fb20962a3f70 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:15:19 -0700 Subject: [PATCH 066/281] Fix a deprecation warning --- src/compression.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compression.rs b/src/compression.rs index 43366ed1..63fef1fa 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -11,7 +11,7 @@ use std::fmt; /// When creating ZIP files, you may choose the method to use with /// [`crate::write::FileOptions::compression_method`] #[derive(Copy, Clone, PartialEq, Eq, Debug)] -#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary), allow(deprecated))] #[non_exhaustive] pub enum CompressionMethod { /// Store the file as is From 188433e7e949cf8c2d9e3756a77b0e4a45c0731c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:18:03 -0700 Subject: [PATCH 067/281] Bug fix for deprecation suppression --- src/compression.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compression.rs b/src/compression.rs index 63fef1fa..f4fe0df0 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -11,8 +11,9 @@ use std::fmt; /// When creating ZIP files, you may choose the method to use with /// [`crate::write::FileOptions::compression_method`] #[derive(Copy, Clone, PartialEq, Eq, Debug)] -#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary), allow(deprecated))] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] #[non_exhaustive] +#[allow(deprecated)] pub enum CompressionMethod { /// Store the file as is Stored, From ffd1083ff44a24a6830d73d10a9c05a7d15cd49e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:23:18 -0700 Subject: [PATCH 068/281] Bug fix: remove failed file from files_by_name after writing a large file without large-file option --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 7f0c0984..c16c9560 100644 --- a/src/write.rs +++ b/src/write.rs @@ -227,7 +227,7 @@ impl Write for ZipWriter { && !self.files.last_mut().unwrap().large_file { self.finish_file()?; - self.files.pop(); + self.files_by_name.remove(&*self.files.pop().unwrap().file_name); return Err(io::Error::new( io::ErrorKind::Other, "Large file option has not been set", From c969cd644ab0a177226023b3c50eb45b1d02f8dc Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:26:24 -0700 Subject: [PATCH 069/281] Bug fix: treat Unsupported as not deprecated in fuzzing config --- src/compression.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index f4fe0df0..ad7ac892 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -13,7 +13,6 @@ use std::fmt; #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] #[non_exhaustive] -#[allow(deprecated)] pub enum CompressionMethod { /// Store the file as is Stored, @@ -37,7 +36,7 @@ pub enum CompressionMethod { #[cfg(feature = "zstd")] Zstd, /// Unsupported compression method - #[deprecated(since = "0.5.7", note = "use the constants instead")] + #[cfg_attr(not(fuzzing), deprecated(since = "0.5.7", note = "use the constants instead"))] Unsupported(u16), } #[allow(deprecated, missing_docs)] From 6ad6f063408a0bbe29607bbb7382cf0a65bfcd99 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:37:43 -0700 Subject: [PATCH 070/281] Fix formatting --- src/compression.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/compression.rs b/src/compression.rs index ad7ac892..1e9ae18c 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -36,7 +36,10 @@ pub enum CompressionMethod { #[cfg(feature = "zstd")] Zstd, /// Unsupported compression method - #[cfg_attr(not(fuzzing), deprecated(since = "0.5.7", note = "use the constants instead"))] + #[cfg_attr( + not(fuzzing), + deprecated(since = "0.5.7", note = "use the constants instead") + )] Unsupported(u16), } #[allow(deprecated, missing_docs)] From 9cc6060eb9c8fc4250b5e2a50a87d80b6564b8f3 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:41:24 -0700 Subject: [PATCH 071/281] Add is_writing_file and update doc --- CHANGELOG.md | 14 +++++++++++++- src/write.rs | 10 +++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab5785f6..c8dae202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,4 +74,16 @@ ### Fixed -- Fixed a possible bug in deep_copy_file. \ No newline at end of file + - Fixed a possible bug in deep_copy_file. + +## [0.7.0] + +### Fixed + + - Calling `start_file` with invalid parameters no longer closes the `ZipWriter`. + - Attempting to write a 4GiB file without calling `FileOptions::large_file(true)` now removes the file from the archive + but does not close the `ZipWriter`. + +### Added + + - Method `is_writing_file` - indicates whether a file is open for writing. \ No newline at end of file diff --git a/src/write.rs b/src/write.rs index c16c9560..016249ee 100644 --- a/src/write.rs +++ b/src/write.rs @@ -227,7 +227,8 @@ impl Write for ZipWriter { && !self.files.last_mut().unwrap().large_file { self.finish_file()?; - self.files_by_name.remove(&*self.files.pop().unwrap().file_name); + self.files_by_name + .remove(&*self.files.pop().unwrap().file_name); return Err(io::Error::new( io::ErrorKind::Other, "Large file option has not been set", @@ -355,6 +356,8 @@ impl ZipWriter { /// Initializes the archive. /// /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. + /// After a successful write, the file remains open for writing. After a failed write, call + /// [`ZipWriter::is_writing_file`] to determine if the file remains open. pub fn new(inner: W) -> ZipWriter { ZipWriter { inner: Storer(inner), @@ -369,6 +372,11 @@ impl ZipWriter { } } + /// Returns true if a file is currently open for writing. + pub fn is_writing_file(&self) -> bool { + self.writing_to_file && !self.inner.is_closed() + } + /// Set ZIP archive comment. pub fn set_comment(&mut self, comment: S) where From a1c7689d069e3e30bc181cc764cdae4c8ef195b2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:42:30 -0700 Subject: [PATCH 072/281] Reduce fuzz_write to 1 million iterations --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f8fc549b..21383cfb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -runs=10000000 + cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 From 00101013f3f9c33aecca8f9f20b9a06a236e7e98 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:56:02 -0700 Subject: [PATCH 073/281] Update CHANGELOG for date/time fix --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8dae202..14d7d221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,8 @@ - Calling `start_file` with invalid parameters no longer closes the `ZipWriter`. - Attempting to write a 4GiB file without calling `FileOptions::large_file(true)` now removes the file from the archive but does not close the `ZipWriter`. + - Attempting to write a file with an unrepresentable or invalid last-modified date will instead add it with a date of + 1980-01-01 00:00:00. ### Added From 84645edd7726c4997eb5febb2f457546f7380255 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 10:56:25 -0700 Subject: [PATCH 074/281] Add write fuzzing to README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 8e886b4e..c0caf98a 100644 --- a/README.md +++ b/README.md @@ -92,3 +92,9 @@ To start fuzzing zip extraction: ```bash cargo +nightly fuzz run fuzz_read ``` + +To start fuzzing zip creation: + +```bash +cargo +nightly fuzz run fuzz_write +``` From c3d0fd145b4398aa1b5f885023c1e0ea6701ff72 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 11:01:05 -0700 Subject: [PATCH 075/281] Bump version to 0.7.0 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6825e867..108cc0a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.6.13" +version = "0.7.0" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index c0caf98a..dab76774 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.6.13" +zip_next = "0.7.0" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.6.13", default-features = false } +zip_next = { version = "0.7.0", default-features = false } ``` The features available are: From a23e3889e26e2dd1387cf2f97d3430cedeae8e76 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 11:56:46 -0700 Subject: [PATCH 076/281] Strengthen fuzz_write: use longer vectors and multiple writes --- .github/workflows/ci.yaml | 2 +- fuzz/fuzz_targets/fuzz_write.rs | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 21383cfb..14337205 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 + cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 -max_len=5000000000 diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 3b2a73d6..8b1a9cd5 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -1,12 +1,21 @@ #![no_main] + use libfuzzer_sys::fuzz_target; use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; +#[derive(Arbitrary,Debug)] +pub struct ExtraData { + pub header_id: u16, + pub data: Vec +} + #[derive(Arbitrary,Debug)] pub struct File { pub name: String, - pub contents: Vec + pub contents: Vec>, + pub local_extra_data: Vec, + pub central_extra_data: Vec } #[derive(Arbitrary,Debug)] @@ -22,7 +31,7 @@ pub enum FileOperation { DeepCopy { base: Box, new_name: String - }, + } } impl FileOperation { @@ -40,11 +49,13 @@ fn do_operation(writer: &mut zip_next::ZipWriter, where T: Read + Write + Seek { match operation { FileOperation::Write {file, mut options} => { - if (*file).contents.len() >= u32::MAX as usize { + if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } writer.start_file(file.name.to_owned(), options)?; - writer.write_all(file.contents.as_slice())?; + for chunk in &file.contents { + writer.write_all(chunk.as_slice())?; + } } FileOperation::ShallowCopy {base, new_name} => { do_operation(writer, base)?; @@ -63,5 +74,5 @@ fuzz_target!(|data: Vec| { for operation in data { let _ = do_operation(&mut writer, &operation); } - writer.finish().unwrap(); + let _ = zip_next::ZipArchive::new(writer.finish().unwrap()); }); \ No newline at end of file From 6582e13fc443da2b0e9c755a25d9033823b20a35 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 12:29:25 -0700 Subject: [PATCH 077/281] Make large files more likely --- fuzz/fuzz_targets/fuzz_write.rs | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 8b1a9cd5..7ad45b5b 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -2,20 +2,29 @@ use libfuzzer_sys::fuzz_target; use arbitrary::Arbitrary; +use std::fmt::{Debug, Formatter}; use std::io::{Cursor, Read, Seek, Write}; -#[derive(Arbitrary,Debug)] -pub struct ExtraData { - pub header_id: u16, - pub data: Vec -} - #[derive(Arbitrary,Debug)] pub struct File { pub name: String, - pub contents: Vec>, - pub local_extra_data: Vec, - pub central_extra_data: Vec + pub contents: Vec> +} + +#[derive(Arbitrary)] +pub struct LargeFile { + pub name: String, + pub large_contents: [u8; u32::MAX as usize + 1], + pub extra_contents: Vec> +} + +impl Debug for LargeFile { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LargeFile") + .field("name", &self.name) + .field("extra_contents", &self.extra_contents) + .finish() + } } #[derive(Arbitrary,Debug)] @@ -24,6 +33,10 @@ pub enum FileOperation { file: File, options: zip_next::write::FileOptions }, + WriteLarge { + file: LargeFile, + options: zip_next::write::FileOptions + }, ShallowCopy { base: Box, new_name: String @@ -38,6 +51,7 @@ impl FileOperation { pub fn get_name(&self) -> String { match self { FileOperation::Write {file, ..} => &file.name, + FileOperation::WriteLarge {file, ..} => &file.name, FileOperation::ShallowCopy {new_name, ..} => new_name, FileOperation::DeepCopy {new_name, ..} => new_name }.to_owned() @@ -57,6 +71,14 @@ fn do_operation(writer: &mut zip_next::ZipWriter, writer.write_all(chunk.as_slice())?; } } + FileOperation::WriteLarge {file, mut options} => { + options = options.large_file(true); + writer.start_file(file.name.to_owned(), options)?; + writer.write_all(&file.large_contents)?; + for chunk in &file.extra_contents { + writer.write_all(chunk.as_slice())?; + } + } FileOperation::ShallowCopy {base, new_name} => { do_operation(writer, base)?; writer.shallow_copy_file(&base.get_name(), new_name)?; From a9aaea306e052932d1a23e97feb676a202c07d28 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 13:02:52 -0700 Subject: [PATCH 078/281] Make large files more likely --- .github/workflows/ci.yaml | 2 +- fuzz/fuzz_targets/fuzz_write.rs | 26 ++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 14337205..f01ab679 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 -max_len=15000000000 diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 7ad45b5b..2c3bdc70 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -1,7 +1,8 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use arbitrary::Arbitrary; +use arbitrary::{Arbitrary, Unstructured}; +use arbitrary::size_hint::and_all; use std::fmt::{Debug, Formatter}; use std::io::{Cursor, Read, Seek, Write}; @@ -11,13 +12,30 @@ pub struct File { pub contents: Vec> } -#[derive(Arbitrary)] +const LARGE_FILE_BUF_SIZE: usize = u32::MAX as usize + 1; + pub struct LargeFile { pub name: String, - pub large_contents: [u8; u32::MAX as usize + 1], + pub large_contents: Vec, pub extra_contents: Vec> } +impl Arbitrary<'_> for LargeFile { + fn arbitrary(u: &mut Unstructured) -> arbitrary::Result { + Ok(LargeFile { + name: String::arbitrary(u)?, + large_contents: u.bytes(LARGE_FILE_BUF_SIZE)?.to_vec(), + extra_contents: Vec::arbitrary(u)? + }) + } + + fn size_hint(depth: usize) -> (usize, Option) { + and_all(&[::size_hint(depth), + > as Arbitrary>::size_hint(depth), + (LARGE_FILE_BUF_SIZE, Some(LARGE_FILE_BUF_SIZE))]) + } +} + impl Debug for LargeFile { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("LargeFile") @@ -74,7 +92,7 @@ fn do_operation(writer: &mut zip_next::ZipWriter, FileOperation::WriteLarge {file, mut options} => { options = options.large_file(true); writer.start_file(file.name.to_owned(), options)?; - writer.write_all(&file.large_contents)?; + writer.write_all(&*file.large_contents)?; for chunk in &file.extra_contents { writer.write_all(chunk.as_slice())?; } From 09d1ef2e3b90d538373c5774dac1685d12133231 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 14:59:04 -0700 Subject: [PATCH 079/281] Make large files more compressible --- .github/workflows/ci.yaml | 2 +- fuzz/Cargo.toml | 1 + fuzz/fuzz_targets/fuzz_write.rs | 75 ++++++++++++++++++++------------- 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f01ab679..5972802f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 -max_len=15000000000 + cargo fuzz run fuzz_write -- -timeout=2m diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index f63ff859..ce6b11fb 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,6 +11,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" arbitrary = { version = "1.3.0", features = ["derive"] } +itertools = "0.10.5" [dependencies.zip_next] path = ".." diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 2c3bdc70..e56add51 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -1,10 +1,10 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use arbitrary::{Arbitrary, Unstructured}; -use arbitrary::size_hint::and_all; -use std::fmt::{Debug, Formatter}; +use arbitrary::{Arbitrary}; +use std::fmt::Debug; use std::io::{Cursor, Read, Seek, Write}; +use std::iter::{repeat, Flatten, Repeat, Take}; #[derive(Arbitrary,Debug)] pub struct File { @@ -14,37 +14,44 @@ pub struct File { const LARGE_FILE_BUF_SIZE: usize = u32::MAX as usize + 1; +#[derive(Arbitrary, Clone, Debug)] +pub enum RepeatedBytes { + Once(Vec), + U8Times { + bytes: Vec, + repeats: u8, + }, + U16Times { + bytes: Vec, + repeats: u16, + } +} + +impl IntoIterator for RepeatedBytes { + type Item = u8; + type IntoIter = Flatten>>>; + fn into_iter(self) -> Self::IntoIter { + match self { + RepeatedBytes::Once(bytes) => { + repeat(bytes).take(1) + }, + RepeatedBytes::U8Times {bytes, repeats} => { + repeat(bytes).take(repeats as usize + 2) + }, + RepeatedBytes::U16Times {bytes, repeats} => { + repeat(bytes).take(repeats as usize + u8::MAX as usize + 2) + } + }.flatten() + } +} + +#[derive(Arbitrary,Debug)] pub struct LargeFile { pub name: String, - pub large_contents: Vec, + pub large_contents: Vec>, pub extra_contents: Vec> } -impl Arbitrary<'_> for LargeFile { - fn arbitrary(u: &mut Unstructured) -> arbitrary::Result { - Ok(LargeFile { - name: String::arbitrary(u)?, - large_contents: u.bytes(LARGE_FILE_BUF_SIZE)?.to_vec(), - extra_contents: Vec::arbitrary(u)? - }) - } - - fn size_hint(depth: usize) -> (usize, Option) { - and_all(&[::size_hint(depth), - > as Arbitrary>::size_hint(depth), - (LARGE_FILE_BUF_SIZE, Some(LARGE_FILE_BUF_SIZE))]) - } -} - -impl Debug for LargeFile { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LargeFile") - .field("name", &self.name) - .field("extra_contents", &self.extra_contents) - .finish() - } -} - #[derive(Arbitrary,Debug)] pub enum FileOperation { Write { @@ -92,7 +99,15 @@ fn do_operation(writer: &mut zip_next::ZipWriter, FileOperation::WriteLarge {file, mut options} => { options = options.large_file(true); writer.start_file(file.name.to_owned(), options)?; - writer.write_all(&*file.large_contents)?; + let written: usize = 0; + while written < LARGE_FILE_BUF_SIZE { + for chunk in &file.large_contents { + let chunk: Vec = chunk.iter() + .flat_map(RepeatedBytes::into_iter) + .collect(); + writer.write_all(chunk.as_slice())?; + } + } for chunk in &file.extra_contents { writer.write_all(chunk.as_slice())?; } From bc14c1ef5e8f494b0e5d66aff6e54b34269bb05a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 15:08:20 -0700 Subject: [PATCH 080/281] Set a minimum size for non-repeating chunks --- fuzz/fuzz_targets/fuzz_write.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index e56add51..104c139a 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -16,7 +16,10 @@ const LARGE_FILE_BUF_SIZE: usize = u32::MAX as usize + 1; #[derive(Arbitrary, Clone, Debug)] pub enum RepeatedBytes { - Once(Vec), + Once { + min_bytes: [u8; 1024], + extra_bytes: Vec + }, U8Times { bytes: Vec, repeats: u8, @@ -32,7 +35,9 @@ impl IntoIterator for RepeatedBytes { type IntoIter = Flatten>>>; fn into_iter(self) -> Self::IntoIter { match self { - RepeatedBytes::Once(bytes) => { + RepeatedBytes::Once {min_bytes, extra_bytes} => { + let mut bytes = min_bytes.to_vec(); + bytes.extend(extra_bytes); repeat(bytes).take(1) }, RepeatedBytes::U8Times {bytes, repeats} => { @@ -102,7 +107,7 @@ fn do_operation(writer: &mut zip_next::ZipWriter, let written: usize = 0; while written < LARGE_FILE_BUF_SIZE { for chunk in &file.large_contents { - let chunk: Vec = chunk.iter() + let chunk: Vec = chunk.to_owned().into_iter() .flat_map(RepeatedBytes::into_iter) .collect(); writer.write_all(chunk.as_slice())?; From e75527ffc74f87b2c9772a7b2a8b5e0a0ee05a0d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 15:13:44 -0700 Subject: [PATCH 081/281] Bug fix: all timeouts are apparently in seconds --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5972802f..f9e7a1e2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,7 @@ jobs: cargo fuzz build fuzz_read - name: run fuzz run: | - cargo fuzz run fuzz_read -- -timeout=1s -runs=10000000 + cargo fuzz run fuzz_read -- -timeout=1 -runs=10000000 fuzz_write: runs-on: ubuntu-latest @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=2m + cargo fuzz run fuzz_write -- -timeout=120 -runs=1000000 From 5220222f8437196f4b7e8d4e58eb986f9a0fbefd Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 15:35:36 -0700 Subject: [PATCH 082/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 104c139a..a13d9b06 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -104,12 +104,13 @@ fn do_operation(writer: &mut zip_next::ZipWriter, FileOperation::WriteLarge {file, mut options} => { options = options.large_file(true); writer.start_file(file.name.to_owned(), options)?; - let written: usize = 0; + let mut written: usize = 0; while written < LARGE_FILE_BUF_SIZE { for chunk in &file.large_contents { let chunk: Vec = chunk.to_owned().into_iter() .flat_map(RepeatedBytes::into_iter) .collect(); + written += chunk.len(); writer.write_all(chunk.as_slice())?; } } From c18fae1f7703065966e44b89f5c89852f8284efd Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:01:38 -0700 Subject: [PATCH 083/281] Use sparse files that are mostly the same byte pattern --- .github/workflows/ci.yaml | 2 +- fuzz/fuzz_targets/fuzz_write.rs | 65 +++++++++------------------------ 2 files changed, 19 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f9e7a1e2..37e7fffb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=120 -runs=1000000 + cargo fuzz run fuzz_write -- -timeout=300 -runs=1000000 -max_len=1000000 diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index a13d9b06..9c67e829 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -4,7 +4,7 @@ use libfuzzer_sys::fuzz_target; use arbitrary::{Arbitrary}; use std::fmt::Debug; use std::io::{Cursor, Read, Seek, Write}; -use std::iter::{repeat, Flatten, Repeat, Take}; +use std::iter::{repeat}; #[derive(Arbitrary,Debug)] pub struct File { @@ -15,46 +15,18 @@ pub struct File { const LARGE_FILE_BUF_SIZE: usize = u32::MAX as usize + 1; #[derive(Arbitrary, Clone, Debug)] -pub enum RepeatedBytes { - Once { - min_bytes: [u8; 1024], - extra_bytes: Vec - }, - U8Times { - bytes: Vec, - repeats: u8, - }, - U16Times { - bytes: Vec, - repeats: u16, - } -} - -impl IntoIterator for RepeatedBytes { - type Item = u8; - type IntoIter = Flatten>>>; - fn into_iter(self) -> Self::IntoIter { - match self { - RepeatedBytes::Once {min_bytes, extra_bytes} => { - let mut bytes = min_bytes.to_vec(); - bytes.extend(extra_bytes); - repeat(bytes).take(1) - }, - RepeatedBytes::U8Times {bytes, repeats} => { - repeat(bytes).take(repeats as usize + 2) - }, - RepeatedBytes::U16Times {bytes, repeats} => { - repeat(bytes).take(repeats as usize + u8::MAX as usize + 2) - } - }.flatten() - } +pub struct SparseFilePart { + pub start: u32, + pub contents: Vec } #[derive(Arbitrary,Debug)] pub struct LargeFile { pub name: String, - pub large_contents: Vec>, - pub extra_contents: Vec> + pub default_pattern_first_byte: u8, + pub default_pattern_extra_bytes: Vec, + pub parts: Vec, + pub min_extra_length: u32 } #[derive(Arbitrary,Debug)] @@ -104,19 +76,18 @@ fn do_operation(writer: &mut zip_next::ZipWriter, FileOperation::WriteLarge {file, mut options} => { options = options.large_file(true); writer.start_file(file.name.to_owned(), options)?; - let mut written: usize = 0; - while written < LARGE_FILE_BUF_SIZE { - for chunk in &file.large_contents { - let chunk: Vec = chunk.to_owned().into_iter() - .flat_map(RepeatedBytes::into_iter) - .collect(); - written += chunk.len(); - writer.write_all(chunk.as_slice())?; + let mut default_pattern = Vec::with_capacity(file.default_pattern_extra_bytes.len() + 1); + default_pattern.push(file.default_pattern_first_byte); + default_pattern.extend(&file.default_pattern_extra_bytes); + let mut sparse_file: Vec = + repeat(default_pattern.into_iter()).flatten().take(LARGE_FILE_BUF_SIZE + file.min_extra_length as usize) + .collect(); + for part in &file.parts { + for (index, byte) in part.contents.iter().enumerate() { + sparse_file[part.start as usize + index] = *byte; } } - for chunk in &file.extra_contents { - writer.write_all(chunk.as_slice())?; - } + writer.write_all(sparse_file.as_slice())?; } FileOperation::ShallowCopy {base, new_name} => { do_operation(writer, base)?; From 69bb01d5354083c514767c2538b969d0cc27370d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:05:27 -0700 Subject: [PATCH 084/281] Allow repetition inside non-default chunks of sparse files --- fuzz/fuzz_targets/fuzz_write.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 9c67e829..8e57d811 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -17,7 +17,9 @@ const LARGE_FILE_BUF_SIZE: usize = u32::MAX as usize + 1; #[derive(Arbitrary, Clone, Debug)] pub struct SparseFilePart { pub start: u32, - pub contents: Vec + pub first_byte: u8, + pub extra_bytes: Vec, + pub repeats: u8 } #[derive(Arbitrary,Debug)] @@ -83,7 +85,10 @@ fn do_operation(writer: &mut zip_next::ZipWriter, repeat(default_pattern.into_iter()).flatten().take(LARGE_FILE_BUF_SIZE + file.min_extra_length as usize) .collect(); for part in &file.parts { - for (index, byte) in part.contents.iter().enumerate() { + let mut bytes = Vec::with_capacity(part.extra_bytes.len() + 1); + bytes.push(part.first_byte); + bytes.extend(part.extra_bytes); + for (index, byte) in repeat(bytes.iter()).take(part.repeats).flatten().enumerate() { sparse_file[part.start as usize + index] = *byte; } } From 29237770c9be4b3d5ff7f227bfc1cae4efdeeae4 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:06:42 -0700 Subject: [PATCH 085/281] Prevent sparse-file chunks from repeating zero times --- fuzz/fuzz_targets/fuzz_write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 8e57d811..88d299b2 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -88,7 +88,7 @@ fn do_operation(writer: &mut zip_next::ZipWriter, let mut bytes = Vec::with_capacity(part.extra_bytes.len() + 1); bytes.push(part.first_byte); bytes.extend(part.extra_bytes); - for (index, byte) in repeat(bytes.iter()).take(part.repeats).flatten().enumerate() { + for (index, byte) in repeat(bytes.iter()).take(part.repeats as usize + 1).flatten().enumerate() { sparse_file[part.start as usize + index] = *byte; } } From 1006ddb0e95ca38e08abddd8ad68865c46628411 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:10:59 -0700 Subject: [PATCH 086/281] Bug fix: need to make a copy --- fuzz/fuzz_targets/fuzz_write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 88d299b2..51d52b8a 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -28,7 +28,7 @@ pub struct LargeFile { pub default_pattern_first_byte: u8, pub default_pattern_extra_bytes: Vec, pub parts: Vec, - pub min_extra_length: u32 + pub min_extra_length: u16 } #[derive(Arbitrary,Debug)] @@ -87,7 +87,7 @@ fn do_operation(writer: &mut zip_next::ZipWriter, for part in &file.parts { let mut bytes = Vec::with_capacity(part.extra_bytes.len() + 1); bytes.push(part.first_byte); - bytes.extend(part.extra_bytes); + bytes.extend(part.extra_bytes.iter()); for (index, byte) in repeat(bytes.iter()).take(part.repeats as usize + 1).flatten().enumerate() { sparse_file[part.start as usize + index] = *byte; } From 8578e64554b2385686ad47010189fabeb4b3f83f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:18:35 -0700 Subject: [PATCH 087/281] Increase memory limit for write fuzz --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 37e7fffb..c92ef41b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=300 -runs=1000000 -max_len=1000000 + cargo fuzz run fuzz_write -- -timeout=300 -runs=1000000 -max_len=1000000 -rss_limit_mb=7000 From 8ba431d89c31722980b9d7d0a059199e3b7be04e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:28:42 -0700 Subject: [PATCH 088/281] Increase write-fuzz timeout --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c92ef41b..4025f4d5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=300 -runs=1000000 -max_len=1000000 -rss_limit_mb=7000 + cargo fuzz run fuzz_write -- -timeout=600 -runs=1000000 -max_len=1000000 -rss_limit_mb=7000 From 78139acfae4afb6c15068e243a527a8aafc06ff2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:43:38 -0700 Subject: [PATCH 089/281] Prevent WriteLarge from running with no compression --- fuzz/fuzz_targets/fuzz_write.rs | 2 +- src/write.rs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 51d52b8a..f95eeed9 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -76,7 +76,7 @@ fn do_operation(writer: &mut zip_next::ZipWriter, } } FileOperation::WriteLarge {file, mut options} => { - options = options.large_file(true); + options = options.large_file(true).force_compression(); writer.start_file(file.name.to_owned(), options)?; let mut default_pattern = Vec::with_capacity(file.default_pattern_extra_bytes.len() + 1); default_pattern.push(file.default_pattern_first_byte); diff --git a/src/write.rs b/src/write.rs index 016249ee..76e5f4cf 100644 --- a/src/write.rs +++ b/src/write.rs @@ -128,6 +128,13 @@ impl FileOptions { self } + #[cfg(fuzzing)] + pub fn force_compression(mut self) { + if self.compression_method == CompressionMethod::Stored { + self.compression_method = CompressionMethod::Deflated; + } + } + /// Set the compression level for the new file /// /// `None` value specifies default compression level. From de0cbc039ebea8afc96e1c235e3711a4779aa684 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:47:07 -0700 Subject: [PATCH 090/281] Add missing doc --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index 76e5f4cf..3801e892 100644 --- a/src/write.rs +++ b/src/write.rs @@ -129,6 +129,7 @@ impl FileOptions { } #[cfg(fuzzing)] + /// Changes the compression method to the default if it would otherwise be no compression. pub fn force_compression(mut self) { if self.compression_method == CompressionMethod::Stored { self.compression_method = CompressionMethod::Deflated; From 28f045e17e98830c870e53974e6cf6501a217bab Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:50:36 -0700 Subject: [PATCH 091/281] Bug fix --- src/write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 3801e892..0281c94d 100644 --- a/src/write.rs +++ b/src/write.rs @@ -130,10 +130,11 @@ impl FileOptions { #[cfg(fuzzing)] /// Changes the compression method to the default if it would otherwise be no compression. - pub fn force_compression(mut self) { + pub fn force_compression(mut self) -> FileOptions { if self.compression_method == CompressionMethod::Stored { self.compression_method = CompressionMethod::Deflated; } + self } /// Set the compression level for the new file From 2f4acd3900832e8cda24fbc0899b78ab81682d8b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 16:51:55 -0700 Subject: [PATCH 092/281] Formatting tweaks --- src/write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 0281c94d..837b28cc 100644 --- a/src/write.rs +++ b/src/write.rs @@ -128,8 +128,9 @@ impl FileOptions { self } + /// Changes the compression method to Deflate if it would otherwise be no compression. + #[must_use] #[cfg(fuzzing)] - /// Changes the compression method to the default if it would otherwise be no compression. pub fn force_compression(mut self) -> FileOptions { if self.compression_method == CompressionMethod::Stored { self.compression_method = CompressionMethod::Deflated; From 04654fdbe329267ee19e5360e27d70caa1eaa2c3 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 17:06:37 -0700 Subject: [PATCH 093/281] Revert large-file test: impractical --- .github/workflows/ci.yaml | 4 +-- fuzz/Cargo.toml | 1 - fuzz/fuzz_targets/fuzz_write.rs | 57 ++++++--------------------------- src/write.rs | 10 ------ 4 files changed, 12 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4025f4d5..14337205 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,7 @@ jobs: cargo fuzz build fuzz_read - name: run fuzz run: | - cargo fuzz run fuzz_read -- -timeout=1 -runs=10000000 + cargo fuzz run fuzz_read -- -timeout=1s -runs=10000000 fuzz_write: runs-on: ubuntu-latest @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=600 -runs=1000000 -max_len=1000000 -rss_limit_mb=7000 + cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 -max_len=5000000000 diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index ce6b11fb..f63ff859 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,6 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" arbitrary = { version = "1.3.0", features = ["derive"] } -itertools = "0.10.5" [dependencies.zip_next] path = ".." diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index f95eeed9..8b1a9cd5 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -1,34 +1,21 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use arbitrary::{Arbitrary}; -use std::fmt::Debug; +use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; -use std::iter::{repeat}; + +#[derive(Arbitrary,Debug)] +pub struct ExtraData { + pub header_id: u16, + pub data: Vec +} #[derive(Arbitrary,Debug)] pub struct File { pub name: String, - pub contents: Vec> -} - -const LARGE_FILE_BUF_SIZE: usize = u32::MAX as usize + 1; - -#[derive(Arbitrary, Clone, Debug)] -pub struct SparseFilePart { - pub start: u32, - pub first_byte: u8, - pub extra_bytes: Vec, - pub repeats: u8 -} - -#[derive(Arbitrary,Debug)] -pub struct LargeFile { - pub name: String, - pub default_pattern_first_byte: u8, - pub default_pattern_extra_bytes: Vec, - pub parts: Vec, - pub min_extra_length: u16 + pub contents: Vec>, + pub local_extra_data: Vec, + pub central_extra_data: Vec } #[derive(Arbitrary,Debug)] @@ -37,10 +24,6 @@ pub enum FileOperation { file: File, options: zip_next::write::FileOptions }, - WriteLarge { - file: LargeFile, - options: zip_next::write::FileOptions - }, ShallowCopy { base: Box, new_name: String @@ -55,7 +38,6 @@ impl FileOperation { pub fn get_name(&self) -> String { match self { FileOperation::Write {file, ..} => &file.name, - FileOperation::WriteLarge {file, ..} => &file.name, FileOperation::ShallowCopy {new_name, ..} => new_name, FileOperation::DeepCopy {new_name, ..} => new_name }.to_owned() @@ -75,25 +57,6 @@ fn do_operation(writer: &mut zip_next::ZipWriter, writer.write_all(chunk.as_slice())?; } } - FileOperation::WriteLarge {file, mut options} => { - options = options.large_file(true).force_compression(); - writer.start_file(file.name.to_owned(), options)?; - let mut default_pattern = Vec::with_capacity(file.default_pattern_extra_bytes.len() + 1); - default_pattern.push(file.default_pattern_first_byte); - default_pattern.extend(&file.default_pattern_extra_bytes); - let mut sparse_file: Vec = - repeat(default_pattern.into_iter()).flatten().take(LARGE_FILE_BUF_SIZE + file.min_extra_length as usize) - .collect(); - for part in &file.parts { - let mut bytes = Vec::with_capacity(part.extra_bytes.len() + 1); - bytes.push(part.first_byte); - bytes.extend(part.extra_bytes.iter()); - for (index, byte) in repeat(bytes.iter()).take(part.repeats as usize + 1).flatten().enumerate() { - sparse_file[part.start as usize + index] = *byte; - } - } - writer.write_all(sparse_file.as_slice())?; - } FileOperation::ShallowCopy {base, new_name} => { do_operation(writer, base)?; writer.shallow_copy_file(&base.get_name(), new_name)?; diff --git a/src/write.rs b/src/write.rs index 837b28cc..016249ee 100644 --- a/src/write.rs +++ b/src/write.rs @@ -128,16 +128,6 @@ impl FileOptions { self } - /// Changes the compression method to Deflate if it would otherwise be no compression. - #[must_use] - #[cfg(fuzzing)] - pub fn force_compression(mut self) -> FileOptions { - if self.compression_method == CompressionMethod::Stored { - self.compression_method = CompressionMethod::Deflated; - } - self - } - /// Set the compression level for the new file /// /// `None` value specifies default compression level. From 17f059a1bc40007742ec3a5c68c6f9330e665136 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Tue, 2 May 2023 08:57:50 -0700 Subject: [PATCH 094/281] Update README to note this is a fork --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dab76774..1048e3a6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Info ---- -A zip library for rust which supports reading and writing of simple ZIP files. +A zip library for rust which supports reading and writing of simple ZIP files. Forked from https://crates.io/crates/zip to add more features and improve test coverage. Supported compression formats: From 1f70c80eb10af49002d8f4e88e67372996fd1d2f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 2 May 2023 18:31:07 -0700 Subject: [PATCH 095/281] Allow Dependabot to update GitHub Actions --- .github/dependabot.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5cde1657..ac237f1b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,11 @@ version: 2 updates: -- package-ecosystem: cargo - directory: "/" - schedule: - interval: daily - open-pull-requests-limit: 10 + - package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: "github-actions" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" \ No newline at end of file From 9bb9600aced458107cabeb5f84de489662655f93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 01:32:12 +0000 Subject: [PATCH 096/281] chore(deps): bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 14337205..f1d0dd92 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: @@ -77,7 +77,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: profile: minimal From 57f8d6b62c17bb0dc2e384aaddbb9873959b01aa Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 2 May 2023 18:53:54 -0700 Subject: [PATCH 097/281] Bump version to 0.7.0.1 so that crates.io will pick up an updated README --- CHANGELOG.md | 8 +++++++- Cargo.toml | 2 +- README.md | 7 ++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14d7d221..d898f1cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,4 +88,10 @@ ### Added - - Method `is_writing_file` - indicates whether a file is open for writing. \ No newline at end of file + - Method `is_writing_file` - indicates whether a file is open for writing. + +## [0.7.0.1] + +### Changed + + - Bumped the version number in order to upload an updated README to crates.io. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 108cc0a9..edd532f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.0" +version = "0.7.0.1" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 1048e3a6..143ce4d4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ Info ---- -A zip library for rust which supports reading and writing of simple ZIP files. Forked from https://crates.io/crates/zip to add more features and improve test coverage. +A zip library for rust which supports reading and writing of simple ZIP files. Forked from https://crates.io/crates/zip +to add more features and improve test coverage. Supported compression formats: @@ -31,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.0" +zip_next = "0.7.0.1" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.0", default-features = false } +zip_next = { version = "0.7.0.1", default-features = false } ``` The features available are: From 08815b9fceba90b80f1118a699cb5b2c01fd6bd8 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 2 May 2023 18:55:05 -0700 Subject: [PATCH 098/281] Bump version to 0.7.1 to work around a limitation of `cargo publish` where sub-patch versions aren't allowed --- CHANGELOG.md | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d898f1cf..140450b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,7 +90,7 @@ - Method `is_writing_file` - indicates whether a file is open for writing. -## [0.7.0.1] +## [0.7.1] ### Changed diff --git a/README.md b/README.md index 143ce4d4..e6835ef1 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.0.1" +zip_next = "0.7.1" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.0.1", default-features = false } +zip_next = { version = "0.7.1", default-features = false } ``` The features available are: From 4837e92d1b318061597f2447c495ff4e9e38d177 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 2 May 2023 18:55:37 -0700 Subject: [PATCH 099/281] Bug fix: version bump was incomplete --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index edd532f9..a88f532a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.0.1" +version = "0.7.1" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" From d5e45f3c20c62adeccad0826ba54c5a726fecee9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:20:47 -0700 Subject: [PATCH 100/281] Add abort_file, and call it when validations fail --- fuzz/fuzz_targets/fuzz_write.rs | 7 +- src/write.rs | 127 ++++++++++++++++---------------- 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 8b1a9cd5..0e1d57fe 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -69,10 +69,13 @@ fn do_operation(writer: &mut zip_next::ZipWriter, Ok(()) } -fuzz_target!(|data: Vec| { +fuzz_target!(|data: Vec<(FileOperation, bool)>| { let mut writer = zip_next::ZipWriter::new(Cursor::new(Vec::new())); - for operation in data { + for (operation, close_and_reopen) in data { let _ = do_operation(&mut writer, &operation); + if close_and_reopen { + writer = zip_next::ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + } } let _ = zip_next::ZipArchive::new(writer.finish().unwrap()); }); \ No newline at end of file diff --git a/src/write.rs b/src/write.rs index 016249ee..cbeb0e25 100644 --- a/src/write.rs +++ b/src/write.rs @@ -226,9 +226,7 @@ impl Write for ZipWriter { if self.stats.bytes_written > spec::ZIP64_BYTES_THR && !self.files.last_mut().unwrap().large_file { - self.finish_file()?; - self.files_by_name - .remove(&*self.files.pop().unwrap().file_name); + let _ = self.abort_file(); return Err(io::Error::new( io::ErrorKind::Other, "Large file option has not been set", @@ -471,7 +469,8 @@ impl ZipWriter { // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. self.end_extra_data()?; } - self.inner.switch_to(CompressionMethod::Stored, None)?; + let make_plain_writer = self.inner.prepare_switch_to(CompressionMethod::Stored, None)?; + self.inner.switch_to(make_plain_writer)?; let writer = self.inner.get_plain(); if !self.writing_raw { @@ -495,6 +494,18 @@ impl ZipWriter { Ok(()) } + /// Removes the file currently being written from the archive. + pub fn abort_file(&mut self) -> ZipResult<()> { + self.files_by_name + .remove(&*self.files.pop().unwrap().file_name); + let make_plain_writer + = self.inner.prepare_switch_to(CompressionMethod::Stored, None)?; + self.inner.switch_to(make_plain_writer)?; + self.writing_to_file = false; + self.writing_raw = false; + Ok(()) + } + /// Create a file in the archive and start writing its' contents. The file must not have the /// same name as a file already in the archive. /// @@ -504,9 +515,10 @@ impl ZipWriter { S: Into, { Self::normalize_options(&mut options); + let make_new_self = self.inner + .prepare_switch_to(options.compression_method, options.compression_level)?; self.start_entry(name, options, None)?; - self.inner - .switch_to(options.compression_method, options.compression_level)?; + self.inner.switch_to(make_new_self)?; self.writing_to_file = true; Ok(()) } @@ -668,7 +680,19 @@ impl ZipWriter { } let file = self.files.last_mut().unwrap(); - validate_extra_data(file)?; + if let Err(e) = validate_extra_data(file) { + let _ = self.abort_file(); + return Err(e); + } + + let make_compressing_writer = match self.inner + .prepare_switch_to(file.compression_method, file.compression_level) { + Ok(writer) => writer, + Err(e) => { + let _ = self.abort_file(); + return Err(e); + } + }; let mut data_start_result = file.data_start.load(); @@ -692,11 +716,8 @@ impl ZipWriter { writer.seek(SeekFrom::Start(file.header_start + 28))?; writer.write_u16::(extra_field_length)?; writer.seek(SeekFrom::Start(header_end))?; - - self.inner - .switch_to(file.compression_method, file.compression_level)?; } - + self.inner.switch_to(make_compressing_writer)?; self.writing_to_extra_field = false; self.writing_to_central_extra_field_only = false; Ok(data_start_result) @@ -957,53 +978,48 @@ impl Drop for ZipWriter { } impl GenericZipWriter { - fn switch_to( + fn prepare_switch_to( &mut self, compression: CompressionMethod, compression_level: Option, - ) -> ZipResult<()> { - match self.current_compression() { - Some(method) if method == compression => return Ok(()), - _ => {} - } - + ) -> ZipResult GenericZipWriter>> { if let Closed = self { return Err( io::Error::new(io::ErrorKind::BrokenPipe, "ZipWriter was already closed").into(), ); } - let make_new_self: Box GenericZipWriter> = { + { #[allow(deprecated)] match compression { CompressionMethod::Stored => { if compression_level.is_some() { - return Err(ZipError::UnsupportedArchive( + Err(ZipError::UnsupportedArchive( "Unsupported compression level", - )); + )) + } else { + Ok(Box::new(|bare| Storer(bare))) } - - Box::new(|bare| Storer(bare)) } #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" ))] CompressionMethod::Deflated => { let level = clamp_opt( compression_level.unwrap_or(flate2::Compression::default().level() as i32), deflate_compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32; - Box::new(move |bare| { + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; + Ok(Box::new(move |bare| { GenericZipWriter::Deflater(DeflateEncoder::new( bare, flate2::Compression::new(level), )) - }) + })) } #[cfg(feature = "bzip2")] CompressionMethod::Bzip2 => { @@ -1011,18 +1027,18 @@ impl GenericZipWriter { compression_level.unwrap_or(bzip2::Compression::default().level() as i32), bzip2_compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32; - Box::new(move |bare| { + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; + Ok(Box::new(move |bare| { GenericZipWriter::Bzip2(BzEncoder::new( bare, bzip2::Compression::new(level), )) - }) + })) } CompressionMethod::AES => { - return Err(ZipError::UnsupportedArchive( + Err(ZipError::UnsupportedArchive( "AES compression is not supported for writing", )) } @@ -1032,19 +1048,22 @@ impl GenericZipWriter { compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL), zstd::compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))?; - Box::new(move |bare| { + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))?; + Ok(Box::new(move |bare| { GenericZipWriter::Zstd(ZstdEncoder::new(bare, level).unwrap()) - }) + })) } CompressionMethod::Unsupported(..) => { - return Err(ZipError::UnsupportedArchive("Unsupported compression")) + Err(ZipError::UnsupportedArchive("Unsupported compression")) } } - }; + } + } + fn switch_to(&mut self, make_new_self: Box GenericZipWriter>) + -> ZipResult<()>{ let bare = match mem::replace(self, Closed) { Storer(w) => w, #[cfg(any( @@ -1061,8 +1080,7 @@ impl GenericZipWriter { return Err(io::Error::new( io::ErrorKind::BrokenPipe, "ZipWriter was already closed", - ) - .into()) + ).into()); } }; *self = (make_new_self)(bare); @@ -1097,23 +1115,6 @@ impl GenericZipWriter { } } - fn current_compression(&self) -> Option { - match *self { - Storer(..) => Some(CompressionMethod::Stored), - #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" - ))] - GenericZipWriter::Deflater(..) => Some(CompressionMethod::Deflated), - #[cfg(feature = "bzip2")] - GenericZipWriter::Bzip2(..) => Some(CompressionMethod::Bzip2), - #[cfg(feature = "zstd")] - GenericZipWriter::Zstd(..) => Some(CompressionMethod::Zstd), - Closed => None, - } - } - fn unwrap(self) -> W { match self { Storer(w) => w, From 304becbda37886c9a751febe910775e76660da1f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:21:15 -0700 Subject: [PATCH 101/281] Reformat --- src/write.rs | 59 +++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/src/write.rs b/src/write.rs index cbeb0e25..5e5c03a3 100644 --- a/src/write.rs +++ b/src/write.rs @@ -469,7 +469,9 @@ impl ZipWriter { // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. self.end_extra_data()?; } - let make_plain_writer = self.inner.prepare_switch_to(CompressionMethod::Stored, None)?; + let make_plain_writer = self + .inner + .prepare_switch_to(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; let writer = self.inner.get_plain(); @@ -498,8 +500,9 @@ impl ZipWriter { pub fn abort_file(&mut self) -> ZipResult<()> { self.files_by_name .remove(&*self.files.pop().unwrap().file_name); - let make_plain_writer - = self.inner.prepare_switch_to(CompressionMethod::Stored, None)?; + let make_plain_writer = self + .inner + .prepare_switch_to(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; self.writing_to_file = false; self.writing_raw = false; @@ -515,7 +518,8 @@ impl ZipWriter { S: Into, { Self::normalize_options(&mut options); - let make_new_self = self.inner + let make_new_self = self + .inner .prepare_switch_to(options.compression_method, options.compression_level)?; self.start_entry(name, options, None)?; self.inner.switch_to(make_new_self)?; @@ -685,8 +689,10 @@ impl ZipWriter { return Err(e); } - let make_compressing_writer = match self.inner - .prepare_switch_to(file.compression_method, file.compression_level) { + let make_compressing_writer = match self + .inner + .prepare_switch_to(file.compression_method, file.compression_level) + { Ok(writer) => writer, Err(e) => { let _ = self.abort_file(); @@ -1002,18 +1008,18 @@ impl GenericZipWriter { } } #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" ))] CompressionMethod::Deflated => { let level = clamp_opt( compression_level.unwrap_or(flate2::Compression::default().level() as i32), deflate_compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32; + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; Ok(Box::new(move |bare| { GenericZipWriter::Deflater(DeflateEncoder::new( bare, @@ -1027,9 +1033,9 @@ impl GenericZipWriter { compression_level.unwrap_or(bzip2::Compression::default().level() as i32), bzip2_compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))? as u32; + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))? as u32; Ok(Box::new(move |bare| { GenericZipWriter::Bzip2(BzEncoder::new( bare, @@ -1037,20 +1043,18 @@ impl GenericZipWriter { )) })) } - CompressionMethod::AES => { - Err(ZipError::UnsupportedArchive( - "AES compression is not supported for writing", - )) - } + CompressionMethod::AES => Err(ZipError::UnsupportedArchive( + "AES compression is not supported for writing", + )), #[cfg(feature = "zstd")] CompressionMethod::Zstd => { let level = clamp_opt( compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL), zstd::compression_level_range(), ) - .ok_or(ZipError::UnsupportedArchive( - "Unsupported compression level", - ))?; + .ok_or(ZipError::UnsupportedArchive( + "Unsupported compression level", + ))?; Ok(Box::new(move |bare| { GenericZipWriter::Zstd(ZstdEncoder::new(bare, level).unwrap()) })) @@ -1062,8 +1066,10 @@ impl GenericZipWriter { } } - fn switch_to(&mut self, make_new_self: Box GenericZipWriter>) - -> ZipResult<()>{ + fn switch_to( + &mut self, + make_new_self: Box GenericZipWriter>, + ) -> ZipResult<()> { let bare = match mem::replace(self, Closed) { Storer(w) => w, #[cfg(any( @@ -1080,7 +1086,8 @@ impl GenericZipWriter { return Err(io::Error::new( io::ErrorKind::BrokenPipe, "ZipWriter was already closed", - ).into()); + ) + .into()); } }; *self = (make_new_self)(bare); From 8bc1530b2d27ad060ee53804fbbca677c57b7f71 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:23:55 -0700 Subject: [PATCH 102/281] Bug fix --- src/write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 5e5c03a3..91f3f5ea 100644 --- a/src/write.rs +++ b/src/write.rs @@ -498,8 +498,8 @@ impl ZipWriter { /// Removes the file currently being written from the archive. pub fn abort_file(&mut self) -> ZipResult<()> { - self.files_by_name - .remove(&*self.files.pop().unwrap().file_name); + let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; + self.files_by_name.remove(&last_file.file_name); let make_plain_writer = self .inner .prepare_switch_to(CompressionMethod::Stored, None)?; From f165de226952582e6662821b0ae8082742640487 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:25:28 -0700 Subject: [PATCH 103/281] Update doc comment and CHANGELOG --- CHANGELOG.md | 12 +++++++++++- src/write.rs | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 140450b2..c6b438dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,4 +94,14 @@ ### Changed - - Bumped the version number in order to upload an updated README to crates.io. \ No newline at end of file + - Bumped the version number in order to upload an updated README to crates.io. + +## [0.7.2] + +### Added + + - Method `abort_file` - removes the current or most recently-finished file from the archive. + +### Fixed + + - Fixed a bug where a file could remain open for writing after validations failed. diff --git a/src/write.rs b/src/write.rs index 91f3f5ea..ef68cf50 100644 --- a/src/write.rs +++ b/src/write.rs @@ -496,7 +496,8 @@ impl ZipWriter { Ok(()) } - /// Removes the file currently being written from the archive. + /// Removes the file currently being written from the archive if there is one, or else removes + /// the file most recently written. pub fn abort_file(&mut self) -> ZipResult<()> { let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; self.files_by_name.remove(&last_file.file_name); From bb025ef19592be94b76ddfa0fa90cb5f66ab6219 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:46:56 -0700 Subject: [PATCH 104/281] Bug fix --- src/write.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/write.rs b/src/write.rs index ef68cf50..263e0232 100644 --- a/src/write.rs +++ b/src/write.rs @@ -505,6 +505,9 @@ impl ZipWriter { .inner .prepare_switch_to(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; self.writing_to_file = false; self.writing_raw = false; Ok(()) From 55d82a455cbe2b3bd10ecc1ffa92c07a9ac52302 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:47:33 -0700 Subject: [PATCH 105/281] Refactor: rename prepare_switch_to to prepare_next_writer --- src/write.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/write.rs b/src/write.rs index 263e0232..cd7aef7d 100644 --- a/src/write.rs +++ b/src/write.rs @@ -471,7 +471,7 @@ impl ZipWriter { } let make_plain_writer = self .inner - .prepare_switch_to(CompressionMethod::Stored, None)?; + .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; let writer = self.inner.get_plain(); @@ -503,7 +503,7 @@ impl ZipWriter { self.files_by_name.remove(&last_file.file_name); let make_plain_writer = self .inner - .prepare_switch_to(CompressionMethod::Stored, None)?; + .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; self.inner .get_plain() @@ -524,7 +524,7 @@ impl ZipWriter { Self::normalize_options(&mut options); let make_new_self = self .inner - .prepare_switch_to(options.compression_method, options.compression_level)?; + .prepare_next_writer(options.compression_method, options.compression_level)?; self.start_entry(name, options, None)?; self.inner.switch_to(make_new_self)?; self.writing_to_file = true; @@ -695,7 +695,7 @@ impl ZipWriter { let make_compressing_writer = match self .inner - .prepare_switch_to(file.compression_method, file.compression_level) + .prepare_next_writer(file.compression_method, file.compression_level) { Ok(writer) => writer, Err(e) => { @@ -988,7 +988,7 @@ impl Drop for ZipWriter { } impl GenericZipWriter { - fn prepare_switch_to( + fn prepare_next_writer( &mut self, compression: CompressionMethod, compression_level: Option, From a41c92d164145da2a50f908a1864740ec9b6eb06 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 10:48:21 -0700 Subject: [PATCH 106/281] Refactor: don't need to be mutable during prepare_next_writer --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index cd7aef7d..f8af1844 100644 --- a/src/write.rs +++ b/src/write.rs @@ -989,7 +989,7 @@ impl Drop for ZipWriter { impl GenericZipWriter { fn prepare_next_writer( - &mut self, + &self, compression: CompressionMethod, compression_level: Option, ) -> ZipResult GenericZipWriter>> { From 26bf30fc256dc83561f5222726b1c4a5bb7bbcf5 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 11:12:01 -0700 Subject: [PATCH 107/281] Bug fix: abort file if switch_to fails --- src/write.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index f8af1844..8ade9ea1 100644 --- a/src/write.rs +++ b/src/write.rs @@ -501,13 +501,13 @@ impl ZipWriter { pub fn abort_file(&mut self) -> ZipResult<()> { let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; self.files_by_name.remove(&last_file.file_name); + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; let make_plain_writer = self .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; - self.inner - .get_plain() - .seek(SeekFrom::Start(last_file.header_start))?; self.writing_to_file = false; self.writing_raw = false; Ok(()) @@ -526,7 +526,10 @@ impl ZipWriter { .inner .prepare_next_writer(options.compression_method, options.compression_level)?; self.start_entry(name, options, None)?; - self.inner.switch_to(make_new_self)?; + if let Err(e) = self.inner.switch_to(make_new_self) { + let _ = self.abort_file(); + return Err(e); + } self.writing_to_file = true; Ok(()) } From e8208dda87153e268d59d41deede3c097f230988 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 11:29:47 -0700 Subject: [PATCH 108/281] Triple write-fuzz duration to ensure remaining bugs are caught --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f1d0dd92..01c912c1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -runs=1000000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=1s -runs=3000000 -max_len=5000000000 From 31998d90e94235126ce3d66efb87acbdd2a36633 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 11:45:33 -0700 Subject: [PATCH 109/281] Use unwrap to check that abort_file never fails --- src/write.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8ade9ea1..286a2bb1 100644 --- a/src/write.rs +++ b/src/write.rs @@ -226,7 +226,7 @@ impl Write for ZipWriter { if self.stats.bytes_written > spec::ZIP64_BYTES_THR && !self.files.last_mut().unwrap().large_file { - let _ = self.abort_file(); + self.abort_file().unwrap(); return Err(io::Error::new( io::ErrorKind::Other, "Large file option has not been set", @@ -527,7 +527,7 @@ impl ZipWriter { .prepare_next_writer(options.compression_method, options.compression_level)?; self.start_entry(name, options, None)?; if let Err(e) = self.inner.switch_to(make_new_self) { - let _ = self.abort_file(); + self.abort_file().unwrap(); return Err(e); } self.writing_to_file = true; @@ -692,7 +692,7 @@ impl ZipWriter { let file = self.files.last_mut().unwrap(); if let Err(e) = validate_extra_data(file) { - let _ = self.abort_file(); + self.abort_file().unwrap(); return Err(e); } @@ -702,7 +702,7 @@ impl ZipWriter { { Ok(writer) => writer, Err(e) => { - let _ = self.abort_file(); + self.abort_file().unwrap(); return Err(e); } }; From 909473caea6298d1c649e3f709fcbaf69b25b146 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:06:23 -0700 Subject: [PATCH 110/281] Bug fix --- src/write.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index 286a2bb1..1590bed4 100644 --- a/src/write.rs +++ b/src/write.rs @@ -316,9 +316,7 @@ impl ZipWriter { let src_data = &self.files[src_index]; let data_start = src_data.data_start.load(); let compressed_size = src_data.compressed_size; - if compressed_size > write_position - data_start { - return Err(ZipError::InvalidArchive("Source file size too large")); - } + debug_assert!(compressed_size <= write_position - data_start); let uncompressed_size = src_data.uncompressed_size; let mut options = FileOptions::default() .large_file(compressed_size.max(uncompressed_size) > spec::ZIP64_BYTES_THR) @@ -346,7 +344,11 @@ impl ZipWriter { self.start_entry(dest_name, options, Some(raw_values))?; self.writing_to_file = true; self.writing_raw = true; - Ok(self.write_all(©)?) + if let Err(e) = self.write_all(©) { + self.abort_file().unwrap(); + return Err(e.into()); + } + self.finish_file() } } From e2f9d25b1b8bd65145cbe332302e2c23ed11cf03 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:07:34 -0700 Subject: [PATCH 111/281] Bug fix --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index 1590bed4..729d9395 100644 --- a/src/write.rs +++ b/src/write.rs @@ -510,6 +510,7 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; + self.writing_to_extra_field = false; self.writing_to_file = false; self.writing_raw = false; Ok(()) From 6e67bfcda5d7c4c45a942db0bfef98e5b7d0eb23 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:13:19 -0700 Subject: [PATCH 112/281] Bug fix --- src/write.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 729d9395..664648d6 100644 --- a/src/write.rs +++ b/src/write.rs @@ -348,7 +348,9 @@ impl ZipWriter { self.abort_file().unwrap(); return Err(e.into()); } - self.finish_file() + self.writing_to_file = false; + self.writing_raw = false; + Ok(()) } } From 425a05e1c84de33a53bfe61b94e376132662a9f2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:14:30 -0700 Subject: [PATCH 113/281] Revert "Bug fix" This reverts commit 6e67bfcda5d7c4c45a942db0bfef98e5b7d0eb23. --- src/write.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/write.rs b/src/write.rs index 664648d6..729d9395 100644 --- a/src/write.rs +++ b/src/write.rs @@ -348,9 +348,7 @@ impl ZipWriter { self.abort_file().unwrap(); return Err(e.into()); } - self.writing_to_file = false; - self.writing_raw = false; - Ok(()) + self.finish_file() } } From ea25dbd9548bc142a8f2526457e078ffec137aad Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:23:14 -0700 Subject: [PATCH 114/281] Bug fix --- src/write.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/write.rs b/src/write.rs index 729d9395..9f56c77f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -467,6 +467,9 @@ impl ZipWriter { } fn finish_file(&mut self) -> ZipResult<()> { + if !self.writing_to_file { + return Ok(()); + } if self.writing_to_extra_field { // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. self.end_extra_data()?; From 697578b38d304e822a4e5971dfc8e5f05e3a287d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:26:10 -0700 Subject: [PATCH 115/281] Add assertions for state flags in finish_file --- src/write.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/write.rs b/src/write.rs index 9f56c77f..79c1769d 100644 --- a/src/write.rs +++ b/src/write.rs @@ -468,6 +468,8 @@ impl ZipWriter { fn finish_file(&mut self) -> ZipResult<()> { if !self.writing_to_file { + debug_assert!(!self.writing_raw); + debug_assert!(!self.writing_to_extra_field); return Ok(()); } if self.writing_to_extra_field { From 5e0b216320a332918e34cdeacf0ac67e859bb695 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:34:31 -0700 Subject: [PATCH 116/281] Bug fix --- src/write.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 79c1769d..a71af4cb 100644 --- a/src/write.rs +++ b/src/write.rs @@ -909,8 +909,11 @@ impl ZipWriter { self.start_entry(name, options, None)?; self.writing_to_file = true; - self.write_all(target.into().as_bytes())?; - self.writing_to_file = false; + if let Err(e) = self.write_all(target.into().as_bytes()) { + self.abort_file().unwrap(); + return Err(e.into()); + } + self.finish_file()?; Ok(()) } From b8d393a628311a162468a3e3564de91ff3635d23 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:43:28 -0700 Subject: [PATCH 117/281] Bug fix --- src/write.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index a71af4cb..8f365860 100644 --- a/src/write.rs +++ b/src/write.rs @@ -468,7 +468,6 @@ impl ZipWriter { fn finish_file(&mut self) -> ZipResult<()> { if !self.writing_to_file { - debug_assert!(!self.writing_raw); debug_assert!(!self.writing_to_extra_field); return Ok(()); } @@ -1599,6 +1598,9 @@ mod test { writer .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .unwrap(); + writer + .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); let zip = writer.finish().unwrap(); let mut reader = ZipArchive::new(zip).unwrap(); let mut file_names: Vec<&str> = reader.file_names().collect(); From 6003a2cce5095185f46e63981d7bc6ca4dc2d31a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 12:46:46 -0700 Subject: [PATCH 118/281] Bug fix --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 8f365860..1282ea6f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -498,7 +498,6 @@ impl ZipWriter { } self.writing_to_file = false; - self.writing_raw = false; Ok(()) } @@ -538,6 +537,7 @@ impl ZipWriter { return Err(e); } self.writing_to_file = true; + self.writing_raw = false; Ok(()) } From 2f87bd6cfc56ffd8d2d960d969b03faa0ba81a05 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 15:03:25 -0700 Subject: [PATCH 119/281] Bump version to 0.7.2 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a88f532a..64a1bd13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.1" +version = "0.7.2" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index e6835ef1..d6cd8617 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.1" +zip_next = "0.7.2" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.1", default-features = false } +zip_next = { version = "0.7.2", default-features = false } ``` The features available are: From 08ec7ca7107c189ee2214f8267eda16d2edd2bdf Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 16:18:48 -0700 Subject: [PATCH 120/281] Remove extra data from struct since we're not using it --- fuzz/fuzz_targets/fuzz_write.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 0e1d57fe..eb700cf9 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -4,18 +4,10 @@ use libfuzzer_sys::fuzz_target; use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; -#[derive(Arbitrary,Debug)] -pub struct ExtraData { - pub header_id: u16, - pub data: Vec -} - #[derive(Arbitrary,Debug)] pub struct File { pub name: String, - pub contents: Vec>, - pub local_extra_data: Vec, - pub central_extra_data: Vec + pub contents: Vec> } #[derive(Arbitrary,Debug)] From b1ac3912db6e5095c61ed118ac29ca9784b1aba9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 16:22:19 -0700 Subject: [PATCH 121/281] Test opening for append and creating another copy --- src/write.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 1282ea6f..da82fb7b 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1582,6 +1582,8 @@ mod test { const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; #[cfg(test)] const SECOND_FILENAME: &str = "different_name.xyz"; + #[cfg(test)] + const THIRD_FILENAME: &str = "third_name.xyz"; #[test] fn test_shallow_copy() { @@ -1640,10 +1642,13 @@ mod test { .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .unwrap(); let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer.deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME).unwrap(); + let zip = writer.finish().unwrap(); let mut reader = ZipArchive::new(zip).unwrap(); let mut file_names: Vec<&str> = reader.file_names().collect(); file_names.sort(); - let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME]; + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME, THIRD_FILENAME]; expected_file_names.sort(); assert_eq!(file_names, expected_file_names); let mut first_file_content = String::new(); From 44d179355c6e2f01072410f0c9ec4a9f043beb61 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 16:35:04 -0700 Subject: [PATCH 122/281] Reformat --- src/write.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index da82fb7b..62fad178 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1600,6 +1600,11 @@ mod test { writer .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); writer .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) .expect_err("Duplicate filename"); @@ -1643,7 +1648,9 @@ mod test { .unwrap(); let zip = writer.finish().unwrap(); let mut writer = ZipWriter::new_append(zip).unwrap(); - writer.deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME) + .unwrap(); let zip = writer.finish().unwrap(); let mut reader = ZipArchive::new(zip).unwrap(); let mut file_names: Vec<&str> = reader.file_names().collect(); From 475b55df1d87b870870faf0ee62a676940b0d765 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 20:29:26 -0700 Subject: [PATCH 123/281] Perform an extra sanity check on ZIP64 detection --- src/read.rs | 80 ++++++++++++++++++++++++++++++---------------------- src/write.rs | 13 +++++++++ 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/src/read.rs b/src/read.rs index e7400ef3..a76ceb45 100644 --- a/src/read.rs +++ b/src/read.rs @@ -70,7 +70,9 @@ pub(crate) mod zip_archive { } } +use crate::spec::CentralDirectoryEnd; pub use zip_archive::ZipArchive; + #[allow(clippy::large_enum_variant)] pub(crate) enum CryptoReader<'a> { Plaintext(io::Take<&'a mut dyn Read>), @@ -121,14 +123,14 @@ impl<'a> CryptoReader<'a> { pub(crate) enum ZipFileReader<'a> { NoReader, - Raw(io::Take<&'a mut dyn io::Read>), + Raw(io::Take<&'a mut dyn Read>), Stored(Crc32Reader>), #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] - Deflated(Crc32Reader>>), + Deflated(Crc32Reader>>), #[cfg(feature = "bzip2")] Bzip2(Crc32Reader>>), #[cfg(feature = "zstd")] @@ -207,11 +209,11 @@ pub(crate) fn find_content<'a>( #[allow(clippy::too_many_arguments)] pub(crate) fn make_crypto_reader<'a>( - compression_method: crate::compression::CompressionMethod, + compression_method: CompressionMethod, crc32: u32, last_modified_time: DateTime, using_data_descriptor: bool, - reader: io::Take<&'a mut dyn io::Read>, + reader: io::Take<&'a mut dyn Read>, password: Option<&[u8]>, aes_info: Option<(AesMode, AesVendorVersion)>, #[cfg(feature = "aes-crypto")] compressed_size: u64, @@ -291,12 +293,12 @@ pub(crate) fn make_reader( } } -impl ZipArchive { +impl ZipArchive { /// Get the directory start offset and number of files. This is done in a /// separate function to ease the control flow design. pub(crate) fn get_directory_counts( reader: &mut R, - footer: &spec::CentralDirectoryEnd, + footer: &CentralDirectoryEnd, cde_start_pos: u64, ) -> ZipResult<(u64, u64, usize)> { // See if there's a ZIP64 footer. The ZIP64 locator if present will @@ -327,24 +329,18 @@ impl ZipArchive { }; match zip64locator { - None => { - // Some zip files have data prepended to them, resulting in the - // offsets all being too small. Get the amount of error by comparing - // the actual file position we found the CDE at with the offset - // recorded in the CDE. - let archive_offset = cde_start_pos - .checked_sub(footer.central_directory_size as u64) - .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) - .ok_or(ZipError::InvalidArchive( - "Invalid central directory size or offset", - ))?; - - let directory_start = footer.central_directory_offset as u64 + archive_offset; - let number_of_files = footer.number_of_files_on_this_disk as usize; - Ok((archive_offset, directory_start, number_of_files)) - } + None => Self::get_directory_counts_zip32(footer, cde_start_pos), Some(locator64) => { - // If we got here, this is indeed a ZIP64 file. + if let Ok((archive_offset, directory_start, number_of_files)) = + Self::get_directory_counts_zip32(footer, cde_start_pos) + { + if directory_start >= locator64.end_of_central_directory_offset { + // This isn't a ZIP64, it's a ZIP that happens to have ZIP64 magic in a filename + return Ok((archive_offset, directory_start, number_of_files)); + } + } + + // If we got here, we're pretty sure this is a ZIP64 file. if !footer.record_too_small() && footer.disk_number as u32 != locator64.disk_with_central_directory @@ -395,11 +391,31 @@ impl ZipArchive { } } + fn get_directory_counts_zip32( + footer: &CentralDirectoryEnd, + cde_start_pos: u64, + ) -> Result<(u64, u64, usize), ZipError> { + // Some zip files have data prepended to them, resulting in the + // offsets all being too small. Get the amount of error by comparing + // the actual file position we found the CDE at with the offset + // recorded in the CDE. + let archive_offset = cde_start_pos + .checked_sub(footer.central_directory_size as u64) + .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) + .ok_or(ZipError::InvalidArchive( + "Invalid central directory size or offset", + ))?; + + let directory_start = footer.central_directory_offset as u64 + archive_offset; + let number_of_files = footer.number_of_files_on_this_disk as usize; + Ok((archive_offset, directory_start, number_of_files)) + } + /// Read a ZIP archive, collecting the files it contains /// /// This uses the central directory record of the ZIP file, and ignores local file headers pub fn new(mut reader: R) -> ZipResult> { - let (footer, cde_start_pos) = spec::CentralDirectoryEnd::find_and_parse(&mut reader)?; + let (footer, cde_start_pos) = CentralDirectoryEnd::find_and_parse(&mut reader)?; if !footer.record_too_small() && footer.disk_number != footer.disk_with_central_directory { return unsupported_zip_error("Support for multi-disk files is not implemented"); @@ -643,7 +659,7 @@ fn unsupported_zip_error(detail: &'static str) -> ZipResult { } /// Parse a central directory entry to collect the information for the file. -pub(crate) fn central_header_to_zip_file( +pub(crate) fn central_header_to_zip_file( reader: &mut R, archive_offset: u64, ) -> ZipResult { @@ -873,7 +889,7 @@ impl<'a> ZipFile<'a> { note = "by stripping `..`s from the path, the meaning of paths can change. `mangled_name` can be used if this behaviour is desirable" )] - pub fn sanitized_name(&self) -> ::std::path::PathBuf { + pub fn sanitized_name(&self) -> std::path::PathBuf { self.mangled_name() } @@ -889,7 +905,7 @@ impl<'a> ZipFile<'a> { /// [`ZipFile::enclosed_name`] is the better option in most scenarios. /// /// [`ParentDir`]: `Component::ParentDir` - pub fn mangled_name(&self) -> ::std::path::PathBuf { + pub fn mangled_name(&self) -> std::path::PathBuf { self.data.file_name_sanitized() } @@ -989,13 +1005,13 @@ impl<'a> Drop for ZipFile<'a> { let mut buffer = [0; 1 << 16]; // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. - let mut reader: std::io::Take<&mut dyn std::io::Read> = match &mut self.reader { + let mut reader: io::Take<&mut dyn Read> = match &mut self.reader { ZipFileReader::NoReader => { let innerreader = self.crypto_reader.take(); innerreader.expect("Invalid reader state").into_inner() } reader => { - let innerreader = ::std::mem::replace(reader, ZipFileReader::NoReader); + let innerreader = std::mem::replace(reader, ZipFileReader::NoReader); innerreader.into_inner() } }; @@ -1029,9 +1045,7 @@ impl<'a> Drop for ZipFile<'a> { /// * `comment`: set to an empty string /// * `data_start`: set to 0 /// * `external_attributes`: `unix_mode()`: will return None -pub fn read_zipfile_from_stream<'a, R: io::Read>( - reader: &'a mut R, -) -> ZipResult>> { +pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult>> { let signature = reader.read_u32::()?; match signature { @@ -1105,7 +1119,7 @@ pub fn read_zipfile_from_stream<'a, R: io::Read>( return unsupported_zip_error("The file length is not available in the local header"); } - let limit_reader = (reader as &'a mut dyn io::Read).take(result.compressed_size); + let limit_reader = (reader as &'a mut dyn Read).take(result.compressed_size); let result_crc32 = result.crc32; let result_compression_method = result.compression_method; diff --git a/src/write.rs b/src/write.rs index 62fad178..0226a151 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1688,6 +1688,19 @@ mod test { .expect_err("Expected duplicate filename not to be allowed"); } + #[test] + fn test_filename_looks_like_zip64_locator() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + #[test] fn path_to_string() { let mut path = std::path::PathBuf::new(); From 6e7ff280e96bb32846a93bdd165bd03bc063de01 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 4 May 2023 20:34:44 -0700 Subject: [PATCH 124/281] Revert "Perform an extra sanity check on ZIP64 detection" This reverts commit 475b55df1d87b870870faf0ee62a676940b0d765. --- src/read.rs | 80 ++++++++++++++++++++++------------------------------ src/write.rs | 13 --------- 2 files changed, 33 insertions(+), 60 deletions(-) diff --git a/src/read.rs b/src/read.rs index a76ceb45..e7400ef3 100644 --- a/src/read.rs +++ b/src/read.rs @@ -70,9 +70,7 @@ pub(crate) mod zip_archive { } } -use crate::spec::CentralDirectoryEnd; pub use zip_archive::ZipArchive; - #[allow(clippy::large_enum_variant)] pub(crate) enum CryptoReader<'a> { Plaintext(io::Take<&'a mut dyn Read>), @@ -123,14 +121,14 @@ impl<'a> CryptoReader<'a> { pub(crate) enum ZipFileReader<'a> { NoReader, - Raw(io::Take<&'a mut dyn Read>), + Raw(io::Take<&'a mut dyn io::Read>), Stored(Crc32Reader>), #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] - Deflated(Crc32Reader>>), + Deflated(Crc32Reader>>), #[cfg(feature = "bzip2")] Bzip2(Crc32Reader>>), #[cfg(feature = "zstd")] @@ -209,11 +207,11 @@ pub(crate) fn find_content<'a>( #[allow(clippy::too_many_arguments)] pub(crate) fn make_crypto_reader<'a>( - compression_method: CompressionMethod, + compression_method: crate::compression::CompressionMethod, crc32: u32, last_modified_time: DateTime, using_data_descriptor: bool, - reader: io::Take<&'a mut dyn Read>, + reader: io::Take<&'a mut dyn io::Read>, password: Option<&[u8]>, aes_info: Option<(AesMode, AesVendorVersion)>, #[cfg(feature = "aes-crypto")] compressed_size: u64, @@ -293,12 +291,12 @@ pub(crate) fn make_reader( } } -impl ZipArchive { +impl ZipArchive { /// Get the directory start offset and number of files. This is done in a /// separate function to ease the control flow design. pub(crate) fn get_directory_counts( reader: &mut R, - footer: &CentralDirectoryEnd, + footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, ) -> ZipResult<(u64, u64, usize)> { // See if there's a ZIP64 footer. The ZIP64 locator if present will @@ -329,18 +327,24 @@ impl ZipArchive { }; match zip64locator { - None => Self::get_directory_counts_zip32(footer, cde_start_pos), - Some(locator64) => { - if let Ok((archive_offset, directory_start, number_of_files)) = - Self::get_directory_counts_zip32(footer, cde_start_pos) - { - if directory_start >= locator64.end_of_central_directory_offset { - // This isn't a ZIP64, it's a ZIP that happens to have ZIP64 magic in a filename - return Ok((archive_offset, directory_start, number_of_files)); - } - } + None => { + // Some zip files have data prepended to them, resulting in the + // offsets all being too small. Get the amount of error by comparing + // the actual file position we found the CDE at with the offset + // recorded in the CDE. + let archive_offset = cde_start_pos + .checked_sub(footer.central_directory_size as u64) + .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) + .ok_or(ZipError::InvalidArchive( + "Invalid central directory size or offset", + ))?; - // If we got here, we're pretty sure this is a ZIP64 file. + let directory_start = footer.central_directory_offset as u64 + archive_offset; + let number_of_files = footer.number_of_files_on_this_disk as usize; + Ok((archive_offset, directory_start, number_of_files)) + } + Some(locator64) => { + // If we got here, this is indeed a ZIP64 file. if !footer.record_too_small() && footer.disk_number as u32 != locator64.disk_with_central_directory @@ -391,31 +395,11 @@ impl ZipArchive { } } - fn get_directory_counts_zip32( - footer: &CentralDirectoryEnd, - cde_start_pos: u64, - ) -> Result<(u64, u64, usize), ZipError> { - // Some zip files have data prepended to them, resulting in the - // offsets all being too small. Get the amount of error by comparing - // the actual file position we found the CDE at with the offset - // recorded in the CDE. - let archive_offset = cde_start_pos - .checked_sub(footer.central_directory_size as u64) - .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) - .ok_or(ZipError::InvalidArchive( - "Invalid central directory size or offset", - ))?; - - let directory_start = footer.central_directory_offset as u64 + archive_offset; - let number_of_files = footer.number_of_files_on_this_disk as usize; - Ok((archive_offset, directory_start, number_of_files)) - } - /// Read a ZIP archive, collecting the files it contains /// /// This uses the central directory record of the ZIP file, and ignores local file headers pub fn new(mut reader: R) -> ZipResult> { - let (footer, cde_start_pos) = CentralDirectoryEnd::find_and_parse(&mut reader)?; + let (footer, cde_start_pos) = spec::CentralDirectoryEnd::find_and_parse(&mut reader)?; if !footer.record_too_small() && footer.disk_number != footer.disk_with_central_directory { return unsupported_zip_error("Support for multi-disk files is not implemented"); @@ -659,7 +643,7 @@ fn unsupported_zip_error(detail: &'static str) -> ZipResult { } /// Parse a central directory entry to collect the information for the file. -pub(crate) fn central_header_to_zip_file( +pub(crate) fn central_header_to_zip_file( reader: &mut R, archive_offset: u64, ) -> ZipResult { @@ -889,7 +873,7 @@ impl<'a> ZipFile<'a> { note = "by stripping `..`s from the path, the meaning of paths can change. `mangled_name` can be used if this behaviour is desirable" )] - pub fn sanitized_name(&self) -> std::path::PathBuf { + pub fn sanitized_name(&self) -> ::std::path::PathBuf { self.mangled_name() } @@ -905,7 +889,7 @@ impl<'a> ZipFile<'a> { /// [`ZipFile::enclosed_name`] is the better option in most scenarios. /// /// [`ParentDir`]: `Component::ParentDir` - pub fn mangled_name(&self) -> std::path::PathBuf { + pub fn mangled_name(&self) -> ::std::path::PathBuf { self.data.file_name_sanitized() } @@ -1005,13 +989,13 @@ impl<'a> Drop for ZipFile<'a> { let mut buffer = [0; 1 << 16]; // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. - let mut reader: io::Take<&mut dyn Read> = match &mut self.reader { + let mut reader: std::io::Take<&mut dyn std::io::Read> = match &mut self.reader { ZipFileReader::NoReader => { let innerreader = self.crypto_reader.take(); innerreader.expect("Invalid reader state").into_inner() } reader => { - let innerreader = std::mem::replace(reader, ZipFileReader::NoReader); + let innerreader = ::std::mem::replace(reader, ZipFileReader::NoReader); innerreader.into_inner() } }; @@ -1045,7 +1029,9 @@ impl<'a> Drop for ZipFile<'a> { /// * `comment`: set to an empty string /// * `data_start`: set to 0 /// * `external_attributes`: `unix_mode()`: will return None -pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult>> { +pub fn read_zipfile_from_stream<'a, R: io::Read>( + reader: &'a mut R, +) -> ZipResult>> { let signature = reader.read_u32::()?; match signature { @@ -1119,7 +1105,7 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult Date: Fri, 5 May 2023 09:11:49 -0700 Subject: [PATCH 125/281] Bug fix: try decoding file as ZIP32 if it's not valid as ZIP64 --- src/read.rs | 192 +++++++++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 101 deletions(-) diff --git a/src/read.rs b/src/read.rs index e7400ef3..ebde0209 100644 --- a/src/read.rs +++ b/src/read.rs @@ -121,14 +121,14 @@ impl<'a> CryptoReader<'a> { pub(crate) enum ZipFileReader<'a> { NoReader, - Raw(io::Take<&'a mut dyn io::Read>), + Raw(io::Take<&'a mut dyn Read>), Stored(Crc32Reader>), #[cfg(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib" ))] - Deflated(Crc32Reader>>), + Deflated(Crc32Reader>>), #[cfg(feature = "bzip2")] Bzip2(Crc32Reader>>), #[cfg(feature = "zstd")] @@ -207,11 +207,11 @@ pub(crate) fn find_content<'a>( #[allow(clippy::too_many_arguments)] pub(crate) fn make_crypto_reader<'a>( - compression_method: crate::compression::CompressionMethod, + compression_method: CompressionMethod, crc32: u32, last_modified_time: DateTime, using_data_descriptor: bool, - reader: io::Take<&'a mut dyn io::Read>, + reader: io::Take<&'a mut dyn Read>, password: Option<&[u8]>, aes_info: Option<(AesMode, AesVendorVersion)>, #[cfg(feature = "aes-crypto")] compressed_size: u64, @@ -291,10 +291,28 @@ pub(crate) fn make_reader( } } -impl ZipArchive { - /// Get the directory start offset and number of files. This is done in a - /// separate function to ease the control flow design. - pub(crate) fn get_directory_counts( +impl ZipArchive { + fn get_directory_counts_zip32( + footer: &spec::CentralDirectoryEnd, + cde_start_pos: u64, + ) -> ZipResult<(u64, u64, usize)> { + // Some zip files have data prepended to them, resulting in the + // offsets all being too small. Get the amount of error by comparing + // the actual file position we found the CDE at with the offset + // recorded in the CDE. + let archive_offset = cde_start_pos + .checked_sub(footer.central_directory_size as u64) + .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) + .ok_or(ZipError::InvalidArchive( + "Invalid central directory size or offset", + ))?; + + let directory_start = footer.central_directory_offset as u64 + archive_offset; + let number_of_files = footer.number_of_files_on_this_disk as usize; + Ok((archive_offset, directory_start, number_of_files)) + } + + fn get_directory_counts_zip64( reader: &mut R, footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, @@ -303,96 +321,68 @@ impl ZipArchive { // have its signature 20 bytes in front of the standard footer. The // standard footer, in turn, is 22+N bytes large, where N is the // comment length. Therefore: - let zip64locator = if reader + let locator64 = reader .seek(io::SeekFrom::End( -(20 + 22 + footer.zip_file_comment.len() as i64), - )) - .is_ok() + ))?; + if !footer.record_too_small() + && footer.disk_number as u32 != locator64.disk_with_central_directory { - match spec::Zip64CentralDirectoryEndLocator::parse(reader) { - Ok(loc) => Some(loc), - Err(ZipError::InvalidArchive(_)) => { - // No ZIP64 header; that's actually fine. We're done here. - None - } - Err(e) => { - // Yikes, a real problem - return Err(e); - } - } - } else { - // Empty Zip files will have nothing else so this error might be fine. If - // not, we'll find out soon. - None - }; - - match zip64locator { - None => { - // Some zip files have data prepended to them, resulting in the - // offsets all being too small. Get the amount of error by comparing - // the actual file position we found the CDE at with the offset - // recorded in the CDE. - let archive_offset = cde_start_pos - .checked_sub(footer.central_directory_size as u64) - .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) - .ok_or(ZipError::InvalidArchive( - "Invalid central directory size or offset", - ))?; - - let directory_start = footer.central_directory_offset as u64 + archive_offset; - let number_of_files = footer.number_of_files_on_this_disk as usize; - Ok((archive_offset, directory_start, number_of_files)) - } - Some(locator64) => { - // If we got here, this is indeed a ZIP64 file. - - if !footer.record_too_small() - && footer.disk_number as u32 != locator64.disk_with_central_directory - { - return unsupported_zip_error( - "Support for multi-disk files is not implemented", - ); - } - - // We need to reassess `archive_offset`. We know where the ZIP64 - // central-directory-end structure *should* be, but unfortunately we - // don't know how to precisely relate that location to our current - // actual offset in the file, since there may be junk at its - // beginning. Therefore we need to perform another search, as in - // read::CentralDirectoryEnd::find_and_parse, except now we search - // forward. - - let search_upper_bound = cde_start_pos - .checked_sub(60) // minimum size of Zip64CentralDirectoryEnd + Zip64CentralDirectoryEndLocator - .ok_or(ZipError::InvalidArchive( - "File cannot contain ZIP64 central directory end", - ))?; - let (footer, archive_offset) = spec::Zip64CentralDirectoryEnd::find_and_parse( - reader, - locator64.end_of_central_directory_offset, - search_upper_bound, - )?; - - if footer.disk_number != footer.disk_with_central_directory { - return unsupported_zip_error( - "Support for multi-disk files is not implemented", - ); - } - - let directory_start = footer - .central_directory_offset - .checked_add(archive_offset) - .ok_or({ - ZipError::InvalidArchive("Invalid central directory size or offset") - })?; - - Ok(( - archive_offset, - directory_start, - footer.number_of_files as usize, - )) - } + return unsupported_zip_error( + "Support for multi-disk files is not implemented", + ); } + + // We need to reassess `archive_offset`. We know where the ZIP64 + // central-directory-end structure *should* be, but unfortunately we + // don't know how to precisely relate that location to our current + // actual offset in the file, since there may be junk at its + // beginning. Therefore we need to perform another search, as in + // read::CentralDirectoryEnd::find_and_parse, except now we search + // forward. + + let search_upper_bound = cde_start_pos + .checked_sub(60) // minimum size of Zip64CentralDirectoryEnd + Zip64CentralDirectoryEndLocator + .ok_or(ZipError::InvalidArchive( + "File cannot contain ZIP64 central directory end", + ))?; + let (footer, archive_offset) = spec::Zip64CentralDirectoryEnd::find_and_parse( + reader, + locator64.end_of_central_directory_offset, + search_upper_bound, + )?; + + if footer.disk_number != footer.disk_with_central_directory { + return unsupported_zip_error( + "Support for multi-disk files is not implemented", + ); + } + + let directory_start = footer + .central_directory_offset + .checked_add(archive_offset) + .ok_or({ + ZipError::InvalidArchive("Invalid central directory size or offset") + })?; + + Ok(( + archive_offset, + directory_start, + footer.number_of_files as usize, + )) + } + + /// Get the directory start offset and number of files. This is done in a + /// separate function to ease the control flow design. + pub(crate) fn get_directory_counts( + reader: &mut R, + footer: &spec::CentralDirectoryEnd, + cde_start_pos: u64, + ) -> ZipResult<(u64, u64, usize)> { + // Check if file is valid as ZIP64 first; if not, try it as ZIP32 + Self::get_directory_counts_zip64(reader, footer, cde_start_pos).or_else( + || get_directory_counts_zip32(footer, cde_start_pos) + ) } /// Read a ZIP archive, collecting the files it contains @@ -643,7 +633,7 @@ fn unsupported_zip_error(detail: &'static str) -> ZipResult { } /// Parse a central directory entry to collect the information for the file. -pub(crate) fn central_header_to_zip_file( +pub(crate) fn central_header_to_zip_file( reader: &mut R, archive_offset: u64, ) -> ZipResult { @@ -873,7 +863,7 @@ impl<'a> ZipFile<'a> { note = "by stripping `..`s from the path, the meaning of paths can change. `mangled_name` can be used if this behaviour is desirable" )] - pub fn sanitized_name(&self) -> ::std::path::PathBuf { + pub fn sanitized_name(&self) -> std::path::PathBuf { self.mangled_name() } @@ -889,7 +879,7 @@ impl<'a> ZipFile<'a> { /// [`ZipFile::enclosed_name`] is the better option in most scenarios. /// /// [`ParentDir`]: `Component::ParentDir` - pub fn mangled_name(&self) -> ::std::path::PathBuf { + pub fn mangled_name(&self) -> std::path::PathBuf { self.data.file_name_sanitized() } @@ -989,13 +979,13 @@ impl<'a> Drop for ZipFile<'a> { let mut buffer = [0; 1 << 16]; // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. - let mut reader: std::io::Take<&mut dyn std::io::Read> = match &mut self.reader { + let mut reader: io::Take<&mut dyn Read> = match &mut self.reader { ZipFileReader::NoReader => { let innerreader = self.crypto_reader.take(); innerreader.expect("Invalid reader state").into_inner() } reader => { - let innerreader = ::std::mem::replace(reader, ZipFileReader::NoReader); + let innerreader = std::mem::replace(reader, ZipFileReader::NoReader); innerreader.into_inner() } }; @@ -1029,7 +1019,7 @@ impl<'a> Drop for ZipFile<'a> { /// * `comment`: set to an empty string /// * `data_start`: set to 0 /// * `external_attributes`: `unix_mode()`: will return None -pub fn read_zipfile_from_stream<'a, R: io::Read>( +pub fn read_zipfile_from_stream<'a, R: Read>( reader: &'a mut R, ) -> ZipResult>> { let signature = reader.read_u32::()?; @@ -1105,7 +1095,7 @@ pub fn read_zipfile_from_stream<'a, R: io::Read>( return unsupported_zip_error("The file length is not available in the local header"); } - let limit_reader = (reader as &'a mut dyn io::Read).take(result.compressed_size); + let limit_reader = (reader as &'a mut dyn Read).take(result.compressed_size); let result_crc32 = result.crc32; let result_compression_method = result.compression_method; From 0beb5b42949e368a32f933a660198dae5375bbfd Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 5 May 2023 09:15:46 -0700 Subject: [PATCH 126/281] Bug fixes and formatting --- src/read.rs | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/read.rs b/src/read.rs index ebde0209..0833d787 100644 --- a/src/read.rs +++ b/src/read.rs @@ -321,16 +321,14 @@ impl ZipArchive { // have its signature 20 bytes in front of the standard footer. The // standard footer, in turn, is 22+N bytes large, where N is the // comment length. Therefore: - let locator64 = reader - .seek(io::SeekFrom::End( - -(20 + 22 + footer.zip_file_comment.len() as i64), - ))?; + reader.seek(io::SeekFrom::End( + -(20 + 22 + footer.zip_file_comment.len() as i64), + ))?; + let locator64 = spec::Zip64CentralDirectoryEndLocator::parse(reader)?; if !footer.record_too_small() && footer.disk_number as u32 != locator64.disk_with_central_directory { - return unsupported_zip_error( - "Support for multi-disk files is not implemented", - ); + return unsupported_zip_error("Support for multi-disk files is not implemented"); } // We need to reassess `archive_offset`. We know where the ZIP64 @@ -353,17 +351,15 @@ impl ZipArchive { )?; if footer.disk_number != footer.disk_with_central_directory { - return unsupported_zip_error( - "Support for multi-disk files is not implemented", - ); + return unsupported_zip_error("Support for multi-disk files is not implemented"); } let directory_start = footer .central_directory_offset .checked_add(archive_offset) - .ok_or({ - ZipError::InvalidArchive("Invalid central directory size or offset") - })?; + .ok_or(ZipError::InvalidArchive( + "Invalid central directory size or offset", + ))?; Ok(( archive_offset, @@ -380,9 +376,8 @@ impl ZipArchive { cde_start_pos: u64, ) -> ZipResult<(u64, u64, usize)> { // Check if file is valid as ZIP64 first; if not, try it as ZIP32 - Self::get_directory_counts_zip64(reader, footer, cde_start_pos).or_else( - || get_directory_counts_zip32(footer, cde_start_pos) - ) + Self::get_directory_counts_zip64(reader, footer, cde_start_pos) + .or_else(|_| Self::get_directory_counts_zip32(footer, cde_start_pos)) } /// Read a ZIP archive, collecting the files it contains @@ -1019,9 +1014,7 @@ impl<'a> Drop for ZipFile<'a> { /// * `comment`: set to an empty string /// * `data_start`: set to 0 /// * `external_attributes`: `unix_mode()`: will return None -pub fn read_zipfile_from_stream<'a, R: Read>( - reader: &'a mut R, -) -> ZipResult>> { +pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult>> { let signature = reader.read_u32::()?; match signature { From 006fd57bf5d326cc8085ecf4b164d91dc35a701d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 5 May 2023 09:25:52 -0700 Subject: [PATCH 127/281] Bug fix: reject file that's valid but unsupported as ZIP64 --- src/read.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/read.rs b/src/read.rs index 0833d787..75096950 100644 --- a/src/read.rs +++ b/src/read.rs @@ -325,11 +325,6 @@ impl ZipArchive { -(20 + 22 + footer.zip_file_comment.len() as i64), ))?; let locator64 = spec::Zip64CentralDirectoryEndLocator::parse(reader)?; - if !footer.record_too_small() - && footer.disk_number as u32 != locator64.disk_with_central_directory - { - return unsupported_zip_error("Support for multi-disk files is not implemented"); - } // We need to reassess `archive_offset`. We know where the ZIP64 // central-directory-end structure *should* be, but unfortunately we @@ -344,27 +339,33 @@ impl ZipArchive { .ok_or(ZipError::InvalidArchive( "File cannot contain ZIP64 central directory end", ))?; - let (footer, archive_offset) = spec::Zip64CentralDirectoryEnd::find_and_parse( + let (footer64, archive_offset) = spec::Zip64CentralDirectoryEnd::find_and_parse( reader, locator64.end_of_central_directory_offset, search_upper_bound, )?; - if footer.disk_number != footer.disk_with_central_directory { - return unsupported_zip_error("Support for multi-disk files is not implemented"); - } - - let directory_start = footer + let directory_start = footer64 .central_directory_offset .checked_add(archive_offset) .ok_or(ZipError::InvalidArchive( "Invalid central directory size or offset", ))?; + if footer64.disk_number != footer64.disk_with_central_directory { + return unsupported_zip_error("Support for multi-disk files is not implemented"); + } + + if !footer.record_too_small() + && footer.disk_number as u32 != locator64.disk_with_central_directory + { + return unsupported_zip_error("Support for multi-disk files is not implemented"); + } + Ok(( archive_offset, directory_start, - footer.number_of_files as usize, + footer64.number_of_files as usize, )) } @@ -376,8 +377,11 @@ impl ZipArchive { cde_start_pos: u64, ) -> ZipResult<(u64, u64, usize)> { // Check if file is valid as ZIP64 first; if not, try it as ZIP32 - Self::get_directory_counts_zip64(reader, footer, cde_start_pos) - .or_else(|_| Self::get_directory_counts_zip32(footer, cde_start_pos)) + match Self::get_directory_counts_zip64(reader, footer, cde_start_pos) { + Ok(result) => Ok(result), + Err(ZipError::UnsupportedArchive(e)) => Err(ZipError::UnsupportedArchive(e)), + Err(_) => Self::get_directory_counts_zip32(footer, cde_start_pos), + } } /// Read a ZIP archive, collecting the files it contains From 4d5e9ad718494ad1226f598fdad3f16bdf2ce3f6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 5 May 2023 11:37:48 -0700 Subject: [PATCH 128/281] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6b438dd..a790a06f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,3 +105,9 @@ ### Fixed - Fixed a bug where a file could remain open for writing after validations failed. + +## [0.7.3] + +### Fixed + + - Fixed a bug that occurs when a filename in a ZIP32 file includes the ZIP64 magic bytes. From 88f4788758bb8273c5b8ed74e4dff46d85f50962 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 5 May 2023 11:38:56 -0700 Subject: [PATCH 129/281] Bump version to 0.7.3 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 64a1bd13..83df6493 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.2" +version = "0.7.3" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index d6cd8617..2eeeafa3 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.2" +zip_next = "0.7.3" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.2", default-features = false } +zip_next = { version = "0.7.3", default-features = false } ``` The features available are: From 89989e02a30c0cab262f33c4f69d14b46cce4413 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 8 May 2023 18:54:35 -0700 Subject: [PATCH 130/281] Reformat --- src/unstable.rs | 4 ++-- src/write.rs | 6 +++++- src/zipcrypto.rs | 1 - tests/zip_crypto.rs | 10 +++++++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/unstable.rs b/src/unstable.rs index f8b46a97..cc03ff9a 100644 --- a/src/unstable.rs +++ b/src/unstable.rs @@ -8,7 +8,7 @@ pub mod write { /// Unstable methods for [`FileOptions`]. pub trait FileOptionsExt { /// Write the file with the given password using the deprecated ZipCrypto algorithm. - /// + /// /// This is not recommended for new archives, as ZipCrypto is not secure. fn with_deprecated_encryption(self, password: &[u8]) -> Self; } @@ -17,4 +17,4 @@ pub mod write { self.with_deprecated_encryption(password) } } -} \ No newline at end of file +} diff --git a/src/write.rs b/src/write.rs index 45f36452..4c2fea55 100644 --- a/src/write.rs +++ b/src/write.rs @@ -475,7 +475,11 @@ impl ZipWriter { self.stats.hasher = Hasher::new(); } if let Some(keys) = options.encrypt_with { - let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { writer: mem::replace(&mut self.inner, Closed).unwrap(), buffer: vec![], keys }; + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + buffer: vec![], + keys, + }; let crypto_header = [0u8; 12]; zipwriter.write_all(&crypto_header)?; diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index bddc6408..7ad8ddcb 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -8,7 +8,6 @@ use std::fmt::{Debug, Formatter}; use std::hash::{Hash, Hasher}; use std::num::Wrapping; - /// A container to hold the current key state #[derive(Clone, Copy, Hash, Ord, PartialOrd, Eq, PartialEq)] pub(crate) struct ZipCryptoKeys { diff --git a/tests/zip_crypto.rs b/tests/zip_crypto.rs index af39e275..0f112cf5 100644 --- a/tests/zip_crypto.rs +++ b/tests/zip_crypto.rs @@ -22,11 +22,16 @@ use std::io::Read; #[test] fn encrypting_file() { - use zip::unstable::write::FileOptionsExt; use std::io::{Read, Write}; + use zip::unstable::write::FileOptionsExt; let mut buf = vec![0; 2048]; let mut archive = zip::write::ZipWriter::new(Cursor::new(&mut buf)); - archive.start_file("name", zip::write::FileOptions::default().with_deprecated_encryption(b"password")).unwrap(); + archive + .start_file( + "name", + zip::write::FileOptions::default().with_deprecated_encryption(b"password"), + ) + .unwrap(); archive.write_all(b"test").unwrap(); archive.finish().unwrap(); drop(archive); @@ -35,7 +40,6 @@ fn encrypting_file() { let mut buf = Vec::new(); file.read_to_end(&mut buf).unwrap(); assert_eq!(buf, b"test"); - } #[test] fn encrypted_file() { From 6c4ae5333a1860ca3b183953409f447adbeeed0a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 8 May 2023 18:55:28 -0700 Subject: [PATCH 131/281] Fix merge --- CHANGELOG.md | 4 ++-- tests/zip_crypto.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a64c400..da190dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,5 +116,5 @@ ### Changed -- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files - with PKWARE encryption. +- Added experimental [`zip_next::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting + files with PKWARE encryption. diff --git a/tests/zip_crypto.rs b/tests/zip_crypto.rs index 0f112cf5..45612aac 100644 --- a/tests/zip_crypto.rs +++ b/tests/zip_crypto.rs @@ -23,19 +23,19 @@ use std::io::Read; #[test] fn encrypting_file() { use std::io::{Read, Write}; - use zip::unstable::write::FileOptionsExt; + use zip_next::unstable::write::FileOptionsExt; let mut buf = vec![0; 2048]; - let mut archive = zip::write::ZipWriter::new(Cursor::new(&mut buf)); + let mut archive = zip_next::write::ZipWriter::new(Cursor::new(&mut buf)); archive .start_file( "name", - zip::write::FileOptions::default().with_deprecated_encryption(b"password"), + zip_next::write::FileOptions::default().with_deprecated_encryption(b"password"), ) .unwrap(); archive.write_all(b"test").unwrap(); archive.finish().unwrap(); drop(archive); - let mut archive = zip::ZipArchive::new(Cursor::new(&mut buf)).unwrap(); + let mut archive = zip_next::ZipArchive::new(Cursor::new(&mut buf)).unwrap(); let mut file = archive.by_index_decrypt(0, b"password").unwrap().unwrap(); let mut buf = Vec::new(); file.read_to_end(&mut buf).unwrap(); From b179709639527e91531c0c6f99ebf8c06ddcfb7f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 8 May 2023 19:07:42 -0700 Subject: [PATCH 132/281] Bug fix: derive Arbitrary for ZipCryptoKeys --- src/zipcrypto.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index 7ad8ddcb..2e97a647 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -9,6 +9,7 @@ use std::hash::{Hash, Hasher}; use std::num::Wrapping; /// A container to hold the current key state +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] #[derive(Clone, Copy, Hash, Ord, PartialOrd, Eq, PartialEq)] pub(crate) struct ZipCryptoKeys { key_0: Wrapping, From 5bb40012d20f8f00846a7edb96b886311b286eac Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 8 May 2023 19:37:55 -0700 Subject: [PATCH 133/281] WIP: Fix upstream merge --- src/read.rs | 2 +- src/write.rs | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/read.rs b/src/read.rs index 75096950..bb2855b6 100644 --- a/src/read.rs +++ b/src/read.rs @@ -57,7 +57,7 @@ pub(crate) mod zip_archive { /// for i in 0..zip.len() { /// let mut file = zip.by_index(i)?; /// println!("Filename: {}", file.name()); - /// std::io::copy(&mut file, &mut std::io::stdout()); + /// std::io::copy(&mut file, &mut std::io::stdout())?; /// } /// /// Ok(()) diff --git a/src/write.rs b/src/write.rs index 4c2fea55..a5f47083 100644 --- a/src/write.rs +++ b/src/write.rs @@ -475,12 +475,8 @@ impl ZipWriter { self.stats.hasher = Hasher::new(); } if let Some(keys) = options.encrypt_with { - let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { - writer: mem::replace(&mut self.inner, Closed).unwrap(), - buffer: vec![], - keys, - }; - let crypto_header = [0u8; 12]; + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { writer: core::mem::replace(&mut self.inner, GenericZipWriter::Closed).unwrap(), buffer: vec![], keys }; + let mut crypto_header = [0u8; 12]; zipwriter.write_all(&crypto_header)?; self.inner = Storer(MaybeEncrypted::Encrypted(zipwriter)); @@ -509,9 +505,15 @@ impl ZipWriter { // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. self.end_extra_data()?; } - let make_plain_writer = self + let make_plain_writer = match self .inner - .prepare_next_writer(CompressionMethod::Stored, None)?; + .prepare_next_writer(CompressionMethod::Stored, None)? { + MaybeEncrypted::Encrypted(writer) => { + let crc32 = self.stats.hasher.clone().finalize(); + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) + }, + MaybeEncrypted::Unencrypted(writer) => writer + } self.inner.switch_to(make_plain_writer)?; let writer = self.inner.get_plain(); From 822a734ae4522fee4eaeeebd7a7668eb49c3e1db Mon Sep 17 00:00:00 2001 From: Nick Babcock Date: Sat, 6 May 2023 07:33:59 -0500 Subject: [PATCH 134/281] Support zstd in Wasm by disabling default features Use case: creating zstd zips in Wasm Currently it is not possible to create zstd zips in Wasm due to the default `zdict_builder` needing additional code that will cause compilation to fail for wasm32-unknown-unknown: https://github.com/gyscos/zstd-rs/issues/210 Since the zip crate's zstd implementation does not use anything from the `zdict_builder` feature, I've disabled it. I've gone a step further and disabled the other features as well (`arrays` and `legacy`), as they are either unused (`arrays`) or a legacy format from 7 years ago that predates when the zip format supported zstd. So far this patch has been working well for the Wasm application, and figured I should upstream it. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 83df6493..989bb20b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ hmac = { version = "0.12.1", optional = true, features = ["reset"] } pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.20", optional = true, default-features = false, features = ["std"] } -zstd = { version = "0.12.3", optional = true } +zstd = { version = "0.12.3", optional = true, default-features = false } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" From 9efec6b61fbfedd4109750f4699a3349ccb4f3bf Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 12:53:30 -0700 Subject: [PATCH 135/281] Fix merge --- src/write.rs | 20 ++++++++------------ src/zipcrypto.rs | 2 ++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/write.rs b/src/write.rs index a5f47083..de5d9e23 100644 --- a/src/write.rs +++ b/src/write.rs @@ -475,8 +475,8 @@ impl ZipWriter { self.stats.hasher = Hasher::new(); } if let Some(keys) = options.encrypt_with { - let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { writer: core::mem::replace(&mut self.inner, GenericZipWriter::Closed).unwrap(), buffer: vec![], keys }; - let mut crypto_header = [0u8; 12]; + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { writer: mem::replace(&mut self.inner, Closed).unwrap(), buffer: vec![], keys }; + let crypto_header = [0u8; 12]; zipwriter.write_all(&crypto_header)?; self.inner = Storer(MaybeEncrypted::Encrypted(zipwriter)); @@ -505,15 +505,9 @@ impl ZipWriter { // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. self.end_extra_data()?; } - let make_plain_writer = match self + let make_plain_writer = self .inner - .prepare_next_writer(CompressionMethod::Stored, None)? { - MaybeEncrypted::Encrypted(writer) => { - let crc32 = self.stats.hasher.clone().finalize(); - self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) - }, - MaybeEncrypted::Unencrypted(writer) => writer - } + .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; let writer = self.inner.get_plain(); @@ -1036,12 +1030,14 @@ impl Drop for ZipWriter { } } +type SwitchWriterFunction = Box) -> GenericZipWriter>; + impl GenericZipWriter { fn prepare_next_writer( &self, compression: CompressionMethod, compression_level: Option, - ) -> ZipResult) -> GenericZipWriter>> { + ) -> ZipResult> { if let Closed = self { return Err( io::Error::new(io::ErrorKind::BrokenPipe, "ZipWriter was already closed").into(), @@ -1121,7 +1117,7 @@ impl GenericZipWriter { fn switch_to( &mut self, - make_new_self: Box) -> GenericZipWriter>, + make_new_self: SwitchWriterFunction, ) -> ZipResult<()> { let bare = match mem::replace(self, Closed) { Storer(w) => w, diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index 2e97a647..71af39e3 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -135,12 +135,14 @@ impl ZipCryptoReader { Ok(Some(ZipCryptoReaderValid { reader: self })) } } +#[allow(unused)] pub(crate) struct ZipCryptoWriter { pub(crate) writer: W, pub(crate) buffer: Vec, pub(crate) keys: ZipCryptoKeys, } impl ZipCryptoWriter { + #[allow(unused)] pub(crate) fn finish(mut self, crc32: u32) -> std::io::Result { self.buffer[11] = (crc32 >> 24) as u8; for byte in self.buffer.iter_mut() { From 82cd28af3c2dec6ce59242ece8d3cf8f03833a2e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 12:55:20 -0700 Subject: [PATCH 136/281] Reformat --- src/write.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/write.rs b/src/write.rs index de5d9e23..5652933c 100644 --- a/src/write.rs +++ b/src/write.rs @@ -475,7 +475,11 @@ impl ZipWriter { self.stats.hasher = Hasher::new(); } if let Some(keys) = options.encrypt_with { - let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { writer: mem::replace(&mut self.inner, Closed).unwrap(), buffer: vec![], keys }; + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + buffer: vec![], + keys, + }; let crypto_header = [0u8; 12]; zipwriter.write_all(&crypto_header)?; @@ -1115,10 +1119,7 @@ impl GenericZipWriter { } } - fn switch_to( - &mut self, - make_new_self: SwitchWriterFunction, - ) -> ZipResult<()> { + fn switch_to(&mut self, make_new_self: SwitchWriterFunction) -> ZipResult<()> { let bare = match mem::replace(self, Closed) { Storer(w) => w, #[cfg(any( From 7c39dbd4c2f332420f6661e66d67e941121b9938 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 13:01:01 -0700 Subject: [PATCH 137/281] Bug fix --- CHANGELOG.md | 2 +- src/write.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da190dc7..be5fc267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,7 +114,7 @@ ## [0.7.4] -### Changed +### Merged from upstream - Added experimental [`zip_next::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files with PKWARE encryption. diff --git a/src/write.rs b/src/write.rs index 5652933c..6a41d337 100644 --- a/src/write.rs +++ b/src/write.rs @@ -513,6 +513,14 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; + match mem::replace(&mut self.inner, Closed) { + Storer(MaybeEncrypted::Encrypted(writer)) => { + let crc32 = self.stats.hasher.clone().finalize(); + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) + } + Storer(w) => self.inner = Storer(w), + _ => unreachable!() + } let writer = self.inner.get_plain(); if !self.writing_raw { From ab302b20b74e71c3b9b559192f5758a0d88dacf2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 13:04:54 -0700 Subject: [PATCH 138/281] Reformat --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 6a41d337..fdfcd8bc 100644 --- a/src/write.rs +++ b/src/write.rs @@ -519,7 +519,7 @@ impl ZipWriter { self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) } Storer(w) => self.inner = Storer(w), - _ => unreachable!() + _ => unreachable!(), } let writer = self.inner.get_plain(); From bb85ef1de1b122174005478316dafa9a00bb4f8c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 13:06:15 -0700 Subject: [PATCH 139/281] Update dependencies --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 576716ad..88e3a3ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ flate2 = { version = "1.0.26", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } -time = { version = "0.3.20", optional = true, default-features = false, features = ["std"] } +time = { version = "0.3.21", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] @@ -35,7 +35,7 @@ arbitrary = { version = "1.3.0", features = ["derive"] } bencher = "0.1.5" getrandom = "0.2.9" walkdir = "2.3.3" -time = { version = "0.3.20", features = ["formatting", "macros"] } +time = { version = "0.3.21", features = ["formatting", "macros"] } [features] aes-crypto = [ "aes", "constant_time_eq", "hmac", "pbkdf2", "sha1" ] From 3f8e7ec12cb857f7634e050829967888b8ec8a41 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 13:06:55 -0700 Subject: [PATCH 140/281] Bump version to 0.7.4 --- Cargo.toml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 88e3a3ce..f8068edf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.3" +version = "0.7.4" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 2eeeafa3..a52116eb 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.3" +zip_next = "0.7.4" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.3", default-features = false } +zip_next = { version = "0.7.4", default-features = false } ``` The features available are: From 6be75a9f0400696b67f0d31f6e578fa1ddac8503 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 14:22:57 -0700 Subject: [PATCH 141/281] Enable parallel fuzzing --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 01c912c1..03383564 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,7 @@ jobs: cargo fuzz build fuzz_read - name: run fuzz run: | - cargo fuzz run fuzz_read -- -timeout=1s -runs=10000000 + cargo fuzz run fuzz_read -- -timeout=1s -jobs=10 -workers=2 -runs=1000000 -max_len=5000000000 fuzz_write: runs-on: ubuntu-latest @@ -109,4 +109,4 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -runs=3000000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=1s -jobs=30 -workers=2 -runs=100000 -max_len=5000000000 From 3af7f187d737b8add5fa57773d28a9172b07b35b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 14:54:21 -0700 Subject: [PATCH 142/281] Strengthen fuzz_write: can now close and reopen before copying --- fuzz/fuzz_targets/fuzz_write.rs | 49 +++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index eb700cf9..ff495ffa 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -1,5 +1,6 @@ #![no_main] +use std::cell::RefCell; use libfuzzer_sys::fuzz_target; use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; @@ -14,15 +15,18 @@ pub struct File { pub enum FileOperation { Write { file: File, - options: zip_next::write::FileOptions + options: zip_next::write::FileOptions, + reopen: bool, }, ShallowCopy { base: Box, - new_name: String + new_name: String, + reopen: bool, }, DeepCopy { base: Box, - new_name: String + new_name: String, + reopen: bool, } } @@ -34,40 +38,49 @@ impl FileOperation { FileOperation::DeepCopy {new_name, ..} => new_name }.to_owned() } + + pub fn should_reopen(&self) -> bool { + match self { + FileOperation::Write {reopen, ..} => *reopen, + FileOperation::ShallowCopy {reopen, ..} => *reopen, + FileOperation::DeepCopy {reopen, ..} => *reopen + } + } } -fn do_operation(writer: &mut zip_next::ZipWriter, +fn do_operation(writer: &mut RefCell>, operation: &FileOperation) -> Result<(), Box> where T: Read + Write + Seek { match operation { - FileOperation::Write {file, mut options} => { + FileOperation::Write {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } - writer.start_file(file.name.to_owned(), options)?; + writer.borrow_mut().start_file(file.name.to_owned(), options)?; for chunk in &file.contents { - writer.write_all(chunk.as_slice())?; + writer.borrow_mut().write_all(chunk.as_slice())?; } } - FileOperation::ShallowCopy {base, new_name} => { + FileOperation::ShallowCopy {base, new_name, .. } => { do_operation(writer, base)?; - writer.shallow_copy_file(&base.get_name(), new_name)?; + writer.borrow_mut().shallow_copy_file(&base.get_name(), new_name)?; } - FileOperation::DeepCopy {base, new_name} => { + FileOperation::DeepCopy {base, new_name, .. } => { do_operation(writer, base)?; - writer.deep_copy_file(&base.get_name(), new_name)?; + writer.borrow_mut().deep_copy_file(&base.get_name(), new_name)?; } } + if operation.should_reopen() { + let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); + *writer = new_writer.into(); + } Ok(()) } -fuzz_target!(|data: Vec<(FileOperation, bool)>| { - let mut writer = zip_next::ZipWriter::new(Cursor::new(Vec::new())); - for (operation, close_and_reopen) in data { +fuzz_target!(|data: Vec| { + let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()))); + for operation in data { let _ = do_operation(&mut writer, &operation); - if close_and_reopen { - writer = zip_next::ZipWriter::new_append(writer.finish().unwrap()).unwrap(); - } } - let _ = zip_next::ZipArchive::new(writer.finish().unwrap()); + let _ = zip_next::ZipArchive::new(writer.borrow_mut().finish().unwrap()); }); \ No newline at end of file From 9b0e620c501ed5c590b233c4967f1f2d67e474d4 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 16:30:59 -0700 Subject: [PATCH 143/281] Dump full encryption key while testing or fuzzing --- src/zipcrypto.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index 71af39e3..a61dd354 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -18,10 +18,17 @@ pub(crate) struct ZipCryptoKeys { } impl Debug for ZipCryptoKeys { + #[allow(unreachable_code)] fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut t = DefaultHasher::new(); - self.hash(&mut t); - f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())) + #[cfg(not(any(test,fuzzing)))] + { + let mut t = DefaultHasher::new(); + self.hash(&mut t); + return f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())); + } + #[cfg(any(test,fuzzing))] + return f.write_fmt(format_args!("ZipCryptoKeys({:#10x},{:#10x},{:#10x})", + self.key_0, self.key_1, self.key_2)); } } From f17012ca63a72a6b6d98589e47e8a9b54c768610 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 16:32:12 -0700 Subject: [PATCH 144/281] Bug fix: scope Hasher and DefaultHasher imports to the configs that use them --- src/zipcrypto.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index a61dd354..ae53f86e 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -3,9 +3,8 @@ //! The following paper was used to implement the ZipCrypto algorithm: //! [https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf](https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf) -use std::collections::hash_map::DefaultHasher; use std::fmt::{Debug, Formatter}; -use std::hash::{Hash, Hasher}; +use std::hash::{Hash}; use std::num::Wrapping; /// A container to hold the current key state @@ -22,6 +21,8 @@ impl Debug for ZipCryptoKeys { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { #[cfg(not(any(test,fuzzing)))] { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; let mut t = DefaultHasher::new(); self.hash(&mut t); return f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())); From cfa91e75b0b628692b73e1f670e43d65dd135b8d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 16:34:57 -0700 Subject: [PATCH 145/281] Reformat --- src/zipcrypto.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index ae53f86e..7826ac2d 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -4,7 +4,7 @@ //! [https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf](https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf) use std::fmt::{Debug, Formatter}; -use std::hash::{Hash}; +use std::hash::Hash; use std::num::Wrapping; /// A container to hold the current key state @@ -19,7 +19,7 @@ pub(crate) struct ZipCryptoKeys { impl Debug for ZipCryptoKeys { #[allow(unreachable_code)] fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - #[cfg(not(any(test,fuzzing)))] + #[cfg(not(any(test, fuzzing)))] { use std::collections::hash_map::DefaultHasher; use std::hash::Hasher; @@ -27,9 +27,11 @@ impl Debug for ZipCryptoKeys { self.hash(&mut t); return f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())); } - #[cfg(any(test,fuzzing))] - return f.write_fmt(format_args!("ZipCryptoKeys({:#10x},{:#10x},{:#10x})", - self.key_0, self.key_1, self.key_2)); + #[cfg(any(test, fuzzing))] + return f.write_fmt(format_args!( + "ZipCryptoKeys({:#10x},{:#10x},{:#10x})", + self.key_0, self.key_1, self.key_2 + )); } } From 5e9ccfb27d0e3278d57b6ee8f78d0b028f9b3260 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 19:34:15 -0700 Subject: [PATCH 146/281] Upload failed fuzz inputs if any --- .github/workflows/ci.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 03383564..82fbc609 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -91,6 +91,13 @@ jobs: - name: run fuzz run: | cargo fuzz run fuzz_read -- -timeout=1s -jobs=10 -workers=2 -runs=1000000 -max_len=5000000000 + - name: Upload any failure inputs + if-no-files-found: ignore + if: always() + uses: actions/upload-artifact@v3 + with: + name: fuzz_read_bad_inputs + path: fuzz/artifacts/fuzz_read/crash-* fuzz_write: runs-on: ubuntu-latest @@ -110,3 +117,10 @@ jobs: - name: run fuzz run: | cargo fuzz run fuzz_write -- -timeout=1s -jobs=30 -workers=2 -runs=100000 -max_len=5000000000 + - name: Upload any failure inputs + if-no-files-found: ignore + if: always() + uses: actions/upload-artifact@v3 + with: + name: fuzz_write_bad_inputs + path: fuzz/artifacts/fuzz_write/crash-* From bfe274a779576901e4ef5c8fe7a1e33433625e20 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 19:36:04 -0700 Subject: [PATCH 147/281] Bug fix --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 82fbc609..da4d4cbd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -92,12 +92,12 @@ jobs: run: | cargo fuzz run fuzz_read -- -timeout=1s -jobs=10 -workers=2 -runs=1000000 -max_len=5000000000 - name: Upload any failure inputs - if-no-files-found: ignore if: always() uses: actions/upload-artifact@v3 with: name: fuzz_read_bad_inputs path: fuzz/artifacts/fuzz_read/crash-* + if-no-files-found: ignore fuzz_write: runs-on: ubuntu-latest @@ -118,9 +118,9 @@ jobs: run: | cargo fuzz run fuzz_write -- -timeout=1s -jobs=30 -workers=2 -runs=100000 -max_len=5000000000 - name: Upload any failure inputs - if-no-files-found: ignore if: always() uses: actions/upload-artifact@v3 with: name: fuzz_write_bad_inputs path: fuzz/artifacts/fuzz_write/crash-* + if-no-files-found: ignore \ No newline at end of file From 61502b22a74f0e7ca53918c62ad42a08fbaa5dbd Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 10 May 2023 19:45:13 -0700 Subject: [PATCH 148/281] Increase fuzz_write jobs to reproduce a recent failure --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index da4d4cbd..56e98d0b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,7 +116,7 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -jobs=30 -workers=2 -runs=100000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=1s -jobs=100 -workers=2 -runs=100000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 From 8c6816fb332f0c0228ef52fadc6187eb480c3fba Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 09:33:06 -0700 Subject: [PATCH 149/281] Bug fixes: file can't be ZIP64 if CDR start is after CDR end --- src/read.rs | 3 +++ src/spec.rs | 12 +++++------ src/write.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/src/read.rs b/src/read.rs index bb2855b6..f172a291 100644 --- a/src/read.rs +++ b/src/read.rs @@ -351,6 +351,9 @@ impl ZipArchive { .ok_or(ZipError::InvalidArchive( "Invalid central directory size or offset", ))?; + if directory_start > search_upper_bound { + return Err(ZipError::InvalidArchive("Invalid central directory size or offset")); + } if footer64.disk_number != footer64.disk_with_central_directory { return unsupported_zip_error("Support for multi-disk files is not implemented"); diff --git a/src/spec.rs b/src/spec.rs index 1d8cb0a6..403b1eeb 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -61,14 +61,14 @@ impl CentralDirectoryEnd { }) } - pub fn find_and_parse( + pub fn find_and_parse( reader: &mut T, ) -> ZipResult<(CentralDirectoryEnd, u64)> { const HEADER_SIZE: u64 = 22; const BYTES_BETWEEN_MAGIC_AND_COMMENT_SIZE: u64 = HEADER_SIZE - 6; let file_length = reader.seek(io::SeekFrom::End(0))?; - let search_upper_bound = file_length.saturating_sub(HEADER_SIZE + ::std::u16::MAX as u64); + let search_upper_bound = file_length.saturating_sub(HEADER_SIZE + u16::MAX as u64); if file_length < HEADER_SIZE { return Err(ZipError::InvalidArchive("Invalid zip header")); @@ -155,14 +155,14 @@ pub struct Zip64CentralDirectoryEnd { } impl Zip64CentralDirectoryEnd { - pub fn find_and_parse( + pub fn find_and_parse( reader: &mut T, nominal_offset: u64, search_upper_bound: u64, ) -> ZipResult<(Zip64CentralDirectoryEnd, u64)> { - let mut pos = nominal_offset; + let mut pos = search_upper_bound; - while pos <= search_upper_bound { + while pos >= nominal_offset { reader.seek(io::SeekFrom::Start(pos))?; if reader.read_u32::()? == ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE { @@ -195,7 +195,7 @@ impl Zip64CentralDirectoryEnd { )); } - pos += 1; + pos -= 1; } Err(ZipError::InvalidArchive( diff --git a/src/write.rs b/src/write.rs index fdfcd8bc..679864ea 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1481,6 +1481,7 @@ mod test { use crate::ZipArchive; use std::io; use std::io::{Read, Write}; + use zstd::zstd_safe::WriteBuf; #[test] fn write_empty_zip() { @@ -1732,6 +1733,62 @@ mod test { .expect_err("Expected duplicate filename not to be allowed"); } + #[test] + fn test_filename_looks_like_zip64_locator() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2a() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_3() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer.start_file("\0PK\u{6}\u{6}", FileOptions::default()).unwrap(); + writer + .start_file( + "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } + #[test] fn path_to_string() { let mut path = std::path::PathBuf::new(); From 2ad4c206811ebf78d21781fffd5dae020620ca69 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 09:35:12 -0700 Subject: [PATCH 150/281] Reformat and fix Clippy warning --- src/read.rs | 4 +++- src/spec.rs | 4 +--- src/write.rs | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/read.rs b/src/read.rs index f172a291..a4f74d85 100644 --- a/src/read.rs +++ b/src/read.rs @@ -352,7 +352,9 @@ impl ZipArchive { "Invalid central directory size or offset", ))?; if directory_start > search_upper_bound { - return Err(ZipError::InvalidArchive("Invalid central directory size or offset")); + return Err(ZipError::InvalidArchive( + "Invalid central directory size or offset", + )); } if footer64.disk_number != footer64.disk_with_central_directory { diff --git a/src/spec.rs b/src/spec.rs index 403b1eeb..2b133383 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -61,9 +61,7 @@ impl CentralDirectoryEnd { }) } - pub fn find_and_parse( - reader: &mut T, - ) -> ZipResult<(CentralDirectoryEnd, u64)> { + pub fn find_and_parse(reader: &mut T) -> ZipResult<(CentralDirectoryEnd, u64)> { const HEADER_SIZE: u64 = 22; const BYTES_BETWEEN_MAGIC_AND_COMMENT_SIZE: u64 = HEADER_SIZE - 6; let file_length = reader.seek(io::SeekFrom::End(0))?; diff --git a/src/write.rs b/src/write.rs index 679864ea..4d2b1eec 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1481,7 +1481,6 @@ mod test { use crate::ZipArchive; use std::io; use std::io::{Read, Write}; - use zstd::zstd_safe::WriteBuf; #[test] fn write_empty_zip() { @@ -1777,7 +1776,9 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_3() { let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer.start_file("\0PK\u{6}\u{6}", FileOptions::default()).unwrap(); + writer + .start_file("\0PK\u{6}\u{6}", FileOptions::default()) + .unwrap(); writer .start_file( "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", From ad26a3f7f970bc66fee3d51ad4b0dbf40b0f0c82 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 09:39:18 -0700 Subject: [PATCH 151/281] Bug fix --- src/spec.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/spec.rs b/src/spec.rs index 2b133383..f1b5d6cc 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -192,8 +192,11 @@ impl Zip64CentralDirectoryEnd { archive_offset, )); } - - pos -= 1; + if pos > 0 { + pos -= 1; + } else { + break; + } } Err(ZipError::InvalidArchive( From 06b89f93eaa4985e15e952e2acec3cfb41fa7622 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 09:59:56 -0700 Subject: [PATCH 152/281] Fix a failing unit test --- src/read.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/read.rs b/src/read.rs index a4f74d85..6bb5addc 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1257,7 +1257,7 @@ mod test { "../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" )); let reader = ZipArchive::new(io::Cursor::new(v)); - assert!(reader.is_err()); + assert!(reader.is_err() || reader.unwrap().file_names().next().is_none()); } /// test case to ensure we don't preemptively over allocate based on the From ebb4e01329418f592f5eb2372913ea0e6b17022b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 10:13:09 -0700 Subject: [PATCH 153/281] Slightly stronger assertion --- src/read.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/read.rs b/src/read.rs index 6bb5addc..a47586ea 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1257,7 +1257,7 @@ mod test { "../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" )); let reader = ZipArchive::new(io::Cursor::new(v)); - assert!(reader.is_err() || reader.unwrap().file_names().next().is_none()); + assert!(reader.is_err() || reader.unwrap().is_empty()); } /// test case to ensure we don't preemptively over allocate based on the From dc351196e2a13f1e1463fefc8ae06f08337ca957 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 18:52:41 -0700 Subject: [PATCH 154/281] Bug fix: don't allow writing files with certain ZIP64 magic strings in their names --- src/read.rs | 38 +++++++-- src/spec.rs | 4 +- src/write.rs | 100 ++++++----------------- tests/data/zip64_magic_in_filename_1.zip | Bin 0 -> 81 bytes tests/data/zip64_magic_in_filename_2.zip | Bin 0 -> 81 bytes tests/data/zip64_magic_in_filename_3.zip | Bin 0 -> 81 bytes 6 files changed, 56 insertions(+), 86 deletions(-) create mode 100644 tests/data/zip64_magic_in_filename_1.zip create mode 100644 tests/data/zip64_magic_in_filename_2.zip create mode 100644 tests/data/zip64_magic_in_filename_3.zip diff --git a/src/read.rs b/src/read.rs index a47586ea..e7a1b192 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1123,6 +1123,10 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult, } } +use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; pub use zip_writer::ZipWriter; @@ -300,7 +301,7 @@ impl ZipWriter { ZipArchive::get_directory_counts(&mut readwriter, &footer, cde_start_pos)?; if readwriter.seek(SeekFrom::Start(directory_start)).is_err() { - return Err(ZipError::InvalidArchive( + return Err(InvalidArchive( "Could not seek to start of central directory", )); } @@ -438,6 +439,7 @@ impl ZipWriter { { let header_start = self.inner.get_plain().stream_position()?; let name = name.into(); + Self::validate_name(&name)?; let permissions = options.permissions.unwrap_or(0o100644); let file = ZipFileData { @@ -491,7 +493,7 @@ impl ZipWriter { fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { let name = &file.file_name; if self.files_by_name.contains_key(name) { - return Err(ZipError::InvalidArchive("Duplicate filename")); + return Err(InvalidArchive("Duplicate filename")); } let name = name.to_owned(); self.files.push(file); @@ -1030,6 +1032,27 @@ impl ZipWriter { self.insert_file_data(dest_data)?; Ok(()) } + fn validate_name(name: &String) -> ZipResult<()> { + for (index, _) in name.match_indices("PK") { + if name.len() >= index + 4 { + let magic_number = name[index..index + 4] + .as_bytes() + .read_u32::()?; + match magic_number { + spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { + return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); + } + spec::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE => { + return Err(InvalidArchive( + "Filename can't contain ZIP64 end-locator signature", + )); + } + _ => {} + } + } + } + Ok(()) + } } impl Drop for ZipWriter { @@ -1731,79 +1754,6 @@ mod test { .start_file("foo/bar/test", FileOptions::default()) .expect_err("Expected duplicate filename not to be allowed"); } - - #[test] - fn test_filename_looks_like_zip64_locator() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer - .start_file( - "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", - FileOptions::default(), - ) - .unwrap(); - let zip = writer.finish().unwrap(); - let _ = ZipArchive::new(zip).unwrap(); - } - - #[test] - fn test_filename_looks_like_zip64_locator_2() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer - .start_file( - "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", - FileOptions::default(), - ) - .unwrap(); - let zip = writer.finish().unwrap(); - println!("{:02x?}", zip.get_ref()); - let _ = ZipArchive::new(zip).unwrap(); - } - - #[test] - fn test_filename_looks_like_zip64_locator_2a() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer - .start_file( - "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", - FileOptions::default(), - ) - .unwrap(); - let zip = writer.finish().unwrap(); - println!("{:02x?}", zip.get_ref()); - let _ = ZipArchive::new(zip).unwrap(); - } - - #[test] - fn test_filename_looks_like_zip64_locator_3() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer - .start_file("\0PK\u{6}\u{6}", FileOptions::default()) - .unwrap(); - writer - .start_file( - "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", - FileOptions::default(), - ) - .unwrap(); - let zip = writer.finish().unwrap(); - println!("{:02x?}", zip.get_ref()); - let _ = ZipArchive::new(zip).unwrap(); - } - - #[test] - fn path_to_string() { - let mut path = std::path::PathBuf::new(); - #[cfg(windows)] - path.push(r"C:\"); - #[cfg(unix)] - path.push("/"); - path.push("windows"); - path.push(".."); - path.push("."); - path.push("system32"); - let path_str = super::path_to_string(&path); - assert_eq!(path_str, "windows/system32"); - } } #[cfg(not(feature = "unreserved"))] diff --git a/tests/data/zip64_magic_in_filename_1.zip b/tests/data/zip64_magic_in_filename_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..36e2bd49c894771223b18a3ea5ee3b4018fde0da GIT binary patch literal 81 McmZQzpf2zR005i-OaK4? literal 0 HcmV?d00001 diff --git a/tests/data/zip64_magic_in_filename_2.zip b/tests/data/zip64_magic_in_filename_2.zip new file mode 100644 index 0000000000000000000000000000000000000000..36e2bd49c894771223b18a3ea5ee3b4018fde0da GIT binary patch literal 81 McmZQzpf2zR005i-OaK4? literal 0 HcmV?d00001 diff --git a/tests/data/zip64_magic_in_filename_3.zip b/tests/data/zip64_magic_in_filename_3.zip new file mode 100644 index 0000000000000000000000000000000000000000..36e2bd49c894771223b18a3ea5ee3b4018fde0da GIT binary patch literal 81 McmZQzpf2zR005i-OaK4? literal 0 HcmV?d00001 From bf867c50126602f4d53ecf96ed641b8f32fbdafd Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 19:25:32 -0700 Subject: [PATCH 155/281] Bug fix: skip invalid filenames during write fuzz --- Cargo.toml | 1 + fuzz/fuzz_targets/fuzz_write.rs | 3 +++ src/write.rs | 39 ++++++++++++++++++--------------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f8068edf..71717485 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.21", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } +visibility = "0.0.1" [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index ff495ffa..aa87278d 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -51,6 +51,9 @@ impl FileOperation { fn do_operation(writer: &mut RefCell>, operation: &FileOperation) -> Result<(), Box> where T: Read + Write + Seek { + if zip_next::write::validate_name(&operation.get_name()).is_err() { + return Ok(()); + } match operation { FileOperation::Write {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { diff --git a/src/write.rs b/src/write.rs index 622b39d2..7dfef832 100644 --- a/src/write.rs +++ b/src/write.rs @@ -439,7 +439,7 @@ impl ZipWriter { { let header_start = self.inner.get_plain().stream_position()?; let name = name.into(); - Self::validate_name(&name)?; + validate_name(&name)?; let permissions = options.permissions.unwrap_or(0o100644); let file = ZipFileData { @@ -1032,27 +1032,30 @@ impl ZipWriter { self.insert_file_data(dest_data)?; Ok(()) } - fn validate_name(name: &String) -> ZipResult<()> { - for (index, _) in name.match_indices("PK") { - if name.len() >= index + 4 { - let magic_number = name[index..index + 4] - .as_bytes() - .read_u32::()?; - match magic_number { - spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { - return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); - } - spec::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE => { - return Err(InvalidArchive( - "Filename can't contain ZIP64 end-locator signature", - )); - } - _ => {} +} + +#[cfg_attr(fuzzing, visibility::make(pub))] +#[cfg_attr(fuzzing, allow(missing_docs))] +pub(crate) fn validate_name(name: &String) -> ZipResult<()> { + for (index, _) in name.match_indices("PK") { + if name.len() >= index + 4 { + let magic_number = name[index..index + 4] + .as_bytes() + .read_u32::()?; + match magic_number { + spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { + return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); } + spec::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE => { + return Err(InvalidArchive( + "Filename can't contain ZIP64 end-locator signature", + )); + } + _ => {} } } - Ok(()) } + Ok(()) } impl Drop for ZipWriter { From 3c44f2812c682658308c987995f334da923d3291 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 19:28:54 -0700 Subject: [PATCH 156/281] Remove unused imports --- src/read.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/read.rs b/src/read.rs index e7a1b192..ee763157 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1124,13 +1124,11 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult Date: Thu, 11 May 2023 20:11:44 -0700 Subject: [PATCH 157/281] Bug fix: test files were truncated --- tests/data/zip64_magic_in_filename_1.zip | Bin 81 -> 168 bytes tests/data/zip64_magic_in_filename_2.zip | Bin 81 -> 148 bytes tests/data/zip64_magic_in_filename_3.zip | Bin 81 -> 236 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/data/zip64_magic_in_filename_1.zip b/tests/data/zip64_magic_in_filename_1.zip index 36e2bd49c894771223b18a3ea5ee3b4018fde0da..18b9494a5f12463aec546600cefa6ea115c868db 100644 GIT binary patch literal 168 zcmWIWW@Zs#U|`^2cq6eU49H+$0uex<1jGT}Y;0f-m;mzF5j+Ts8K{PlNsk$~88977 Z8bK_=CI)!3g6(8vU}OjY>Tm)Z001lZ3jP29 literal 81 McmZQzpf2zR005i-OaK4? diff --git a/tests/data/zip64_magic_in_filename_2.zip b/tests/data/zip64_magic_in_filename_2.zip index 36e2bd49c894771223b18a3ea5ee3b4018fde0da..5628acd413b46291b462f4554ee1e0ffe44ed0c3 100644 GIT binary patch literal 148 zcmWIWW@Zs#U|`^2cq6eU49H+$0uewU0mK2`Y-~Wx4i-Qnn1M2kOnS_?b;1-cX#}xw VnG@j6$_A2W1VT3;Z2{si005;-3f2Gs literal 81 McmZQzpf2zR005i-OaK4? diff --git a/tests/data/zip64_magic_in_filename_3.zip b/tests/data/zip64_magic_in_filename_3.zip index 36e2bd49c894771223b18a3ea5ee3b4018fde0da..4398604cf919e27c55f37fbf377562aae543ebc5 100644 GIT binary patch literal 236 zcmWIWW@Zs#U|`^2cq6eU49H+$0ueyK3L*l$+1QwYBrZh~AVn--b?hJ}5?}`DV`S1} p#$!BG`I1I317sOKEnv&wT2w(S999Q-v$BDtnSihoNXLRW3;;m?5>5aB literal 81 McmZQzpf2zR005i-OaK4? From 332d09f51a6acf004125512205138b36deaec4c9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 20:14:26 -0700 Subject: [PATCH 158/281] Bug fix --- src/read.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/read.rs b/src/read.rs index ee763157..4b849bb7 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1244,16 +1244,14 @@ mod test { #[test] fn zip64_magic_in_filenames() { let files = vec![ - include_bytes!("../tests/data/zip64_magic_in_filename_1.zip"), - include_bytes!("../tests/data/zip64_magic_in_filename_2.zip"), - include_bytes!("../tests/data/zip64_magic_in_filename_3.zip"), + include_bytes!("../tests/data/zip64_magic_in_filename_1.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec() ]; // Although we don't allow adding files whose names contain the ZIP64 CDB-end or // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. for file in files { - let mut v = Vec::new(); - v.extend_from_slice(file); - ZipArchive::new(Cursor::new(v)).unwrap(); + ZipArchive::new(Cursor::new(file)).unwrap(); } } From 5cd0b75209ad46510e4b70f748c8073b9be9b521 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 20:17:01 -0700 Subject: [PATCH 159/281] Reformat --- src/read.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/read.rs b/src/read.rs index 4b849bb7..02156ca8 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1246,7 +1246,7 @@ mod test { let files = vec![ include_bytes!("../tests/data/zip64_magic_in_filename_1.zip").to_vec(), include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), - include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec() + include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec(), ]; // Although we don't allow adding files whose names contain the ZIP64 CDB-end or // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. From 609865ba75c23b305766c8f6a6af252e425f848b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 20:35:01 -0700 Subject: [PATCH 160/281] Increase read fuzzing 10x since write fuzzing is the bottleneck --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56e98d0b..b2fa83ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,7 @@ jobs: cargo fuzz build fuzz_read - name: run fuzz run: | - cargo fuzz run fuzz_read -- -timeout=1s -jobs=10 -workers=2 -runs=1000000 -max_len=5000000000 + cargo fuzz run fuzz_read -- -timeout=1s -jobs=100 -workers=2 -runs=1000000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 From 2e257141e62ece421f0149002ee449b789c514e4 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 20:35:44 -0700 Subject: [PATCH 161/281] Enable js feature (needed for wasm32-unknown-unknown target) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 71717485..1a1b72bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ arbitrary = { version = "1.3.0", features = ["derive"] } [dev-dependencies] bencher = "0.1.5" -getrandom = "0.2.9" +getrandom = { version = "0.2.9", features = ["js"] } walkdir = "2.3.3" time = { version = "0.3.21", features = ["formatting", "macros"] } From 93a5be27ffb1d387899c3f59a8de1462f0d48697 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 20:49:42 -0700 Subject: [PATCH 162/281] Bug fix --- src/write.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index 7dfef832..daf93465 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1037,11 +1037,10 @@ impl ZipWriter { #[cfg_attr(fuzzing, visibility::make(pub))] #[cfg_attr(fuzzing, allow(missing_docs))] pub(crate) fn validate_name(name: &String) -> ZipResult<()> { + let bytes = name.as_bytes(); for (index, _) in name.match_indices("PK") { - if name.len() >= index + 4 { - let magic_number = name[index..index + 4] - .as_bytes() - .read_u32::()?; + if bytes.len() >= index + 4 { + let magic_number = (&bytes[index..]).read_u32::()?; match magic_number { spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); From 5a11cbeaee82199679a35897f5495c8e88f5e870 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 11 May 2023 21:02:51 -0700 Subject: [PATCH 163/281] Bug fix --- src/write.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/write.rs b/src/write.rs index daf93465..83698965 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1038,20 +1038,20 @@ impl ZipWriter { #[cfg_attr(fuzzing, allow(missing_docs))] pub(crate) fn validate_name(name: &String) -> ZipResult<()> { let bytes = name.as_bytes(); - for (index, _) in name.match_indices("PK") { - if bytes.len() >= index + 4 { - let magic_number = (&bytes[index..]).read_u32::()?; - match magic_number { - spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { - return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); - } - spec::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE => { - return Err(InvalidArchive( - "Filename can't contain ZIP64 end-locator signature", - )); - } - _ => {} + let mut current_window = [0u8; 4]; + for window in bytes.windows(4) { + current_window.copy_from_slice(window); + let magic_number = u32::from_le_bytes(current_window); + match magic_number { + spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { + return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); } + spec::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE => { + return Err(InvalidArchive( + "Filename can't contain ZIP64 end-locator signature", + )); + } + _ => {} } } Ok(()) From 70db61c26e7d54999a842e66fb26171d9670864a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 12 May 2023 08:28:30 -0700 Subject: [PATCH 164/281] Perform sanity checks when both ZIP32 and ZIP64 footers are found --- CHANGELOG.md | 7 +++ src/read.rs | 54 +++++++++++++++++------ tests/data/zip64_magic_in_filename_4.zip | Bin 0 -> 250 bytes 3 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 tests/data/zip64_magic_in_filename_4.zip diff --git a/CHANGELOG.md b/CHANGELOG.md index be5fc267..29e60dae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,3 +118,10 @@ - Added experimental [`zip_next::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files with PKWARE encryption. + +## [0.7.5] + +### Fixed + +- Two magic strings are no longer allowed in filenames, because they can make files impossible to read correctly. +- Archives containing these magic files can still be opened in many cases. diff --git a/src/read.rs b/src/read.rs index 02156ca8..6d98c6bb 100644 --- a/src/read.rs +++ b/src/read.rs @@ -316,7 +316,7 @@ impl ZipArchive { reader: &mut R, footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, - ) -> ZipResult<(u64, u64, usize)> { + ) -> ZipResult<(u64, u64, usize, ZipResult<()>)> { // See if there's a ZIP64 footer. The ZIP64 locator if present will // have its signature 20 bytes in front of the standard footer. The // standard footer, in turn, is 22+N bytes large, where N is the @@ -357,20 +357,21 @@ impl ZipArchive { )); } - if footer64.disk_number != footer64.disk_with_central_directory { - return unsupported_zip_error("Support for multi-disk files is not implemented"); - } - - if !footer.record_too_small() + let supported = if footer64.disk_number != footer64.disk_with_central_directory { + unsupported_zip_error("Support for multi-disk files is not implemented") + } else if !footer.record_too_small() && footer.disk_number as u32 != locator64.disk_with_central_directory { - return unsupported_zip_error("Support for multi-disk files is not implemented"); - } + unsupported_zip_error("Support for multi-disk files is not implemented") + } else { + Ok(()) + }; Ok(( archive_offset, directory_start, footer64.number_of_files as usize, + supported )) } @@ -381,12 +382,38 @@ impl ZipArchive { footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, ) -> ZipResult<(u64, u64, usize)> { - // Check if file is valid as ZIP64 first; if not, try it as ZIP32 - match Self::get_directory_counts_zip64(reader, footer, cde_start_pos) { - Ok(result) => Ok(result), - Err(ZipError::UnsupportedArchive(e)) => Err(ZipError::UnsupportedArchive(e)), - Err(_) => Self::get_directory_counts_zip32(footer, cde_start_pos), + // Check if file has a zip64 footer + let (archive_offset_64, directory_start_64, number_of_files_64, + supported_64) = + match Self::get_directory_counts_zip64(reader, footer, cde_start_pos) { + Ok(result) => result, + Err(_) => return Self::get_directory_counts_zip32(footer, cde_start_pos) + }; + // Check if it also has a zip32 footer + let (archive_offset_32, directory_start_32, number_of_files_32) = + match Self::get_directory_counts_zip32(footer, cde_start_pos) { + Ok(result) => result, + Err(_) => { + supported_64?; + return Ok((archive_offset_64, directory_start_64, number_of_files_64)) + } + }; + // It has both, so check if the zip64 footer is valid; if not, assume zip32 + if archive_offset_64 < u32::MAX as u64 && archive_offset_64 != archive_offset_32 + || archive_offset_32 != u32::MAX as u64 { + return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } + if directory_start_64 < u32::MAX as u64 && directory_start_64 != directory_start_32 + || directory_start_32 != u32::MAX as u64 { + return Ok((archive_offset_32, directory_start_32, number_of_files_32)); + } + if number_of_files_64 < u32::MAX as usize && number_of_files_64 != number_of_files_32 + || number_of_files_32 != u32::MAX as usize { + return Ok((archive_offset_32, directory_start_32, number_of_files_32)); + } + // It is, so we assume a zip64 + supported_64?; + return Ok((archive_offset_64, directory_start_64, number_of_files_64)) } /// Read a ZIP archive, collecting the files it contains @@ -1247,6 +1274,7 @@ mod test { include_bytes!("../tests/data/zip64_magic_in_filename_1.zip").to_vec(), include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_4.zip").to_vec(), ]; // Although we don't allow adding files whose names contain the ZIP64 CDB-end or // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. diff --git a/tests/data/zip64_magic_in_filename_4.zip b/tests/data/zip64_magic_in_filename_4.zip new file mode 100644 index 0000000000000000000000000000000000000000..9726a982341f792f2dc41bbc137d6b4e7e2fccd2 GIT binary patch literal 250 zcmWIWW@Zs#U|`^2IL5Fh49H+$0uewU0K@^_Y;0f-lwbym;8G7V1*o1KSv`mi(! Date: Fri, 12 May 2023 08:36:33 -0700 Subject: [PATCH 165/281] Test allowing ZIP64 magic with the sanity checks --- CHANGELOG.md | 7 ------ Cargo.toml | 1 - fuzz/fuzz_targets/fuzz_write.rs | 3 --- src/read.rs | 41 +++++++++++++++++---------------- src/write.rs | 24 ------------------- 5 files changed, 21 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29e60dae..be5fc267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,10 +118,3 @@ - Added experimental [`zip_next::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files with PKWARE encryption. - -## [0.7.5] - -### Fixed - -- Two magic strings are no longer allowed in filenames, because they can make files impossible to read correctly. -- Archives containing these magic files can still be opened in many cases. diff --git a/Cargo.toml b/Cargo.toml index 1a1b72bf..2c579db7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,6 @@ pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.21", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } -visibility = "0.0.1" [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index aa87278d..ff495ffa 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -51,9 +51,6 @@ impl FileOperation { fn do_operation(writer: &mut RefCell>, operation: &FileOperation) -> Result<(), Box> where T: Read + Write + Seek { - if zip_next::write::validate_name(&operation.get_name()).is_err() { - return Ok(()); - } match operation { FileOperation::Write {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { diff --git a/src/read.rs b/src/read.rs index 6d98c6bb..d36e853c 100644 --- a/src/read.rs +++ b/src/read.rs @@ -357,10 +357,9 @@ impl ZipArchive { )); } - let supported = if footer64.disk_number != footer64.disk_with_central_directory { - unsupported_zip_error("Support for multi-disk files is not implemented") - } else if !footer.record_too_small() - && footer.disk_number as u32 != locator64.disk_with_central_directory + let supported = if (footer64.disk_number != footer64.disk_with_central_directory) + || (!footer.record_too_small() + && footer.disk_number as u32 != locator64.disk_with_central_directory) { unsupported_zip_error("Support for multi-disk files is not implemented") } else { @@ -371,7 +370,7 @@ impl ZipArchive { archive_offset, directory_start, footer64.number_of_files as usize, - supported + supported, )) } @@ -383,37 +382,39 @@ impl ZipArchive { cde_start_pos: u64, ) -> ZipResult<(u64, u64, usize)> { // Check if file has a zip64 footer - let (archive_offset_64, directory_start_64, number_of_files_64, - supported_64) = + let (archive_offset_64, directory_start_64, number_of_files_64, supported_64) = match Self::get_directory_counts_zip64(reader, footer, cde_start_pos) { - Ok(result) => result, - Err(_) => return Self::get_directory_counts_zip32(footer, cde_start_pos) - }; + Ok(result) => result, + Err(_) => return Self::get_directory_counts_zip32(footer, cde_start_pos), + }; // Check if it also has a zip32 footer let (archive_offset_32, directory_start_32, number_of_files_32) = match Self::get_directory_counts_zip32(footer, cde_start_pos) { - Ok(result) => result, - Err(_) => { - supported_64?; - return Ok((archive_offset_64, directory_start_64, number_of_files_64)) - } - }; + Ok(result) => result, + Err(_) => { + supported_64?; + return Ok((archive_offset_64, directory_start_64, number_of_files_64)); + } + }; // It has both, so check if the zip64 footer is valid; if not, assume zip32 if archive_offset_64 < u32::MAX as u64 && archive_offset_64 != archive_offset_32 - || archive_offset_32 != u32::MAX as u64 { + || archive_offset_32 != u32::MAX as u64 + { return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } if directory_start_64 < u32::MAX as u64 && directory_start_64 != directory_start_32 - || directory_start_32 != u32::MAX as u64 { + || directory_start_32 != u32::MAX as u64 + { return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } if number_of_files_64 < u32::MAX as usize && number_of_files_64 != number_of_files_32 - || number_of_files_32 != u32::MAX as usize { + || number_of_files_32 != u32::MAX as usize + { return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } // It is, so we assume a zip64 supported_64?; - return Ok((archive_offset_64, directory_start_64, number_of_files_64)) + Ok((archive_offset_64, directory_start_64, number_of_files_64)) } /// Read a ZIP archive, collecting the files it contains diff --git a/src/write.rs b/src/write.rs index 83698965..bf50a1eb 100644 --- a/src/write.rs +++ b/src/write.rs @@ -439,7 +439,6 @@ impl ZipWriter { { let header_start = self.inner.get_plain().stream_position()?; let name = name.into(); - validate_name(&name)?; let permissions = options.permissions.unwrap_or(0o100644); let file = ZipFileData { @@ -1034,29 +1033,6 @@ impl ZipWriter { } } -#[cfg_attr(fuzzing, visibility::make(pub))] -#[cfg_attr(fuzzing, allow(missing_docs))] -pub(crate) fn validate_name(name: &String) -> ZipResult<()> { - let bytes = name.as_bytes(); - let mut current_window = [0u8; 4]; - for window in bytes.windows(4) { - current_window.copy_from_slice(window); - let magic_number = u32::from_le_bytes(current_window); - match magic_number { - spec::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE => { - return Err(InvalidArchive("Filename can't contain ZIP64 end signature")); - } - spec::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE => { - return Err(InvalidArchive( - "Filename can't contain ZIP64 end-locator signature", - )); - } - _ => {} - } - } - Ok(()) -} - impl Drop for ZipWriter { fn drop(&mut self) { if !self.inner.is_closed() { From 7bc245d7aacb251093dfb8eb4b2ab9ee34faa93c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 12 May 2023 08:38:12 -0700 Subject: [PATCH 166/281] Restore old write test cases --- src/write.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/write.rs b/src/write.rs index bf50a1eb..d3e0f505 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1732,6 +1732,62 @@ mod test { .start_file("foo/bar/test", FileOptions::default()) .expect_err("Expected duplicate filename not to be allowed"); } + + #[test] + fn test_filename_looks_like_zip64_locator() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2a() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_3() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer.start_file("\0PK\u{6}\u{6}", FileOptions::default()).unwrap(); + writer + .start_file( + "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", + FileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } } #[cfg(not(feature = "unreserved"))] From c90a8fd3cf9885be0636a29c0edeadadc8843482 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 12 May 2023 08:40:06 -0700 Subject: [PATCH 167/281] Reformat --- src/write.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index d3e0f505..69f58d65 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1777,7 +1777,9 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_3() { let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer.start_file("\0PK\u{6}\u{6}", FileOptions::default()).unwrap(); + writer + .start_file("\0PK\u{6}\u{6}", FileOptions::default()) + .unwrap(); writer .start_file( "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", From dd8318692802976854236d81a300afd1edcd1e3d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 12 May 2023 08:54:00 -0700 Subject: [PATCH 168/281] Bug fixes: validation was too stringent --- src/read.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/read.rs b/src/read.rs index d36e853c..2fe02dc7 100644 --- a/src/read.rs +++ b/src/read.rs @@ -397,19 +397,10 @@ impl ZipArchive { } }; // It has both, so check if the zip64 footer is valid; if not, assume zip32 - if archive_offset_64 < u32::MAX as u64 && archive_offset_64 != archive_offset_32 - || archive_offset_32 != u32::MAX as u64 - { + if directory_start_64 - archive_offset_64 != directory_start_32 - archive_offset_32 { return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } - if directory_start_64 < u32::MAX as u64 && directory_start_64 != directory_start_32 - || directory_start_32 != u32::MAX as u64 - { - return Ok((archive_offset_32, directory_start_32, number_of_files_32)); - } - if number_of_files_64 < u32::MAX as usize && number_of_files_64 != number_of_files_32 - || number_of_files_32 != u32::MAX as usize - { + if number_of_files_64 != number_of_files_32 && number_of_files_32 != u16::MAX as usize { return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } // It is, so we assume a zip64 From 9b65b8a523d1ab6abb4ceee1d4856aa21bac3d60 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 12 May 2023 09:07:42 -0700 Subject: [PATCH 169/281] Bug fix --- src/read.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/read.rs b/src/read.rs index 2fe02dc7..05e554aa 100644 --- a/src/read.rs +++ b/src/read.rs @@ -397,9 +397,6 @@ impl ZipArchive { } }; // It has both, so check if the zip64 footer is valid; if not, assume zip32 - if directory_start_64 - archive_offset_64 != directory_start_32 - archive_offset_32 { - return Ok((archive_offset_32, directory_start_32, number_of_files_32)); - } if number_of_files_64 != number_of_files_32 && number_of_files_32 != u16::MAX as usize { return Ok((archive_offset_32, directory_start_32, number_of_files_32)); } From 24752c2822727ccab84d71d2e64ec640215cd00b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 12 May 2023 12:43:24 -0700 Subject: [PATCH 170/281] Bump version to 0.7.5 --- CHANGELOG.md | 10 ++++++++-- Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be5fc267..0a8843ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,5 +116,11 @@ ### Merged from upstream -- Added experimental [`zip_next::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting - files with PKWARE encryption. + - Added experimental [`zip_next::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting + files with PKWARE encryption. + +## [0.7.5] + +### Fixed + + - Fixed a bug that occurs when ZIP64 magic bytes occur twice in a filename or across two filenames. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 2c579db7..6d1fe2f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.4" +version = "0.7.5" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index a52116eb..cd418b81 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.4" +zip_next = "0.7.5" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.4", default-features = false } +zip_next = { version = "0.7.5", default-features = false } ``` The features available are: From 80f836a6616d3f27b3d590c374ec075573a289b2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 1 May 2023 17:42:04 -0700 Subject: [PATCH 171/281] Use a struct for extra data --- src/types.rs | 9 ++++++++- src/write.rs | 35 ++++++----------------------------- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/src/types.rs b/src/types.rs index 821a275c..e25bbcc4 100644 --- a/src/types.rs +++ b/src/types.rs @@ -320,6 +320,13 @@ impl Clone for AtomicU64 { } } +#[derive(Debug, Clone)] +pub struct ZipExtraDataField { + /// Header ID; indicates the type of the extra data + pub(crate) header_id: u16, + pub(crate) data: Vec +} + /// Structure representing a ZIP file. #[derive(Debug, Clone)] pub struct ZipFileData { @@ -348,7 +355,7 @@ pub struct ZipFileData { /// Raw file name. To be used when file_name was incorrectly decoded. pub file_name_raw: Vec, /// Extra field usually used for storage expansion - pub extra_field: Vec, + pub extra_field: Vec, /// File comment pub file_comment: String, /// Specifies where the local header of the file starts diff --git a/src/write.rs b/src/write.rs index 69f58d65..1ab5b595 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1345,28 +1345,14 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) } fn validate_extra_data(file: &ZipFileData) -> ZipResult<()> { - let mut data = file.extra_field.as_slice(); - - if data.len() > spec::ZIP64_ENTRY_THR { - return Err(ZipError::Io(io::Error::new( - io::ErrorKind::InvalidData, - "Extra data exceeds extra field", - ))); - } - - while !data.is_empty() { - let left = data.len(); - if left < 4 { + for field in &file.extra_field { + if field.data.len() > u16::MAX as usize { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, - "Incomplete extra data header", + "Extra-data field can't exceed u16::MAX bytes", ))); } - let kind = data.read_u16::()?; - let size = data.read_u16::()? as usize; - let left = left - 4; - - if kind == 0x0001 { + if field.header_id == 0x0001 { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, "No custom ZIP64 extra data allowed", @@ -1375,24 +1361,15 @@ fn validate_extra_data(file: &ZipFileData) -> ZipResult<()> { #[cfg(not(feature = "unreserved"))] { - if kind <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == kind) { + if field.header_id <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == kind) { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, format!( - "Extra data header ID {kind:#06} requires crate feature \"unreserved\"", + "Extra data header ID {field.header_id:#06} requires crate feature \"unreserved\"", ), ))); } } - - if size > left { - return Err(ZipError::Io(io::Error::new( - io::ErrorKind::Other, - "Extra data size exceeds extra field", - ))); - } - - data = &data[size..]; } Ok(()) From 92c45cf8dd5395e09dfd8ddcf2a6157f13609b27 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 8 May 2023 10:03:30 -0700 Subject: [PATCH 172/281] WIP: Use ZipExtraDataField --- src/write.rs | 205 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 130 insertions(+), 75 deletions(-) diff --git a/src/write.rs b/src/write.rs index 1ab5b595..6399c81b 100644 --- a/src/write.rs +++ b/src/write.rs @@ -64,6 +64,15 @@ enum GenericZipWriter { Zstd(ZstdEncoder<'static, MaybeEncrypted>), } +enum ZipWriterState { + NotWritingFile, + WritingLocalExtraData, + WritingCentralOnlyExtraData, + WritingFileContents, + WritingFileContentsRaw, + Closed +} + // Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely pub(crate) mod zip_writer { use super::*; @@ -101,16 +110,15 @@ pub(crate) mod zip_writer { pub(super) files: Vec, pub(super) files_by_name: HashMap, pub(super) stats: ZipWriterStats, - pub(super) writing_to_file: bool, - pub(super) writing_to_extra_field: bool, - pub(super) writing_to_central_extra_field_only: bool, - pub(super) writing_raw: bool, + pub(super) state: ZipWriterState, pub(super) comment: Vec, } } use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; pub use zip_writer::ZipWriter; +use crate::write::ZipFileInitialWritingMode::Content; +use crate::write::ZipWriterState::NotWritingFile; #[derive(Default)] struct ZipWriterStats { @@ -125,6 +133,23 @@ struct ZipRawValues { uncompressed_size: u64, } +/// What operation the new file will be ready for when it is opened. +#[derive(Copy, Clone, Debug)] +pub enum ZipFileInitialWritingMode { + /// File will be empty and not opened for writing. + Empty, + /// File will be ready to have extra data added. + ExtraData, + /// File will be open for writing its contents. + Content +} + +#[derive(Copy, Clone, Debug)] +pub struct ZipExtraDataField { + header_id: u16, + data: Vec +} + /// Metadata for a file to be written #[derive(Copy, Clone, Debug)] #[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] @@ -135,6 +160,7 @@ pub struct FileOptions { pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, + pub(crate) write_first: ZipFileInitialWritingMode, } impl FileOptions { @@ -202,6 +228,14 @@ impl FileOptions { self.encrypt_with = Some(crate::zipcrypto::ZipCryptoKeys::derive(password)); self } + + /// Set whether extra data will be written before the content, or only content, or nothing at + /// all (the file will be empty). + #[must_use] + pub fn write_first(mut self, write_first: ZipFileInitialWritingMode) -> FileOptions { + self.write_first = write_first; + self + } } impl Default for FileOptions { @@ -228,22 +262,38 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, + write_first: Content } } } impl Write for ZipWriter { fn write(&mut self, buf: &[u8]) -> io::Result { - if !self.writing_to_file { - return Err(io::Error::new( - io::ErrorKind::Other, - "No file has been started", - )); - } match self.inner.ref_mut() { Some(ref mut w) => { + match self.state { + NotWritingFile => { + return Err(io::Error::new( + io::ErrorKind::Other, + "No file has been started", + )); + }, + ZipWriterState::Closed => { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "ZipWriter is closed", + )); + }, + ZipWriterState::WritingLocalExtraData => { + self.files.last_mut().unwrap().extra_field.write(buf) + } + ZipWriterState::WritingCentralOnlyExtraData => { + + } + ZipWriterState::WritingFileContents => {} + ZipWriterState::WritingFileContentsRaw => {} + } if self.writing_to_extra_field { - self.files.last_mut().unwrap().extra_field.write(buf) } else { let write_result = w.write(buf); if let Ok(count) = write_result { @@ -322,11 +372,8 @@ impl ZipWriter { files, files_by_name, stats: Default::default(), - writing_to_file: false, - writing_to_extra_field: false, - writing_to_central_extra_field_only: false, - comment: footer.zip_file_comment, - writing_raw: true, // avoid recomputing the last file's header + state: NotWritingFile, + comment: footer.zip_file_comment }) } } @@ -389,17 +436,21 @@ impl ZipWriter { files: Vec::new(), files_by_name: HashMap::new(), stats: Default::default(), - writing_to_file: false, - writing_to_extra_field: false, - writing_to_central_extra_field_only: false, - writing_raw: false, + state: NotWritingFile, comment: Vec::new(), } } /// Returns true if a file is currently open for writing. pub fn is_writing_file(&self) -> bool { - self.writing_to_file && !self.inner.is_closed() + match self.state { + NotWritingFile => false, + ZipWriterState::WritingLocalExtraData => true, + ZipWriterState::WritingCentralOnlyExtraData => true, + ZipWriterState::WritingFileContents => true, + ZipWriterState::WritingFileContentsRaw => true, + ZipWriterState::Closed => false + } } /// Set ZIP archive comment. @@ -502,19 +553,22 @@ impl ZipWriter { } fn finish_file(&mut self) -> ZipResult<()> { - if !self.writing_to_file { - debug_assert!(!self.writing_to_extra_field); - return Ok(()); - } - if self.writing_to_extra_field { - // Implicitly calling [`ZipWriter::end_extra_data`] for empty files. - self.end_extra_data()?; - } - let make_plain_writer = self - .inner - .prepare_next_writer(CompressionMethod::Stored, None)?; - self.inner.switch_to(make_plain_writer)?; - match mem::replace(&mut self.inner, Closed) { + match self.state { + NotWritingFile => { + return Ok(()); + } + ZipWriterState::WritingLocalExtraData => { + self.end_extra_data()?; + } + ZipWriterState::WritingCentralOnlyExtraData => { + self.end_extra_data()?; + } + ZipWriterState::WritingFileContents => { + let make_plain_writer = self + .inner + .prepare_next_writer(CompressionMethod::Stored, None)?; + self.inner.switch_to(make_plain_writer)?; + match mem::replace(&mut self.inner, Closed) { Storer(MaybeEncrypted::Encrypted(writer)) => { let crc32 = self.stats.hasher.clone().finalize(); self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) @@ -524,23 +578,28 @@ impl ZipWriter { } let writer = self.inner.get_plain(); - if !self.writing_raw { - let file = match self.files.last_mut() { - None => return Ok(()), - Some(f) => f, - }; - file.crc32 = self.stats.hasher.clone().finalize(); - file.uncompressed_size = self.stats.bytes_written; + if !self.writing_raw { + let file = match self.files.last_mut() { + None => return Ok(()), + Some(f) => f, + }; + file.crc32 = self.stats.hasher.clone().finalize(); + file.uncompressed_size = self.stats.bytes_written; - let file_end = writer.stream_position()?; - debug_assert!(file_end >= self.stats.start); - file.compressed_size = file_end - self.stats.start; + let file_end = writer.stream_position()?; + debug_assert!(file_end >= self.stats.start); + file.compressed_size = file_end - self.stats.start; - update_local_file_header(writer, file)?; - writer.seek(SeekFrom::Start(file_end))?; + update_local_file_header(writer, file)?; + writer.seek(SeekFrom::Start(file_end))?; + } + } + ZipWriterState::WritingFileContentsRaw => {} + ZipWriterState::Closed => { + return Ok(()); + } } - - self.writing_to_file = false; + self.state = NotWritingFile; Ok(()) } @@ -556,9 +615,7 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; - self.writing_to_extra_field = false; - self.writing_to_file = false; - self.writing_raw = false; + self.state = NotWritingFile; Ok(()) } @@ -1344,31 +1401,29 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) Ok(()) } -fn validate_extra_data(file: &ZipFileData) -> ZipResult<()> { - for field in &file.extra_field { - if field.data.len() > u16::MAX as usize { - return Err(ZipError::Io(io::Error::new( - io::ErrorKind::Other, - "Extra-data field can't exceed u16::MAX bytes", - ))); - } - if field.header_id == 0x0001 { - return Err(ZipError::Io(io::Error::new( - io::ErrorKind::Other, - "No custom ZIP64 extra data allowed", - ))); - } +fn validate_extra_data(field: &ZipExtraDataField) -> ZipResult<()> { + if field.data.len() > u16::MAX as usize { + return Err(ZipError::Io(io::Error::new( + io::ErrorKind::Other, + "Extra-data field can't exceed u16::MAX bytes", + ))); + } + if field.header_id == 0x0001 { + return Err(ZipError::Io(io::Error::new( + io::ErrorKind::Other, + "No custom ZIP64 extra data allowed", + ))); + } - #[cfg(not(feature = "unreserved"))] - { - if field.header_id <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == kind) { - return Err(ZipError::Io(io::Error::new( - io::ErrorKind::Other, - format!( - "Extra data header ID {field.header_id:#06} requires crate feature \"unreserved\"", - ), - ))); - } + #[cfg(not(feature = "unreserved"))] + { + if field.header_id <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == kind) { + return Err(ZipError::Io(io::Error::new( + io::ErrorKind::Other, + format!( + "Extra data header ID {field.header_id:#06} requires crate feature \"unreserved\"", + ), + ))); } } From 4faebb44688ca0aa72e115686bb09c421a4017cc Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 13:59:14 -0700 Subject: [PATCH 173/281] Overhaul extra-data interface --- examples/write_dir.rs | 4 +- src/read.rs | 2 + src/types.rs | 12 +- src/write.rs | 575 +++++++++++++++--------------------------- tests/end_to_end.rs | 16 +- 5 files changed, 224 insertions(+), 385 deletions(-) diff --git a/examples/write_dir.rs b/examples/write_dir.rs index 77136703..f0d9efcd 100644 --- a/examples/write_dir.rs +++ b/examples/write_dir.rs @@ -88,7 +88,7 @@ where if path.is_file() { println!("adding file {path:?} as {name:?} ..."); #[allow(deprecated)] - zip.start_file_from_path(name, options)?; + zip.start_file_from_path(name, options.clone())?; let mut f = File::open(path)?; f.read_to_end(&mut buffer)?; @@ -99,7 +99,7 @@ where // and mapname conversion failed error on unzip println!("adding dir {path:?} as {name:?} ..."); #[allow(deprecated)] - zip.add_directory_from_path(name, options)?; + zip.add_directory_from_path(name, options.clone())?; } } zip.finish()?; diff --git a/src/read.rs b/src/read.rs index 05e554aa..7efd1f9a 100644 --- a/src/read.rs +++ b/src/read.rs @@ -727,6 +727,7 @@ fn central_header_to_zip_file_inner( file_name, file_name_raw, extra_field, + central_extra_field: vec![], file_comment, header_start: offset, central_header_start, @@ -1087,6 +1088,7 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult -} - /// Structure representing a ZIP file. #[derive(Debug, Clone)] pub struct ZipFileData { @@ -355,7 +348,9 @@ pub struct ZipFileData { /// Raw file name. To be used when file_name was incorrectly decoded. pub file_name_raw: Vec, /// Extra field usually used for storage expansion - pub extra_field: Vec, + pub extra_field: Vec, + /// Extra field only written to central directory + pub central_extra_field: Vec, /// File comment pub file_comment: String, /// Specifies where the local header of the file starts @@ -523,6 +518,7 @@ mod test { file_name: file_name.clone(), file_name_raw: file_name.into_bytes(), extra_field: Vec::new(), + central_extra_field: vec![], file_comment: String::new(), header_start: 0, data_start: AtomicU64::new(0), diff --git a/src/write.rs b/src/write.rs index 6399c81b..f94a1992 100644 --- a/src/write.rs +++ b/src/write.rs @@ -5,7 +5,7 @@ use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, use crate::result::{ZipError, ZipResult}; use crate::spec; use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use byteorder::{LittleEndian, WriteBytesExt}; use crc32fast::Hasher; use std::collections::HashMap; use std::convert::TryInto; @@ -64,15 +64,6 @@ enum GenericZipWriter { Zstd(ZstdEncoder<'static, MaybeEncrypted>), } -enum ZipWriterState { - NotWritingFile, - WritingLocalExtraData, - WritingCentralOnlyExtraData, - WritingFileContents, - WritingFileContentsRaw, - Closed -} - // Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely pub(crate) mod zip_writer { use super::*; @@ -110,15 +101,15 @@ pub(crate) mod zip_writer { pub(super) files: Vec, pub(super) files_by_name: HashMap, pub(super) stats: ZipWriterStats, - pub(super) state: ZipWriterState, + pub(super) writing_to_file: bool, + pub(super) writing_raw: bool, pub(super) comment: Vec, } } use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; pub use zip_writer::ZipWriter; -use crate::write::ZipFileInitialWritingMode::Content; -use crate::write::ZipWriterState::NotWritingFile; +use crate::zipcrypto::ZipCryptoKeys; #[derive(Default)] struct ZipWriterStats { @@ -133,34 +124,45 @@ struct ZipRawValues { uncompressed_size: u64, } -/// What operation the new file will be ready for when it is opened. -#[derive(Copy, Clone, Debug)] -pub enum ZipFileInitialWritingMode { - /// File will be empty and not opened for writing. - Empty, - /// File will be ready to have extra data added. - ExtraData, - /// File will be open for writing its contents. - Content -} - -#[derive(Copy, Clone, Debug)] -pub struct ZipExtraDataField { - header_id: u16, - data: Vec -} - /// Metadata for a file to be written -#[derive(Copy, Clone, Debug)] -#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] +#[derive(Clone, Debug)] pub struct FileOptions { pub(crate) compression_method: CompressionMethod, pub(crate) compression_level: Option, pub(crate) last_modified_time: DateTime, pub(crate) permissions: Option, pub(crate) large_file: bool, - encrypt_with: Option, - pub(crate) write_first: ZipFileInitialWritingMode, + encrypt_with: Option, + extra_data: Vec, + central_extra_data: Vec, + alignment: u16 +} + +#[cfg(fuzzing)] +impl arbitrary::Arbitrary for FileOptions { + fn arbitrary(u: &mut Unstructured) -> arbitrary::Result { + let mut options = FileOptions { + compression_method: CompressionMethod::arbitrary(&mut u), + compression_level: Option::::arbitrary(&mut u), + last_modified_time: DateTime::arbitrary(&mut u), + permissions: Option::::arbitrary(&mut u), + large_file: bool::arbitrary(&mut u), + encrypt_with: Option::::arbitrary(&mut u), + extra_data: Vec::with_capacity(u16::MAX as usize), + central_extra_data: Vec::with_capacity(u16::MAX as usize), + }; + #[derive(arbitrary::Arbitrary)] + struct ExtraDataField { + header_id: u16, + data: Vec, + central_only: bool + } + let extra_data = Vec::::arbitrary(&mut u); + for field in extra_data { + let _ = options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only); + } + options + } } impl FileOptions { @@ -216,24 +218,43 @@ impl FileOptions { /// Set whether the new file's compressed and uncompressed size is less than 4 GiB. /// - /// If set to `false` and the file exceeds the limit, an I/O error is thrown. If set to `true`, - /// readers will require ZIP64 support and if the file does not exceed the limit, 20 B are - /// wasted. The default is `false`. + /// If set to `false` and the file exceeds the limit, an I/O error is thrown and the file is + /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not + /// exceed the limit, 20 B are wasted. The default is `false`. #[must_use] pub fn large_file(mut self, large: bool) -> FileOptions { self.large_file = large; self } pub(crate) fn with_deprecated_encryption(mut self, password: &[u8]) -> FileOptions { - self.encrypt_with = Some(crate::zipcrypto::ZipCryptoKeys::derive(password)); + self.encrypt_with = Some(ZipCryptoKeys::derive(password)); self } - /// Set whether extra data will be written before the content, or only content, or nothing at - /// all (the file will be empty). + /// Adds an extra data field. + pub fn add_extra_data(&mut self, header_id: u16, data: &[u8], central_only: bool) -> ZipResult<()> { + validate_extra_data(header_id, data)?; + let len = data.len() + 4; + if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { + Err(InvalidArchive("Extra data field would be longer than allowed")) + } else { + let field = if central_only { + &mut self.central_extra_data + } else { + &mut self.extra_data + }; + field.write_u16::(header_id)?; + field.write_u16::(data.len() as u16)?; + field.write_all(data)?; + Ok(()) + } + } + + /// Removes the extra data fields. #[must_use] - pub fn write_first(mut self, write_first: ZipFileInitialWritingMode) -> FileOptions { - self.write_first = write_first; + pub fn clear_extra_data(mut self) -> FileOptions { + self.extra_data.clear(); + self.central_extra_data.clear(); self } } @@ -262,54 +283,37 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, - write_first: Content + extra_data: Vec::with_capacity(u16::MAX as usize), + central_extra_data: Vec::with_capacity(u16::MAX as usize), + alignment: 1 } } } impl Write for ZipWriter { fn write(&mut self, buf: &[u8]) -> io::Result { + if !self.writing_to_file { + return Err(io::Error::new( + io::ErrorKind::Other, + "No file has been started", + )); + } match self.inner.ref_mut() { Some(ref mut w) => { - match self.state { - NotWritingFile => { + let write_result = w.write(buf); + if let Ok(count) = write_result { + self.stats.update(&buf[0..count]); + if self.stats.bytes_written > spec::ZIP64_BYTES_THR + && !self.files.last_mut().unwrap().large_file + { + self.abort_file().unwrap(); return Err(io::Error::new( io::ErrorKind::Other, - "No file has been started", + "Large file option has not been set", )); - }, - ZipWriterState::Closed => { - return Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "ZipWriter is closed", - )); - }, - ZipWriterState::WritingLocalExtraData => { - self.files.last_mut().unwrap().extra_field.write(buf) } - ZipWriterState::WritingCentralOnlyExtraData => { - - } - ZipWriterState::WritingFileContents => {} - ZipWriterState::WritingFileContentsRaw => {} - } - if self.writing_to_extra_field { - } else { - let write_result = w.write(buf); - if let Ok(count) = write_result { - self.stats.update(&buf[0..count]); - if self.stats.bytes_written > spec::ZIP64_BYTES_THR - && !self.files.last_mut().unwrap().large_file - { - self.abort_file().unwrap(); - return Err(io::Error::new( - io::ErrorKind::Other, - "Large file option has not been set", - )); - } - } - write_result } + write_result } None => Err(io::Error::new( io::ErrorKind::BrokenPipe, @@ -372,8 +376,9 @@ impl ZipWriter { files, files_by_name, stats: Default::default(), - state: NotWritingFile, - comment: footer.zip_file_comment + writing_to_file: false, + comment: footer.zip_file_comment, + writing_raw: true, // avoid recomputing the last file's header }) } } @@ -436,21 +441,15 @@ impl ZipWriter { files: Vec::new(), files_by_name: HashMap::new(), stats: Default::default(), - state: NotWritingFile, + writing_to_file: false, + writing_raw: false, comment: Vec::new(), } } /// Returns true if a file is currently open for writing. pub fn is_writing_file(&self) -> bool { - match self.state { - NotWritingFile => false, - ZipWriterState::WritingLocalExtraData => true, - ZipWriterState::WritingCentralOnlyExtraData => true, - ZipWriterState::WritingFileContents => true, - ZipWriterState::WritingFileContentsRaw => true, - ZipWriterState::Closed => false - } + self.writing_to_file && !self.inner.is_closed() } /// Set ZIP archive comment. @@ -505,7 +504,8 @@ impl ZipWriter { uncompressed_size: raw_values.uncompressed_size, file_name: name, file_name_raw: Vec::new(), // Never used for saving - extra_field: Vec::new(), + extra_field: options.extra_data, + central_extra_field: options.central_extra_data, file_comment: String::new(), header_start, data_start: AtomicU64::new(0), @@ -517,26 +517,93 @@ impl ZipWriter { let index = self.insert_file_data(file)?; let file = &mut self.files[index]; let writer = self.inner.get_plain(); - write_local_file_header(writer, file)?; + writer.write_u32::(spec::LOCAL_FILE_HEADER_SIGNATURE)?; + // version needed to extract + writer.write_u16::(file.version_needed())?; + // general purpose bit flag + let flag = if !file.file_name.is_ascii() { + 1u16 << 11 + } else { + 0 + } | if file.encrypted { 1u16 << 0 } else { 0 }; + writer.write_u16::(flag)?; + // Compression method + #[allow(deprecated)] + writer.write_u16::(file.compression_method.to_u16())?; + // last mod file time and last mod file date + writer.write_u16::(file.last_modified_time.timepart())?; + writer.write_u16::(file.last_modified_time.datepart())?; + // crc-32 + writer.write_u32::(file.crc32)?; + // compressed size and uncompressed size + if file.large_file { + writer.write_u32::(spec::ZIP64_BYTES_THR as u32)?; + writer.write_u32::(spec::ZIP64_BYTES_THR as u32)?; + } else { + writer.write_u32::(file.compressed_size as u32)?; + writer.write_u32::(file.uncompressed_size as u32)?; + } + // file name length + writer.write_u16::(file.file_name.as_bytes().len() as u16)?; + // extra field length + let mut extra_field_length = file.extra_field.len(); + if file.large_file { + extra_field_length += 20; + } + if extra_field_length + file.central_extra_field.len() > u16::MAX as usize { + let _ = self.abort_file(); + return Err(InvalidArchive("Extra data field is too large")); + } + let extra_field_length = extra_field_length as u16; + writer.write_u16::(extra_field_length)?; + // file name + writer.write_all(file.file_name.as_bytes())?; + // zip64 extra field + if file.large_file { + write_local_zip64_extra_field(writer, file)?; + } + writer.write_all(&file.extra_field)?; + let mut header_end = writer.stream_position()?; + if options.alignment > 1 { + let align = options.alignment as u64; + if header_end % align != 0 { + let pad_length = (align - (header_end + 4) % align) % align; + if pad_length + extra_field_length as u64 > u16::MAX as u64 { + let _ = self.abort_file(); + return Err(InvalidArchive("Extra data field would be larger than allowed after aligning")); + } + let pad = vec![0; pad_length as usize]; + writer.write_all(b"za").map_err(ZipError::from)?; // 0x617a + writer.write_u16::(pad.len() as u16) + .map_err(ZipError::from)?; + writer.write_all(&pad).map_err(ZipError::from)?; + header_end = writer.stream_position()?; - let header_end = writer.stream_position()?; + // Update extra field length in local file header. + writer.seek(SeekFrom::Start(file.header_start + 28))?; + writer.write_u16::(pad_length as u16 + extra_field_length)?; + writer.seek(SeekFrom::Start(header_end))?; + debug_assert_eq!(header_end % align, 0); + } + } + if let Some(keys) = options.encrypt_with { + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + buffer: vec![], + keys, + }; + let crypto_header = [0u8; 12]; + + zipwriter.write_all(&crypto_header)?; + header_end = zipwriter.writer.stream_position()?; + self.inner = Storer(MaybeEncrypted::Encrypted(zipwriter)); + } self.stats.start = header_end; *file.data_start.get_mut() = header_end; - + self.writing_to_file = true; self.stats.bytes_written = 0; self.stats.hasher = Hasher::new(); } - if let Some(keys) = options.encrypt_with { - let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { - writer: mem::replace(&mut self.inner, Closed).unwrap(), - buffer: vec![], - keys, - }; - let crypto_header = [0u8; 12]; - - zipwriter.write_all(&crypto_header)?; - self.inner = Storer(MaybeEncrypted::Encrypted(zipwriter)); - } Ok(()) } @@ -553,22 +620,15 @@ impl ZipWriter { } fn finish_file(&mut self) -> ZipResult<()> { - match self.state { - NotWritingFile => { - return Ok(()); - } - ZipWriterState::WritingLocalExtraData => { - self.end_extra_data()?; - } - ZipWriterState::WritingCentralOnlyExtraData => { - self.end_extra_data()?; - } - ZipWriterState::WritingFileContents => { - let make_plain_writer = self - .inner - .prepare_next_writer(CompressionMethod::Stored, None)?; - self.inner.switch_to(make_plain_writer)?; - match mem::replace(&mut self.inner, Closed) { + if !self.writing_to_file { + return Ok(()); + } + + let make_plain_writer = self + .inner + .prepare_next_writer(CompressionMethod::Stored, None)?; + self.inner.switch_to(make_plain_writer)?; + match mem::replace(&mut self.inner, Closed) { Storer(MaybeEncrypted::Encrypted(writer)) => { let crc32 = self.stats.hasher.clone().finalize(); self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) @@ -578,28 +638,23 @@ impl ZipWriter { } let writer = self.inner.get_plain(); - if !self.writing_raw { - let file = match self.files.last_mut() { - None => return Ok(()), - Some(f) => f, - }; - file.crc32 = self.stats.hasher.clone().finalize(); - file.uncompressed_size = self.stats.bytes_written; + if !self.writing_raw { + let file = match self.files.last_mut() { + None => return Ok(()), + Some(f) => f, + }; + file.crc32 = self.stats.hasher.clone().finalize(); + file.uncompressed_size = self.stats.bytes_written; - let file_end = writer.stream_position()?; - debug_assert!(file_end >= self.stats.start); - file.compressed_size = file_end - self.stats.start; + let file_end = writer.stream_position()?; + debug_assert!(file_end >= self.stats.start); + file.compressed_size = file_end - self.stats.start; - update_local_file_header(writer, file)?; - writer.seek(SeekFrom::Start(file_end))?; - } - } - ZipWriterState::WritingFileContentsRaw => {} - ZipWriterState::Closed => { - return Ok(()); - } + update_local_file_header(writer, file)?; + writer.seek(SeekFrom::Start(file_end))?; } - self.state = NotWritingFile; + + self.writing_to_file = false; Ok(()) } @@ -615,7 +670,7 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; - self.state = NotWritingFile; + self.writing_to_file = false; Ok(()) } @@ -636,7 +691,6 @@ impl ZipWriter { self.abort_file().unwrap(); return Err(e); } - self.writing_to_file = true; self.writing_raw = false; Ok(()) } @@ -667,182 +721,6 @@ impl ZipWriter { self.start_file(path_to_string(path), options) } - /// Create an aligned file in the archive and start writing its' contents. - /// - /// Returns the number of padding bytes required to align the file. - /// - /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] - pub fn start_file_aligned( - &mut self, - name: S, - options: FileOptions, - align: u16, - ) -> Result - where - S: Into, - { - let data_start = self.start_file_with_extra_data(name, options)?; - let align = align as u64; - if align > 1 && data_start % align != 0 { - let pad_length = (align - (data_start + 4) % align) % align; - let pad = vec![0; pad_length as usize]; - self.write_all(b"za").map_err(ZipError::from)?; // 0x617a - self.write_u16::(pad.len() as u16) - .map_err(ZipError::from)?; - self.write_all(&pad).map_err(ZipError::from)?; - assert_eq!(self.end_local_start_central_extra_data()? % align, 0); - } - let extra_data_end = self.end_extra_data()?; - Ok(extra_data_end - data_start) - } - - /// Create a file in the archive and start writing its extra data first. - /// - /// Finish writing extra data and start writing file data with [`ZipWriter::end_extra_data`]. - /// Optionally, distinguish local from central extra data with - /// [`ZipWriter::end_local_start_central_extra_data`]. - /// - /// Returns the preliminary starting offset of the file data without any extra data allowing to - /// align the file data by calculating a pad length to be prepended as part of the extra data. - /// - /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] - /// - /// ``` - /// use byteorder::{LittleEndian, WriteBytesExt}; - /// use zip_next::{ZipArchive, ZipWriter, result::ZipResult}; - /// use zip_next::{write::FileOptions, CompressionMethod}; - /// use std::io::{Write, Cursor}; - /// - /// # fn main() -> ZipResult<()> { - /// let mut archive = Cursor::new(Vec::new()); - /// - /// { - /// let mut zip = ZipWriter::new(&mut archive); - /// let options = FileOptions::default() - /// .compression_method(CompressionMethod::Stored); - /// - /// zip.start_file_with_extra_data("identical_extra_data.txt", options)?; - /// let extra_data = b"local and central extra data"; - /// zip.write_u16::(0xbeef)?; - /// zip.write_u16::(extra_data.len() as u16)?; - /// zip.write_all(extra_data)?; - /// zip.end_extra_data()?; - /// zip.write_all(b"file data")?; - /// - /// let data_start = zip.start_file_with_extra_data("different_extra_data.txt", options)?; - /// let extra_data = b"local extra data"; - /// zip.write_u16::(0xbeef)?; - /// zip.write_u16::(extra_data.len() as u16)?; - /// zip.write_all(extra_data)?; - /// let data_start = data_start as usize + 4 + extra_data.len() + 4; - /// let align = 64; - /// let pad_length = (align - data_start % align) % align; - /// assert_eq!(pad_length, 19); - /// zip.write_u16::(0xdead)?; - /// zip.write_u16::(pad_length as u16)?; - /// zip.write_all(&vec![0; pad_length])?; - /// let data_start = zip.end_local_start_central_extra_data()?; - /// assert_eq!(data_start as usize % align, 0); - /// let extra_data = b"central extra data"; - /// zip.write_u16::(0xbeef)?; - /// zip.write_u16::(extra_data.len() as u16)?; - /// zip.write_all(extra_data)?; - /// zip.end_extra_data()?; - /// zip.write_all(b"file data")?; - /// - /// zip.finish()?; - /// } - /// - /// let mut zip = ZipArchive::new(archive)?; - /// assert_eq!(&zip.by_index(0)?.extra_data()[4..], b"local and central extra data"); - /// assert_eq!(&zip.by_index(1)?.extra_data()[4..], b"central extra data"); - /// # Ok(()) - /// # } - /// ``` - pub fn start_file_with_extra_data( - &mut self, - name: S, - mut options: FileOptions, - ) -> ZipResult - where - S: Into, - { - Self::normalize_options(&mut options); - self.start_entry(name, options, None)?; - self.writing_to_file = true; - self.writing_to_extra_field = true; - Ok(self.files.last().unwrap().data_start.load()) - } - - /// End local and start central extra data. Requires [`ZipWriter::start_file_with_extra_data`]. - /// - /// Returns the final starting offset of the file data. - pub fn end_local_start_central_extra_data(&mut self) -> ZipResult { - let data_start = self.end_extra_data()?; - self.files.last_mut().unwrap().extra_field.clear(); - self.writing_to_extra_field = true; - self.writing_to_central_extra_field_only = true; - Ok(data_start) - } - - /// End extra data and start file data. Requires [`ZipWriter::start_file_with_extra_data`]. - /// - /// Returns the final starting offset of the file data. - pub fn end_extra_data(&mut self) -> ZipResult { - // Require `start_file_with_extra_data()`. Ensures `file` is some. - if !self.writing_to_extra_field { - return Err(ZipError::Io(io::Error::new( - io::ErrorKind::Other, - "Not writing to extra field", - ))); - } - let file = self.files.last_mut().unwrap(); - - if let Err(e) = validate_extra_data(file) { - self.abort_file().unwrap(); - return Err(e); - } - - let make_compressing_writer = match self - .inner - .prepare_next_writer(file.compression_method, file.compression_level) - { - Ok(writer) => writer, - Err(e) => { - self.abort_file().unwrap(); - return Err(e); - } - }; - - let mut data_start_result = file.data_start.load(); - - if !self.writing_to_central_extra_field_only { - let writer = self.inner.get_plain(); - - // Append extra data to local file header and keep it for central file header. - writer.write_all(&file.extra_field)?; - - // Update final `data_start`. - let length = file.extra_field.len(); - let data_start = file.data_start.get_mut(); - let header_end = *data_start + length as u64; - self.stats.start = header_end; - *data_start = header_end; - data_start_result = header_end; - - // Update extra field length in local file header. - let extra_field_length = - if file.large_file { 20 } else { 0 } + file.extra_field.len() as u16; - writer.seek(SeekFrom::Start(file.header_start + 28))?; - writer.write_u16::(extra_field_length)?; - writer.seek(SeekFrom::Start(header_end))?; - } - self.inner.switch_to(make_compressing_writer)?; - self.writing_to_extra_field = false; - self.writing_to_central_extra_field_only = false; - Ok(data_start_result) - } - /// Add a new file using the already compressed data from a ZIP file being read and renames it, this /// allows faster copies of the `ZipFile` since there is no need to decompress and compress it again. /// Any `ZipFile` metadata is copied and not checked, for example the file CRC. @@ -1279,49 +1157,6 @@ fn clamp_opt(value: T, range: std::ops::RangeInclusive) -> Opt } } -fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { - // local file header signature - writer.write_u32::(spec::LOCAL_FILE_HEADER_SIGNATURE)?; - // version needed to extract - writer.write_u16::(file.version_needed())?; - // general purpose bit flag - let flag = if !file.file_name.is_ascii() { - 1u16 << 11 - } else { - 0 - } | if file.encrypted { 1u16 << 0 } else { 0 }; - writer.write_u16::(flag)?; - // Compression method - #[allow(deprecated)] - writer.write_u16::(file.compression_method.to_u16())?; - // last mod file time and last mod file date - writer.write_u16::(file.last_modified_time.timepart())?; - writer.write_u16::(file.last_modified_time.datepart())?; - // crc-32 - writer.write_u32::(file.crc32)?; - // compressed size and uncompressed size - if file.large_file { - writer.write_u32::(spec::ZIP64_BYTES_THR as u32)?; - writer.write_u32::(spec::ZIP64_BYTES_THR as u32)?; - } else { - writer.write_u32::(file.compressed_size as u32)?; - writer.write_u32::(file.uncompressed_size as u32)?; - } - // file name length - writer.write_u16::(file.file_name.as_bytes().len() as u16)?; - // extra field length - let extra_field_length = if file.large_file { 20 } else { 0 } + file.extra_field.len() as u16; - writer.write_u16::(extra_field_length)?; - // file name - writer.write_all(file.file_name.as_bytes())?; - // zip64 extra field - if file.large_file { - write_local_zip64_extra_field(writer, file)?; - } - - Ok(()) -} - fn update_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { const CRC32_OFFSET: u64 = 14; writer.seek(SeekFrom::Start(file.header_start + CRC32_OFFSET))?; @@ -1378,7 +1213,8 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // file name length writer.write_u16::(file.file_name.as_bytes().len() as u16)?; // extra field length - writer.write_u16::(zip64_extra_field_length + file.extra_field.len() as u16)?; + writer.write_u16::(zip64_extra_field_length + file.extra_field.len() as u16 + + file.central_extra_field.len() as u16)?; // file comment length writer.write_u16::(0)?; // disk number start @@ -1401,14 +1237,14 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) Ok(()) } -fn validate_extra_data(field: &ZipExtraDataField) -> ZipResult<()> { - if field.data.len() > u16::MAX as usize { +fn validate_extra_data(header_id: u16, data: &[u8]) -> ZipResult<()> { + if data.len() > u16::MAX as usize { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, "Extra-data field can't exceed u16::MAX bytes", ))); } - if field.header_id == 0x0001 { + if header_id == 0x0001 { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, "No custom ZIP64 extra data allowed", @@ -1417,11 +1253,11 @@ fn validate_extra_data(field: &ZipExtraDataField) -> ZipResult<()> { #[cfg(not(feature = "unreserved"))] { - if field.header_id <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == kind) { + if header_id <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == header_id) { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, format!( - "Extra data header ID {field.header_id:#06} requires crate feature \"unreserved\"", + "Extra data header ID {header_id:#06} requires crate feature \"unreserved\"", ), ))); } @@ -1633,6 +1469,9 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, + extra_data: vec![], + central_extra_data: vec![], + alignment: 1, }; writer.start_file("mimetype", options).unwrap(); writer @@ -1670,6 +1509,9 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, + extra_data: vec![], + central_extra_data: vec![], + alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); @@ -1717,6 +1559,9 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, + extra_data: vec![], + central_extra_data: vec![], + alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 047b5955..574e8fdc 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -99,11 +99,11 @@ fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, sha zip.add_directory("test/", Default::default()).unwrap(); - let options = FileOptions::default() + let mut options = FileOptions::default() .compression_method(method) .unix_permissions(0o755); - zip.start_file(ENTRY_NAME, options).unwrap(); + zip.start_file(ENTRY_NAME, options.clone()).unwrap(); zip.write_all(LOREM_IPSUM).unwrap(); if shallow_copy { @@ -114,16 +114,12 @@ fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, sha .unwrap(); } - zip.start_file("test/☃.txt", options).unwrap(); + zip.start_file("test/☃.txt", options.clone()).unwrap(); zip.write_all(b"Hello, World!\n").unwrap(); - zip.start_file_with_extra_data("test_with_extra_data/🐢.txt", options) - .unwrap(); - zip.write_u16::(0xbeef).unwrap(); - zip.write_u16::(EXTRA_DATA.len() as u16) - .unwrap(); - zip.write_all(EXTRA_DATA).unwrap(); - zip.end_extra_data().unwrap(); + options.add_extra_data(0xbeef, EXTRA_DATA, false).unwrap(); + + zip.start_file("test_with_extra_data/🐢.txt", options).unwrap(); zip.write_all(b"Hello, World! Again.\n").unwrap(); zip.finish().unwrap(); From bf0ad491c04868dd712cebdb754173ee63566c1b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:02:34 -0700 Subject: [PATCH 174/281] Bug fix --- benches/read_metadata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index b4d6bb36..f9be2ec3 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -18,7 +18,7 @@ fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { for i in 0..count_files { let name = format!("file_deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef_{i}.dat"); - writer.start_file(name, options).unwrap(); + writer.start_file(name, options.clone()).unwrap(); writer.write_all(&bytes).unwrap(); } From 429b5dfa0b903e6c6e57ec28c31421c700c51859 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:03:02 -0700 Subject: [PATCH 175/281] Reformat --- src/write.rs | 42 ++++++++++++++++++++++++++++++------------ tests/end_to_end.rs | 3 ++- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/write.rs b/src/write.rs index f94a1992..26667882 100644 --- a/src/write.rs +++ b/src/write.rs @@ -108,8 +108,8 @@ pub(crate) mod zip_writer { } use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; -pub use zip_writer::ZipWriter; use crate::zipcrypto::ZipCryptoKeys; +pub use zip_writer::ZipWriter; #[derive(Default)] struct ZipWriterStats { @@ -135,7 +135,7 @@ pub struct FileOptions { encrypt_with: Option, extra_data: Vec, central_extra_data: Vec, - alignment: u16 + alignment: u16, } #[cfg(fuzzing)] @@ -155,11 +155,12 @@ impl arbitrary::Arbitrary for FileOptions { struct ExtraDataField { header_id: u16, data: Vec, - central_only: bool + central_only: bool, } let extra_data = Vec::::arbitrary(&mut u); for field in extra_data { - let _ = options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only); + let _ = + options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only); } options } @@ -232,11 +233,18 @@ impl FileOptions { } /// Adds an extra data field. - pub fn add_extra_data(&mut self, header_id: u16, data: &[u8], central_only: bool) -> ZipResult<()> { + pub fn add_extra_data( + &mut self, + header_id: u16, + data: &[u8], + central_only: bool, + ) -> ZipResult<()> { validate_extra_data(header_id, data)?; let len = data.len() + 4; if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { - Err(InvalidArchive("Extra data field would be longer than allowed")) + Err(InvalidArchive( + "Extra data field would be longer than allowed", + )) } else { let field = if central_only { &mut self.central_extra_data @@ -285,7 +293,7 @@ impl Default for FileOptions { encrypt_with: None, extra_data: Vec::with_capacity(u16::MAX as usize), central_extra_data: Vec::with_capacity(u16::MAX as usize), - alignment: 1 + alignment: 1, } } } @@ -570,11 +578,14 @@ impl ZipWriter { let pad_length = (align - (header_end + 4) % align) % align; if pad_length + extra_field_length as u64 > u16::MAX as u64 { let _ = self.abort_file(); - return Err(InvalidArchive("Extra data field would be larger than allowed after aligning")); + return Err(InvalidArchive( + "Extra data field would be larger than allowed after aligning", + )); } let pad = vec![0; pad_length as usize]; writer.write_all(b"za").map_err(ZipError::from)?; // 0x617a - writer.write_u16::(pad.len() as u16) + writer + .write_u16::(pad.len() as u16) .map_err(ZipError::from)?; writer.write_all(&pad).map_err(ZipError::from)?; header_end = writer.stream_position()?; @@ -1213,8 +1224,11 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // file name length writer.write_u16::(file.file_name.as_bytes().len() as u16)?; // extra field length - writer.write_u16::(zip64_extra_field_length + file.extra_field.len() as u16 - + file.central_extra_field.len() as u16)?; + writer.write_u16::( + zip64_extra_field_length + + file.extra_field.len() as u16 + + file.central_extra_field.len() as u16, + )?; // file comment length writer.write_u16::(0)?; // disk number start @@ -1253,7 +1267,11 @@ fn validate_extra_data(header_id: u16, data: &[u8]) -> ZipResult<()> { #[cfg(not(feature = "unreserved"))] { - if header_id <= 31 || EXTRA_FIELD_MAPPING.iter().any(|&mapped| mapped == header_id) { + if header_id <= 31 + || EXTRA_FIELD_MAPPING + .iter() + .any(|&mapped| mapped == header_id) + { return Err(ZipError::Io(io::Error::new( io::ErrorKind::Other, format!( diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 574e8fdc..70e59f61 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -119,7 +119,8 @@ fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, sha options.add_extra_data(0xbeef, EXTRA_DATA, false).unwrap(); - zip.start_file("test_with_extra_data/🐢.txt", options).unwrap(); + zip.start_file("test_with_extra_data/🐢.txt", options) + .unwrap(); zip.write_all(b"Hello, World! Again.\n").unwrap(); zip.finish().unwrap(); From d53d2d61b6c097c6637d78beb7d3f72c260acaf2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:04:17 -0700 Subject: [PATCH 176/281] Bug fix in Arbitrary impl --- src/write.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/write.rs b/src/write.rs index 26667882..b1e62b3a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -142,14 +142,14 @@ pub struct FileOptions { impl arbitrary::Arbitrary for FileOptions { fn arbitrary(u: &mut Unstructured) -> arbitrary::Result { let mut options = FileOptions { - compression_method: CompressionMethod::arbitrary(&mut u), - compression_level: Option::::arbitrary(&mut u), - last_modified_time: DateTime::arbitrary(&mut u), - permissions: Option::::arbitrary(&mut u), - large_file: bool::arbitrary(&mut u), - encrypt_with: Option::::arbitrary(&mut u), - extra_data: Vec::with_capacity(u16::MAX as usize), - central_extra_data: Vec::with_capacity(u16::MAX as usize), + compression_method: CompressionMethod::arbitrary(&mut u)?, + compression_level: Option::::arbitrary(&mut u)?, + last_modified_time: DateTime::arbitrary(&mut u)?, + permissions: Option::::arbitrary(&mut u)?, + large_file: bool::arbitrary(&mut u)?, + encrypt_with: Option::::arbitrary(&mut u)?, + extra_data: Vec::with_capacity(u16::MAX as usize)?, + central_extra_data: Vec::with_capacity(u16::MAX as usize)?, }; #[derive(arbitrary::Arbitrary)] struct ExtraDataField { @@ -157,12 +157,12 @@ impl arbitrary::Arbitrary for FileOptions { data: Vec, central_only: bool, } - let extra_data = Vec::::arbitrary(&mut u); + let extra_data = Vec::::arbitrary(&mut u)?; for field in extra_data { let _ = options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only); } - options + Ok(options) } } From e86efb24b0d412164e3ceff161cfccd262bbcf08 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:05:32 -0700 Subject: [PATCH 177/281] Refactor: don't need to return Ok if add_extra_data failed --- src/write.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index b1e62b3a..c5020058 100644 --- a/src/write.rs +++ b/src/write.rs @@ -159,8 +159,7 @@ impl arbitrary::Arbitrary for FileOptions { } let extra_data = Vec::::arbitrary(&mut u)?; for field in extra_data { - let _ = - options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only); + options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only)?; } Ok(options) } From 15c7958df2f941b6b8f1b6fd45d47e3f69f93a4c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:10:34 -0700 Subject: [PATCH 178/281] Bug fix --- src/write.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index c5020058..4ca900b8 100644 --- a/src/write.rs +++ b/src/write.rs @@ -148,8 +148,9 @@ impl arbitrary::Arbitrary for FileOptions { permissions: Option::::arbitrary(&mut u)?, large_file: bool::arbitrary(&mut u)?, encrypt_with: Option::::arbitrary(&mut u)?, - extra_data: Vec::with_capacity(u16::MAX as usize)?, - central_extra_data: Vec::with_capacity(u16::MAX as usize)?, + extra_data: Vec::with_capacity(u16::MAX as usize), + central_extra_data: Vec::with_capacity(u16::MAX as usize), + alignment: u8::arbitrary(&mut u)? as u16 + 1, }; #[derive(arbitrary::Arbitrary)] struct ExtraDataField { From c79f734a11610a7539a715131acf5a95461164b7 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:11:43 -0700 Subject: [PATCH 179/281] Bug fix --- src/write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 4ca900b8..e257858d 100644 --- a/src/write.rs +++ b/src/write.rs @@ -160,7 +160,8 @@ impl arbitrary::Arbitrary for FileOptions { } let extra_data = Vec::::arbitrary(&mut u)?; for field in extra_data { - options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only)?; + options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only) + .map_err(|_| arbitrary::Error::IncorrectFormat); } Ok(options) } From 6e4d60e0ca32b840449c87cadd29710835530330 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:16:58 -0700 Subject: [PATCH 180/281] Bug fixes for Arbitrary impl --- src/write.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/write.rs b/src/write.rs index e257858d..25434bb5 100644 --- a/src/write.rs +++ b/src/write.rs @@ -139,18 +139,18 @@ pub struct FileOptions { } #[cfg(fuzzing)] -impl arbitrary::Arbitrary for FileOptions { - fn arbitrary(u: &mut Unstructured) -> arbitrary::Result { +impl arbitrary::Arbitrary<'_> for FileOptions { + fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { let mut options = FileOptions { - compression_method: CompressionMethod::arbitrary(&mut u)?, - compression_level: Option::::arbitrary(&mut u)?, - last_modified_time: DateTime::arbitrary(&mut u)?, - permissions: Option::::arbitrary(&mut u)?, - large_file: bool::arbitrary(&mut u)?, - encrypt_with: Option::::arbitrary(&mut u)?, + compression_method: CompressionMethod::arbitrary(u)?, + compression_level: Option::::arbitrary(u)?, + last_modified_time: DateTime::arbitrary(u)?, + permissions: Option::::arbitrary(u)?, + large_file: bool::arbitrary(u)?, + encrypt_with: Option::::arbitrary(u)?, extra_data: Vec::with_capacity(u16::MAX as usize), central_extra_data: Vec::with_capacity(u16::MAX as usize), - alignment: u8::arbitrary(&mut u)? as u16 + 1, + alignment: u8::arbitrary(u)? as u16 + 1, }; #[derive(arbitrary::Arbitrary)] struct ExtraDataField { @@ -158,10 +158,11 @@ impl arbitrary::Arbitrary for FileOptions { data: Vec, central_only: bool, } - let extra_data = Vec::::arbitrary(&mut u)?; + let extra_data = Vec::::arbitrary(u)?; for field in extra_data { - options.add_extra_data(field.header_id, field.data.as_slice(), field.central_only) - .map_err(|_| arbitrary::Error::IncorrectFormat); + options + .add_extra_data(field.header_id, field.data.as_slice(), field.central_only) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; } Ok(options) } From a2ed77abd3b22be0e1a8f6148a15535d1f3eab71 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:29:00 -0700 Subject: [PATCH 181/281] Bug fixes --- fuzz/fuzz_targets/fuzz_write.rs | 21 ++++++++++++--------- src/write.rs | 1 + 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index ff495ffa..ba8da3cf 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -49,8 +49,9 @@ impl FileOperation { } fn do_operation(writer: &mut RefCell>, - operation: &FileOperation) -> Result<(), Box> + operation: FileOperation) -> Result<(), Box> where T: Read + Write + Seek { + let should_reopen = operation.should_reopen(); match operation { FileOperation::Write {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { @@ -61,16 +62,18 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().write_all(chunk.as_slice())?; } } - FileOperation::ShallowCopy {base, new_name, .. } => { - do_operation(writer, base)?; - writer.borrow_mut().shallow_copy_file(&base.get_name(), new_name)?; + FileOperation::ShallowCopy {base, ref new_name, .. } => { + let base_name = base.get_name(); + do_operation(writer, *base)?; + writer.borrow_mut().shallow_copy_file(&base_name, new_name)?; } - FileOperation::DeepCopy {base, new_name, .. } => { - do_operation(writer, base)?; - writer.borrow_mut().deep_copy_file(&base.get_name(), new_name)?; + FileOperation::DeepCopy {base, ref new_name, .. } => { + let base_name = base.get_name(); + do_operation(writer, *base)?; + writer.borrow_mut().deep_copy_file(&base_name, new_name)?; } } - if operation.should_reopen() { + if should_reopen { let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); *writer = new_writer.into(); } @@ -80,7 +83,7 @@ fn do_operation(writer: &mut RefCell>, fuzz_target!(|data: Vec| { let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()))); for operation in data { - let _ = do_operation(&mut writer, &operation); + let _ = do_operation(&mut writer, operation); } let _ = zip_next::ZipArchive::new(writer.borrow_mut().finish().unwrap()); }); \ No newline at end of file diff --git a/src/write.rs b/src/write.rs index 25434bb5..478cba54 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1247,6 +1247,7 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) writer.write_all(&zip64_extra_field[..zip64_extra_field_length as usize])?; // extra field writer.write_all(&file.extra_field)?; + writer.write_all(&file.central_extra_field)?; // file comment // From ba4d6548edeb7c79c5e8e7939997c618b6477901 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:40:23 -0700 Subject: [PATCH 182/281] Refactor FileOptions generation using arbitrary_loop --- src/write.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/write.rs b/src/write.rs index 478cba54..c129c919 100644 --- a/src/write.rs +++ b/src/write.rs @@ -152,18 +152,16 @@ impl arbitrary::Arbitrary<'_> for FileOptions { central_extra_data: Vec::with_capacity(u16::MAX as usize), alignment: u8::arbitrary(u)? as u16 + 1, }; - #[derive(arbitrary::Arbitrary)] - struct ExtraDataField { - header_id: u16, - data: Vec, - central_only: bool, - } - let extra_data = Vec::::arbitrary(u)?; - for field in extra_data { + u.arbitrary_loop(None, Some(16383), || { options - .add_extra_data(field.header_id, field.data.as_slice(), field.central_only) + .add_extra_data( + u16::arbitrary(u)?, + Vec::::arbitrary(u)?, + bool::arbitrary(u)?, + ) .map_err(|_| arbitrary::Error::IncorrectFormat)?; - } + Ok(core::ops::ControlFlow::Continue(())) + }); Ok(options) } } From 0ae9dab6abb782340294533d10df3cea77130e49 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:44:46 -0700 Subject: [PATCH 183/281] Bug fixes --- src/write.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/write.rs b/src/write.rs index c129c919..6e2af4c9 100644 --- a/src/write.rs +++ b/src/write.rs @@ -152,16 +152,16 @@ impl arbitrary::Arbitrary<'_> for FileOptions { central_extra_data: Vec::with_capacity(u16::MAX as usize), alignment: u8::arbitrary(u)? as u16 + 1, }; - u.arbitrary_loop(None, Some(16383), || { + u.arbitrary_loop(None, Some(16383), |u| { options .add_extra_data( u16::arbitrary(u)?, - Vec::::arbitrary(u)?, + &Vec::::arbitrary(u)?, bool::arbitrary(u)?, ) .map_err(|_| arbitrary::Error::IncorrectFormat)?; Ok(core::ops::ControlFlow::Continue(())) - }); + })?; Ok(options) } } From bc86898cdd6b0f652ed30bf22072803e6b6ed229 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:47:47 -0700 Subject: [PATCH 184/281] Strengthen fuzzing: allow very large alignments --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 6e2af4c9..64a1f352 100644 --- a/src/write.rs +++ b/src/write.rs @@ -150,7 +150,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { encrypt_with: Option::::arbitrary(u)?, extra_data: Vec::with_capacity(u16::MAX as usize), central_extra_data: Vec::with_capacity(u16::MAX as usize), - alignment: u8::arbitrary(u)? as u16 + 1, + alignment: u16::arbitrary(u)?, }; u.arbitrary_loop(None, Some(16383), |u| { options From 65535ffc48f77d9c03803d0867927ddc16115570 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:50:39 -0700 Subject: [PATCH 185/281] Eliminate a magic number --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 64a1f352..98449326 100644 --- a/src/write.rs +++ b/src/write.rs @@ -152,7 +152,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { central_extra_data: Vec::with_capacity(u16::MAX as usize), alignment: u16::arbitrary(u)?, }; - u.arbitrary_loop(None, Some(16383), |u| { + u.arbitrary_loop(None, Some((u16::MAX / 4) as u32), |u| { options .add_extra_data( u16::arbitrary(u)?, From 3c0ccf48ff4a6daad2d1f3406f17d6b6ea54a143 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 14:55:51 -0700 Subject: [PATCH 186/281] Explicitly allow zero loop iterations --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 98449326..b64cc3cf 100644 --- a/src/write.rs +++ b/src/write.rs @@ -152,7 +152,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { central_extra_data: Vec::with_capacity(u16::MAX as usize), alignment: u16::arbitrary(u)?, }; - u.arbitrary_loop(None, Some((u16::MAX / 4) as u32), |u| { + u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { options .add_extra_data( u16::arbitrary(u)?, From e89fa957213855c1d025dec9f3e3bfc50178d2f9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 15:06:22 -0700 Subject: [PATCH 187/281] Copy most attributes in deep_copy_file, and document the one we can't --- src/write.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/write.rs b/src/write.rs index b64cc3cf..2dc7fe37 100644 --- a/src/write.rs +++ b/src/write.rs @@ -393,7 +393,7 @@ impl ZipWriter { impl ZipWriter { /// Adds another copy of a file already in this archive. This will produce a larger but more - /// widely-compatible archive compared to [shallow_copy_file]. + /// widely-compatible archive compared to [shallow_copy_file]. Does not copy alignment. pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { self.finish_file()?; let write_position = self.inner.get_plain().stream_position()?; @@ -403,10 +403,17 @@ impl ZipWriter { let compressed_size = src_data.compressed_size; debug_assert!(compressed_size <= write_position - data_start); let uncompressed_size = src_data.uncompressed_size; - let mut options = FileOptions::default() - .large_file(compressed_size.max(uncompressed_size) > spec::ZIP64_BYTES_THR) - .last_modified_time(src_data.last_modified_time) - .compression_method(src_data.compression_method); + let mut options = FileOptions { + compression_method: src_data.compression_method, + compression_level: src_data.compression_level, + last_modified_time: src_data.last_modified_time, + permissions: src_data.unix_mode(), + large_file: src_data.large_file, + encrypt_with: None, + extra_data: src_data.extra_field.clone(), + central_extra_data: src_data.central_extra_field.clone(), + alignment: 1, + }; if let Some(perms) = src_data.unix_mode() { options = options.unix_permissions(perms); } From ff4dee28d78127dc3b28f9e75b7787bacc1936f8 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 15:38:09 -0700 Subject: [PATCH 188/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 32 +++++++++++++++++++++++++++----- src/types.rs | 15 ++++++++++++++- src/write.rs | 30 +++++++++++++++++++----------- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index ba8da3cf..261c7601 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -4,6 +4,7 @@ use std::cell::RefCell; use libfuzzer_sys::fuzz_target; use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; +use std::path::{PathBuf}; #[derive(Arbitrary,Debug)] pub struct File { @@ -13,11 +14,22 @@ pub struct File { #[derive(Arbitrary,Debug)] pub enum FileOperation { - Write { + WriteNormalFile { file: File, options: zip_next::write::FileOptions, reopen: bool, }, + WriteDirectory { + name: String, + options: zip_next::write::FileOptions, + reopen: bool, + }, + WriteSymlink { + name: String, + target: Box, + options: zip_next::write::FileOptions, + reopen: bool, + }, ShallowCopy { base: Box, new_name: String, @@ -33,7 +45,9 @@ pub enum FileOperation { impl FileOperation { pub fn get_name(&self) -> String { match self { - FileOperation::Write {file, ..} => &file.name, + FileOperation::WriteNormalFile {file, ..} => &file.name, + FileOperation::WriteDirectory {name, ..} => name, + FileOperation::WriteSymlink {name, ..} => name, FileOperation::ShallowCopy {new_name, ..} => new_name, FileOperation::DeepCopy {new_name, ..} => new_name }.to_owned() @@ -41,9 +55,11 @@ impl FileOperation { pub fn should_reopen(&self) -> bool { match self { - FileOperation::Write {reopen, ..} => *reopen, + FileOperation::WriteNormalFile {reopen, ..} => *reopen, FileOperation::ShallowCopy {reopen, ..} => *reopen, - FileOperation::DeepCopy {reopen, ..} => *reopen + FileOperation::DeepCopy {reopen, ..} => *reopen, + FileOperation::WriteDirectory {reopen, ..} => *reopen, + FileOperation::WriteSymlink {reopen, ..} => *reopen, } } } @@ -53,7 +69,7 @@ fn do_operation(writer: &mut RefCell>, where T: Read + Write + Seek { let should_reopen = operation.should_reopen(); match operation { - FileOperation::Write {file, mut options, ..} => { + FileOperation::WriteNormalFile {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } @@ -62,6 +78,12 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().write_all(chunk.as_slice())?; } } + FileOperation::WriteDirectory {name, options, ..} => { + writer.borrow_mut().add_directory(name, options)?; + } + FileOperation::WriteSymlink {name, target, options, ..} => { + writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; + } FileOperation::ShallowCopy {base, ref new_name, .. } => { let base_name = base.get_name(); do_operation(writer, *base)?; diff --git a/src/types.rs b/src/types.rs index 52c0b6ea..d6d1f14f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -92,7 +92,6 @@ impl System { /// Modern zip files store more precise timestamps, which are ignored by [`crate::read::ZipArchive`], /// so keep in mind that these timestamps are unreliable. [We're working on this](https://github.com/zip-rs/zip/issues/156#issuecomment-652981904). #[derive(Debug, Clone, Copy)] -#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] pub struct DateTime { year: u16, month: u8, @@ -102,6 +101,20 @@ pub struct DateTime { second: u8, } +#[cfg(fuzzing)] +impl arbitrary::Arbitrary<'_> for DateTime { + fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { + Ok(DateTime { + year: u.int_in_range(1980..=2107)?, + month: u.int_in_range(1..=12)?, + day: u.int_in_range(1..=31)?, + hour: u.int_in_range(0..=23)?, + minute: u.int_in_range(0..=59)?, + second: u.int_in_range(0..=60)?, + }) + } +} + impl Default for DateTime { /// Constructs an 'default' datetime of 1980-01-01 00:00:00 fn default() -> DateTime { diff --git a/src/write.rs b/src/write.rs index 2dc7fe37..1bdfb6be 100644 --- a/src/write.rs +++ b/src/write.rs @@ -646,14 +646,7 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; - match mem::replace(&mut self.inner, Closed) { - Storer(MaybeEncrypted::Encrypted(writer)) => { - let crc32 = self.stats.hasher.clone().finalize(); - self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) - } - Storer(w) => self.inner = Storer(w), - _ => unreachable!(), - } + self.switch_to_non_encrypting_writer()?; let writer = self.inner.get_plain(); if !self.writing_raw { @@ -676,18 +669,32 @@ impl ZipWriter { Ok(()) } + fn switch_to_non_encrypting_writer(&mut self) -> Result<(), ZipError> { + match mem::replace(&mut self.inner, Closed) { + Storer(MaybeEncrypted::Encrypted(writer)) => { + let crc32 = self.stats.hasher.clone().finalize(); + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish(crc32)?)) + } + Storer(MaybeEncrypted::Unencrypted(w)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(w)) + } + _ => unreachable!(), + } + Ok(()) + } + /// Removes the file currently being written from the archive if there is one, or else removes /// the file most recently written. pub fn abort_file(&mut self) -> ZipResult<()> { let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; self.files_by_name.remove(&last_file.file_name); - self.inner - .get_plain() - .seek(SeekFrom::Start(last_file.header_start))?; let make_plain_writer = self .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; self.writing_to_file = false; Ok(()) } @@ -843,6 +850,7 @@ impl ZipWriter { self.start_entry(name_with_slash, options, None)?; self.writing_to_file = false; + self.switch_to_non_encrypting_writer()?; Ok(()) } From 089f194fd6609668b64a8f9e026d10998d32c8e6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 15:49:54 -0700 Subject: [PATCH 189/281] Simplify FileOperation by splitting into BasicFileOperation and bool reopen --- fuzz/fuzz_targets/fuzz_write.rs | 50 +++++++++++++-------------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 261c7601..7748f5a7 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -13,63 +13,53 @@ pub struct File { } #[derive(Arbitrary,Debug)] -pub enum FileOperation { +pub enum BasicFileOperation { WriteNormalFile { file: File, options: zip_next::write::FileOptions, - reopen: bool, }, WriteDirectory { name: String, options: zip_next::write::FileOptions, - reopen: bool, }, WriteSymlink { name: String, target: Box, options: zip_next::write::FileOptions, - reopen: bool, }, ShallowCopy { base: Box, new_name: String, - reopen: bool, }, DeepCopy { base: Box, new_name: String, - reopen: bool, } } +#[derive(Arbitrary,Debug)] +pub struct FileOperation { + basic: BasicFileOperation, + reopen: bool, +} + impl FileOperation { pub fn get_name(&self) -> String { - match self { - FileOperation::WriteNormalFile {file, ..} => &file.name, - FileOperation::WriteDirectory {name, ..} => name, - FileOperation::WriteSymlink {name, ..} => name, - FileOperation::ShallowCopy {new_name, ..} => new_name, - FileOperation::DeepCopy {new_name, ..} => new_name + match &self.basic { + BasicFileOperation::WriteNormalFile {file, ..} => &file.name, + BasicFileOperation::WriteDirectory {name, ..} => name, + BasicFileOperation::WriteSymlink {name, ..} => name, + BasicFileOperation::ShallowCopy {new_name, ..} => new_name, + BasicFileOperation::DeepCopy {new_name, ..} => new_name }.to_owned() } - - pub fn should_reopen(&self) -> bool { - match self { - FileOperation::WriteNormalFile {reopen, ..} => *reopen, - FileOperation::ShallowCopy {reopen, ..} => *reopen, - FileOperation::DeepCopy {reopen, ..} => *reopen, - FileOperation::WriteDirectory {reopen, ..} => *reopen, - FileOperation::WriteSymlink {reopen, ..} => *reopen, - } - } } fn do_operation(writer: &mut RefCell>, operation: FileOperation) -> Result<(), Box> where T: Read + Write + Seek { - let should_reopen = operation.should_reopen(); - match operation { - FileOperation::WriteNormalFile {file, mut options, ..} => { + match operation.basic { + BasicFileOperation::WriteNormalFile {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } @@ -78,24 +68,24 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().write_all(chunk.as_slice())?; } } - FileOperation::WriteDirectory {name, options, ..} => { + BasicFileOperation::WriteDirectory {name, options, ..} => { writer.borrow_mut().add_directory(name, options)?; } - FileOperation::WriteSymlink {name, target, options, ..} => { + BasicFileOperation::WriteSymlink {name, target, options, ..} => { writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; } - FileOperation::ShallowCopy {base, ref new_name, .. } => { + BasicFileOperation::ShallowCopy {base, ref new_name, .. } => { let base_name = base.get_name(); do_operation(writer, *base)?; writer.borrow_mut().shallow_copy_file(&base_name, new_name)?; } - FileOperation::DeepCopy {base, ref new_name, .. } => { + BasicFileOperation::DeepCopy {base, ref new_name, .. } => { let base_name = base.get_name(); do_operation(writer, *base)?; writer.borrow_mut().deep_copy_file(&base_name, new_name)?; } } - if should_reopen { + if operation.reopen { let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); *writer = new_writer.into(); } From f7acf6fe9c925fc081693203c16fd9af8df1f4d2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 15:58:45 -0700 Subject: [PATCH 190/281] Refactor: simplify FileOperation by factoring out shared fields --- fuzz/fuzz_targets/fuzz_write.rs | 48 ++++++++++----------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 7748f5a7..40174b58 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -18,46 +18,26 @@ pub enum BasicFileOperation { file: File, options: zip_next::write::FileOptions, }, - WriteDirectory { - name: String, - options: zip_next::write::FileOptions, - }, - WriteSymlink { - name: String, + WriteDirectory(zip_next::write::FileOptions), + WriteSymlinkWithTarget { target: Box, options: zip_next::write::FileOptions, }, - ShallowCopy { - base: Box, - new_name: String, - }, - DeepCopy { - base: Box, - new_name: String, - } + ShallowCopy(Box), + DeepCopy(Box), } #[derive(Arbitrary,Debug)] pub struct FileOperation { basic: BasicFileOperation, + name: String, reopen: bool, } -impl FileOperation { - pub fn get_name(&self) -> String { - match &self.basic { - BasicFileOperation::WriteNormalFile {file, ..} => &file.name, - BasicFileOperation::WriteDirectory {name, ..} => name, - BasicFileOperation::WriteSymlink {name, ..} => name, - BasicFileOperation::ShallowCopy {new_name, ..} => new_name, - BasicFileOperation::DeepCopy {new_name, ..} => new_name - }.to_owned() - } -} - fn do_operation(writer: &mut RefCell>, operation: FileOperation) -> Result<(), Box> where T: Read + Write + Seek { + let name = operation.name; match operation.basic { BasicFileOperation::WriteNormalFile {file, mut options, ..} => { if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { @@ -68,21 +48,21 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().write_all(chunk.as_slice())?; } } - BasicFileOperation::WriteDirectory {name, options, ..} => { + BasicFileOperation::WriteDirectory(options) => { writer.borrow_mut().add_directory(name, options)?; } - BasicFileOperation::WriteSymlink {name, target, options, ..} => { + BasicFileOperation::WriteSymlinkWithTarget {target, options} => { writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; } - BasicFileOperation::ShallowCopy {base, ref new_name, .. } => { - let base_name = base.get_name(); + BasicFileOperation::ShallowCopy(base) => { + let base_name = base.name.to_owned(); do_operation(writer, *base)?; - writer.borrow_mut().shallow_copy_file(&base_name, new_name)?; + writer.borrow_mut().shallow_copy_file(&base_name, &name)?; } - BasicFileOperation::DeepCopy {base, ref new_name, .. } => { - let base_name = base.get_name(); + BasicFileOperation::DeepCopy(base) => { + let base_name = base.name.to_owned(); do_operation(writer, *base)?; - writer.borrow_mut().deep_copy_file(&base_name, new_name)?; + writer.borrow_mut().deep_copy_file(&base_name, &name)?; } } if operation.reopen { From e35a4ee9381098d74dd923dce6ddfeb2923b1759 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 13 May 2023 16:07:35 -0700 Subject: [PATCH 191/281] Update CHANGELOG --- CHANGELOG.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8843ec..366522d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,4 +123,21 @@ ### Fixed - - Fixed a bug that occurs when ZIP64 magic bytes occur twice in a filename or across two filenames. \ No newline at end of file + - Fixed a bug that occurs when ZIP64 magic bytes occur twice in a filename or across two filenames. + +## [0.8.0] + +### Deleted + + - Methods `start_file_aligned`, `start_file_with_extra_data`, `end_local_start_central_extra_data` and + `end_extra_data` (see below). + +### Changed + + - Alignment and extra-data fields are now attributes of [`zip_next::unstable::write::FileOptions`], allowing them to be + specified for `add_directory` and `add_symlink`. + - Extra-data fields no longer have to be formatted by the caller. + +### Fixed + + - Fixes a rare bug where the size of the extra-data field could overflow when `large_file` was set. From 21ff795071aae70847a0edeb045c1f9a37e61dc3 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 08:28:52 -0700 Subject: [PATCH 192/281] Additional sanity checks on ZIP64 footer --- src/read.rs | 12 ++++++++++++ src/write.rs | 14 ++++++++++++++ tests/data/zip64_magic_in_filename_5.zip | Bin 0 -> 562 bytes 3 files changed, 26 insertions(+) create mode 100644 tests/data/zip64_magic_in_filename_5.zip diff --git a/src/read.rs b/src/read.rs index 7efd1f9a..1c2faca2 100644 --- a/src/read.rs +++ b/src/read.rs @@ -356,6 +356,17 @@ impl ZipArchive { "Invalid central directory size or offset", )); } + if footer64.number_of_files_on_this_disk > footer64.number_of_files { + return Err(ZipError::InvalidArchive( + "ZIP64 footer indicates more files on this disk than in the whole archive" + )); + } + if footer64.version_needed_to_extract > footer64.version_made_by { + return Err(ZipError::InvalidArchive( + "ZIP64 footer indicates a new version is needed to extract this archive than the \ + version that wrote it" + )); + } let supported = if (footer64.disk_number != footer64.disk_with_central_directory) || (!footer.record_too_small() @@ -1266,6 +1277,7 @@ mod test { include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec(), include_bytes!("../tests/data/zip64_magic_in_filename_4.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_5.zip").to_vec(), ]; // Although we don't allow adding files whose names contain the ZIP64 CDB-end or // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. diff --git a/src/write.rs b/src/write.rs index 1bdfb6be..98629dea 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1701,6 +1701,20 @@ mod test { println!("{:02x?}", zip.get_ref()); let _ = ZipArchive::new(zip).unwrap(); } + + #[test] + fn test_filename_looks_like_zip64_locator_4() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer.start_file("PK\u{6}\u{6}", FileOptions::default()).unwrap(); + writer.start_file("\0\0\0\0\0\0", FileOptions::default()).unwrap(); + writer.start_file("\0", FileOptions::default()).unwrap(); + writer.start_file("", FileOptions::default()).unwrap(); + writer.start_file("\0\0", FileOptions::default()).unwrap(); + writer.start_file("\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", FileOptions::default()).unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + } } #[cfg(not(feature = "unreserved"))] diff --git a/tests/data/zip64_magic_in_filename_5.zip b/tests/data/zip64_magic_in_filename_5.zip new file mode 100644 index 0000000000000000000000000000000000000000..bf6a95dbac040e2131ed1b112125c66bcb74cb33 GIT binary patch literal 562 zcmWIWW@Zs#U|`^2SY5d;49H+$0ueyK0>lB{Y;4Rx5| Date: Sun, 14 May 2023 08:31:37 -0700 Subject: [PATCH 193/281] Eliminate redundant name field in FileOperation for more efficient fuzzing --- fuzz/fuzz_targets/fuzz_write.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 40174b58..6290460a 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -6,16 +6,10 @@ use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; use std::path::{PathBuf}; -#[derive(Arbitrary,Debug)] -pub struct File { - pub name: String, - pub contents: Vec> -} - #[derive(Arbitrary,Debug)] pub enum BasicFileOperation { WriteNormalFile { - file: File, + contents: Vec>, options: zip_next::write::FileOptions, }, WriteDirectory(zip_next::write::FileOptions), @@ -39,12 +33,12 @@ fn do_operation(writer: &mut RefCell>, where T: Read + Write + Seek { let name = operation.name; match operation.basic { - BasicFileOperation::WriteNormalFile {file, mut options, ..} => { - if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { + BasicFileOperation::WriteNormalFile {contents, mut options, ..} => { + if contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } - writer.borrow_mut().start_file(file.name.to_owned(), options)?; - for chunk in &file.contents { + writer.borrow_mut().start_file(name, options)?; + for chunk in contents { writer.borrow_mut().write_all(chunk.as_slice())?; } } From 5a477c2b3e276ac410f74264419d8170c2d72822 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 08:41:28 -0700 Subject: [PATCH 194/281] Wrap extra data in Cow so FileOptions can be cloned faster --- src/write.rs | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/write.rs b/src/write.rs index 98629dea..f573f188 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1,5 +1,6 @@ //! Types for creating ZIP archives +use std::borrow::Cow; use crate::compression::CompressionMethod; use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, ZipFileReader}; use crate::result::{ZipError, ZipResult}; @@ -126,15 +127,15 @@ struct ZipRawValues { /// Metadata for a file to be written #[derive(Clone, Debug)] -pub struct FileOptions { +pub struct FileOptions<'a> { pub(crate) compression_method: CompressionMethod, pub(crate) compression_level: Option, pub(crate) last_modified_time: DateTime, pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, - extra_data: Vec, - central_extra_data: Vec, + extra_data: Cow<'a, Vec>, + central_extra_data: Cow<'a, Vec>, alignment: u16, } @@ -166,13 +167,13 @@ impl arbitrary::Arbitrary<'_> for FileOptions { } } -impl FileOptions { +impl <'a> FileOptions<'a> { /// Set the compression method for the new file /// /// The default is `CompressionMethod::Deflated`. If the deflate compression feature is /// disabled, `CompressionMethod::Stored` becomes the default. #[must_use] - pub fn compression_method(mut self, method: CompressionMethod) -> FileOptions { + pub fn compression_method(mut self, method: CompressionMethod) -> FileOptions<'a> { self.compression_method = method; self } @@ -187,7 +188,7 @@ impl FileOptions { /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 /// * others: only `None` is allowed #[must_use] - pub fn compression_level(mut self, level: Option) -> FileOptions { + pub fn compression_level(mut self, level: Option) -> FileOptions<'a> { self.compression_level = level; self } @@ -197,7 +198,7 @@ impl FileOptions { /// The default is the current timestamp if the 'time' feature is enabled, and 1980-01-01 /// otherwise #[must_use] - pub fn last_modified_time(mut self, mod_time: DateTime) -> FileOptions { + pub fn last_modified_time(mut self, mod_time: DateTime) -> FileOptions<'a> { self.last_modified_time = mod_time; self } @@ -212,7 +213,7 @@ impl FileOptions { /// higher file mode bits. So it cannot be used to denote an entry as a directory, /// symlink, or other special file type. #[must_use] - pub fn unix_permissions(mut self, mode: u32) -> FileOptions { + pub fn unix_permissions(mut self, mode: u32) -> FileOptions<'a> { self.permissions = Some(mode & 0o777); self } @@ -223,11 +224,11 @@ impl FileOptions { /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not /// exceed the limit, 20 B are wasted. The default is `false`. #[must_use] - pub fn large_file(mut self, large: bool) -> FileOptions { + pub fn large_file(mut self, large: bool) -> FileOptions<'a> { self.large_file = large; self } - pub(crate) fn with_deprecated_encryption(mut self, password: &[u8]) -> FileOptions { + pub(crate) fn with_deprecated_encryption(mut self, password: &[u8]) -> FileOptions<'a> { self.encrypt_with = Some(ZipCryptoKeys::derive(password)); self } @@ -260,14 +261,14 @@ impl FileOptions { /// Removes the extra data fields. #[must_use] - pub fn clear_extra_data(mut self) -> FileOptions { + pub fn clear_extra_data(mut self) -> FileOptions<'a> { self.extra_data.clear(); self.central_extra_data.clear(); self } } -impl Default for FileOptions { +impl <'a> Default for FileOptions<'a> { /// Construct a new FileOptions object fn default() -> Self { Self { @@ -291,8 +292,8 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, - extra_data: Vec::with_capacity(u16::MAX as usize), - central_extra_data: Vec::with_capacity(u16::MAX as usize), + extra_data: Cow::Owned(Vec::with_capacity(u16::MAX as usize)), + central_extra_data: Cow::Owned(Vec::with_capacity(u16::MAX as usize)), alignment: 1, } } @@ -410,8 +411,8 @@ impl ZipWriter { permissions: src_data.unix_mode(), large_file: src_data.large_file, encrypt_with: None, - extra_data: src_data.extra_field.clone(), - central_extra_data: src_data.central_extra_field.clone(), + extra_data: Cow::Borrowed(&src_data.extra_field), + central_extra_data: Cow::Borrowed(&src_data.central_extra_field), alignment: 1, }; if let Some(perms) = src_data.unix_mode() { @@ -519,8 +520,8 @@ impl ZipWriter { uncompressed_size: raw_values.uncompressed_size, file_name: name, file_name_raw: Vec::new(), // Never used for saving - extra_field: options.extra_data, - central_extra_field: options.central_extra_data, + extra_field: options.extra_data.to_vec(), + central_extra_field: options.central_extra_data.to_vec(), file_comment: String::new(), header_start, data_start: AtomicU64::new(0), @@ -1378,6 +1379,7 @@ fn path_to_string(path: &std::path::Path) -> String { #[cfg(test)] mod test { + use std::borrow::Cow; use super::{FileOptions, ZipWriter}; use crate::compression::CompressionMethod; use crate::types::DateTime; @@ -1503,8 +1505,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: vec![], - central_extra_data: vec![], + extra_data: Cow::Owned(vec![]), + central_extra_data: Cow::Owned(vec![]), alignment: 1, }; writer.start_file("mimetype", options).unwrap(); @@ -1543,8 +1545,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: vec![], - central_extra_data: vec![], + extra_data: Cow::Owned(vec![]), + central_extra_data: Cow::Owned(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1593,8 +1595,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: vec![], - central_extra_data: vec![], + extra_data: Cow::Owned(vec![]), + central_extra_data: Cow::Owned(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); From bb49f1c4143ae5eee96e5378146d31656c3acf1f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 08:55:50 -0700 Subject: [PATCH 195/281] Revert "Wrap extra data in Cow so FileOptions can be cloned faster" This reverts commit 5a477c2b3e276ac410f74264419d8170c2d72822. --- src/write.rs | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/write.rs b/src/write.rs index f573f188..98629dea 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1,6 +1,5 @@ //! Types for creating ZIP archives -use std::borrow::Cow; use crate::compression::CompressionMethod; use crate::read::{central_header_to_zip_file, find_content, ZipArchive, ZipFile, ZipFileReader}; use crate::result::{ZipError, ZipResult}; @@ -127,15 +126,15 @@ struct ZipRawValues { /// Metadata for a file to be written #[derive(Clone, Debug)] -pub struct FileOptions<'a> { +pub struct FileOptions { pub(crate) compression_method: CompressionMethod, pub(crate) compression_level: Option, pub(crate) last_modified_time: DateTime, pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, - extra_data: Cow<'a, Vec>, - central_extra_data: Cow<'a, Vec>, + extra_data: Vec, + central_extra_data: Vec, alignment: u16, } @@ -167,13 +166,13 @@ impl arbitrary::Arbitrary<'_> for FileOptions { } } -impl <'a> FileOptions<'a> { +impl FileOptions { /// Set the compression method for the new file /// /// The default is `CompressionMethod::Deflated`. If the deflate compression feature is /// disabled, `CompressionMethod::Stored` becomes the default. #[must_use] - pub fn compression_method(mut self, method: CompressionMethod) -> FileOptions<'a> { + pub fn compression_method(mut self, method: CompressionMethod) -> FileOptions { self.compression_method = method; self } @@ -188,7 +187,7 @@ impl <'a> FileOptions<'a> { /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 /// * others: only `None` is allowed #[must_use] - pub fn compression_level(mut self, level: Option) -> FileOptions<'a> { + pub fn compression_level(mut self, level: Option) -> FileOptions { self.compression_level = level; self } @@ -198,7 +197,7 @@ impl <'a> FileOptions<'a> { /// The default is the current timestamp if the 'time' feature is enabled, and 1980-01-01 /// otherwise #[must_use] - pub fn last_modified_time(mut self, mod_time: DateTime) -> FileOptions<'a> { + pub fn last_modified_time(mut self, mod_time: DateTime) -> FileOptions { self.last_modified_time = mod_time; self } @@ -213,7 +212,7 @@ impl <'a> FileOptions<'a> { /// higher file mode bits. So it cannot be used to denote an entry as a directory, /// symlink, or other special file type. #[must_use] - pub fn unix_permissions(mut self, mode: u32) -> FileOptions<'a> { + pub fn unix_permissions(mut self, mode: u32) -> FileOptions { self.permissions = Some(mode & 0o777); self } @@ -224,11 +223,11 @@ impl <'a> FileOptions<'a> { /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not /// exceed the limit, 20 B are wasted. The default is `false`. #[must_use] - pub fn large_file(mut self, large: bool) -> FileOptions<'a> { + pub fn large_file(mut self, large: bool) -> FileOptions { self.large_file = large; self } - pub(crate) fn with_deprecated_encryption(mut self, password: &[u8]) -> FileOptions<'a> { + pub(crate) fn with_deprecated_encryption(mut self, password: &[u8]) -> FileOptions { self.encrypt_with = Some(ZipCryptoKeys::derive(password)); self } @@ -261,14 +260,14 @@ impl <'a> FileOptions<'a> { /// Removes the extra data fields. #[must_use] - pub fn clear_extra_data(mut self) -> FileOptions<'a> { + pub fn clear_extra_data(mut self) -> FileOptions { self.extra_data.clear(); self.central_extra_data.clear(); self } } -impl <'a> Default for FileOptions<'a> { +impl Default for FileOptions { /// Construct a new FileOptions object fn default() -> Self { Self { @@ -292,8 +291,8 @@ impl <'a> Default for FileOptions<'a> { permissions: None, large_file: false, encrypt_with: None, - extra_data: Cow::Owned(Vec::with_capacity(u16::MAX as usize)), - central_extra_data: Cow::Owned(Vec::with_capacity(u16::MAX as usize)), + extra_data: Vec::with_capacity(u16::MAX as usize), + central_extra_data: Vec::with_capacity(u16::MAX as usize), alignment: 1, } } @@ -411,8 +410,8 @@ impl ZipWriter { permissions: src_data.unix_mode(), large_file: src_data.large_file, encrypt_with: None, - extra_data: Cow::Borrowed(&src_data.extra_field), - central_extra_data: Cow::Borrowed(&src_data.central_extra_field), + extra_data: src_data.extra_field.clone(), + central_extra_data: src_data.central_extra_field.clone(), alignment: 1, }; if let Some(perms) = src_data.unix_mode() { @@ -520,8 +519,8 @@ impl ZipWriter { uncompressed_size: raw_values.uncompressed_size, file_name: name, file_name_raw: Vec::new(), // Never used for saving - extra_field: options.extra_data.to_vec(), - central_extra_field: options.central_extra_data.to_vec(), + extra_field: options.extra_data, + central_extra_field: options.central_extra_data, file_comment: String::new(), header_start, data_start: AtomicU64::new(0), @@ -1379,7 +1378,6 @@ fn path_to_string(path: &std::path::Path) -> String { #[cfg(test)] mod test { - use std::borrow::Cow; use super::{FileOptions, ZipWriter}; use crate::compression::CompressionMethod; use crate::types::DateTime; @@ -1505,8 +1503,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Cow::Owned(vec![]), - central_extra_data: Cow::Owned(vec![]), + extra_data: vec![], + central_extra_data: vec![], alignment: 1, }; writer.start_file("mimetype", options).unwrap(); @@ -1545,8 +1543,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Cow::Owned(vec![]), - central_extra_data: Cow::Owned(vec![]), + extra_data: vec![], + central_extra_data: vec![], alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1595,8 +1593,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Cow::Owned(vec![]), - central_extra_data: Cow::Owned(vec![]), + extra_data: vec![], + central_extra_data: vec![], alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); From dbf39339decb9b36560227abe7c1f632f1f9a318 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 09:20:15 -0700 Subject: [PATCH 196/281] Wrap extra data in Rc so FileOptions and ZipFileData can be cloned faster --- src/read.rs | 15 ++++++------- src/types.rs | 9 ++++---- src/write.rs | 59 ++++++++++++++++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 29 deletions(-) diff --git a/src/read.rs b/src/read.rs index 1c2faca2..051b11b6 100644 --- a/src/read.rs +++ b/src/read.rs @@ -14,6 +14,7 @@ use std::borrow::Cow; use std::collections::HashMap; use std::io::{self, prelude::*}; use std::path::Path; +use std::rc::Rc; use std::sync::Arc; #[cfg(any( @@ -358,13 +359,13 @@ impl ZipArchive { } if footer64.number_of_files_on_this_disk > footer64.number_of_files { return Err(ZipError::InvalidArchive( - "ZIP64 footer indicates more files on this disk than in the whole archive" + "ZIP64 footer indicates more files on this disk than in the whole archive", )); } if footer64.version_needed_to_extract > footer64.version_made_by { return Err(ZipError::InvalidArchive( "ZIP64 footer indicates a new version is needed to extract this archive than the \ - version that wrote it" + version that wrote it", )); } @@ -737,8 +738,8 @@ fn central_header_to_zip_file_inner( uncompressed_size: uncompressed_size as u64, file_name, file_name_raw, - extra_field, - central_extra_field: vec![], + extra_field: Rc::new(extra_field), + central_extra_field: Rc::new(vec![]), file_comment, header_start: offset, central_header_start, @@ -770,7 +771,7 @@ fn central_header_to_zip_file_inner( } fn parse_extra_field(file: &mut ZipFileData) -> ZipResult<()> { - let mut reader = io::Cursor::new(&file.extra_field); + let mut reader = io::Cursor::new(file.extra_field.as_ref()); while (reader.position() as usize) < file.extra_field.len() { let kind = reader.read_u16::()?; @@ -1098,8 +1099,8 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult, /// Extra field usually used for storage expansion - pub extra_field: Vec, + pub extra_field: Rc>, /// Extra field only written to central directory - pub central_extra_field: Vec, + pub central_extra_field: Rc>, /// File comment pub file_comment: String, /// Specifies where the local header of the file starts @@ -530,8 +531,8 @@ mod test { uncompressed_size: 0, file_name: file_name.clone(), file_name_raw: file_name.into_bytes(), - extra_field: Vec::new(), - central_extra_field: vec![], + extra_field: Rc::new(vec![]), + central_extra_field: Rc::new(vec![]), file_comment: String::new(), header_start: 0, data_start: AtomicU64::new(0), diff --git a/src/write.rs b/src/write.rs index 98629dea..27b70150 100644 --- a/src/write.rs +++ b/src/write.rs @@ -14,6 +14,7 @@ use std::io; use std::io::prelude::*; use std::io::{BufReader, SeekFrom}; use std::mem; +use std::rc::Rc; #[cfg(any( feature = "deflate", @@ -133,8 +134,8 @@ pub struct FileOptions { pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, - extra_data: Vec, - central_extra_data: Vec, + extra_data: Rc>, + central_extra_data: Rc>, alignment: u16, } @@ -251,9 +252,17 @@ impl FileOptions { } else { &mut self.extra_data }; - field.write_u16::(header_id)?; - field.write_u16::(data.len() as u16)?; - field.write_all(data)?; + let vec = Rc::get_mut(field); + let vec = match vec { + Some(exclusive) => exclusive, + None => { + *field = Rc::new(field.to_vec()); + Rc::get_mut(field).unwrap() + } + }; + vec.write_u16::(header_id)?; + vec.write_u16::(data.len() as u16)?; + vec.write_all(data)?; Ok(()) } } @@ -261,8 +270,12 @@ impl FileOptions { /// Removes the extra data fields. #[must_use] pub fn clear_extra_data(mut self) -> FileOptions { - self.extra_data.clear(); - self.central_extra_data.clear(); + if self.extra_data.len() > 0 { + self.extra_data = Rc::new(vec![]); + } + if self.central_extra_data.len() > 0 { + self.central_extra_data = Rc::new(vec![]); + } self } } @@ -291,8 +304,8 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, - extra_data: Vec::with_capacity(u16::MAX as usize), - central_extra_data: Vec::with_capacity(u16::MAX as usize), + extra_data: Rc::new(vec![]), + central_extra_data: Rc::new(vec![]), alignment: 1, } } @@ -1384,6 +1397,7 @@ mod test { use crate::ZipArchive; use std::io; use std::io::{Read, Write}; + use std::rc::Rc; #[test] fn write_empty_zip() { @@ -1503,8 +1517,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: vec![], - central_extra_data: vec![], + extra_data: Rc::new(vec![]), + central_extra_data: Rc::new(vec![]), alignment: 1, }; writer.start_file("mimetype", options).unwrap(); @@ -1543,8 +1557,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: vec![], - central_extra_data: vec![], + extra_data: Rc::new(vec![]), + central_extra_data: Rc::new(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1593,8 +1607,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: vec![], - central_extra_data: vec![], + extra_data: Rc::new(vec![]), + central_extra_data: Rc::new(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1705,12 +1719,21 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_4() { let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); - writer.start_file("PK\u{6}\u{6}", FileOptions::default()).unwrap(); - writer.start_file("\0\0\0\0\0\0", FileOptions::default()).unwrap(); + writer + .start_file("PK\u{6}\u{6}", FileOptions::default()) + .unwrap(); + writer + .start_file("\0\0\0\0\0\0", FileOptions::default()) + .unwrap(); writer.start_file("\0", FileOptions::default()).unwrap(); writer.start_file("", FileOptions::default()).unwrap(); writer.start_file("\0\0", FileOptions::default()).unwrap(); - writer.start_file("\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", FileOptions::default()).unwrap(); + writer + .start_file( + "\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) + .unwrap(); let zip = writer.finish().unwrap(); println!("{:02x?}", zip.get_ref()); let _ = ZipArchive::new(zip).unwrap(); From 56f9ee9ab0a688ccd95367b8b242632732bf4db8 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 09:25:14 -0700 Subject: [PATCH 197/281] Bug fix --- src/write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 27b70150..ed332e59 100644 --- a/src/write.rs +++ b/src/write.rs @@ -149,8 +149,8 @@ impl arbitrary::Arbitrary<'_> for FileOptions { permissions: Option::::arbitrary(u)?, large_file: bool::arbitrary(u)?, encrypt_with: Option::::arbitrary(u)?, - extra_data: Vec::with_capacity(u16::MAX as usize), - central_extra_data: Vec::with_capacity(u16::MAX as usize), + extra_data: Rc::new(vec![]), + central_extra_data: Rc::new(vec![]), alignment: u16::arbitrary(u)?, }; u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { From 170e5c6f58b9f340a8f4ed5ff677361659fd921d Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 09:26:33 -0700 Subject: [PATCH 198/281] Refactor: reserve space before adding extra data --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index ed332e59..7eaa8d27 100644 --- a/src/write.rs +++ b/src/write.rs @@ -260,6 +260,7 @@ impl FileOptions { Rc::get_mut(field).unwrap() } }; + vec.reserve_exact(data.len() + 4); vec.write_u16::(header_id)?; vec.write_u16::(data.len() as u16)?; vec.write_all(data)?; From 651a87f8814984db0f01b0667c770bec8d2f04e9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 09:47:25 -0700 Subject: [PATCH 199/281] Update CHANGELOG and bump version --- CHANGELOG.md | 6 ++++-- Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 366522d9..c1c8fedd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,8 +136,10 @@ - Alignment and extra-data fields are now attributes of [`zip_next::unstable::write::FileOptions`], allowing them to be specified for `add_directory` and `add_symlink`. - - Extra-data fields no longer have to be formatted by the caller. - + - Extra-data fields are now formatted by the `FileOptions` method `add_extra_data`. + - Improved performance, especially for `shallow_copy_file` and `deep_copy_file` on files with extra data. + ### Fixed - Fixes a rare bug where the size of the extra-data field could overflow when `large_file` was set. + - Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 6d1fe2f0..f537a4e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.7.5" +version = "0.8.0" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index cd418b81..cdcadb09 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.7.5" +zip_next = "0.8.0" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.7.5", default-features = false } +zip_next = { version = "0.8.0", default-features = false } ``` The features available are: From df3dfc199127bb994f398d3f6b5806958b39fee4 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 10:45:13 -0700 Subject: [PATCH 200/281] Change Rc to Arc since sharing across ZipWriter instances is possible --- CHANGELOG.md | 6 +++++- src/read.rs | 23 +++++++++++------------ src/types.rs | 11 ++++++----- src/write.rs | 38 +++++++++++++++++++------------------- 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1c8fedd..b000d5d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,4 +142,8 @@ ### Fixed - Fixes a rare bug where the size of the extra-data field could overflow when `large_file` was set. - - Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. \ No newline at end of file + - Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.8.1] + + - `ZipWriter` now implements `Send` if the underlying writer does. \ No newline at end of file diff --git a/src/read.rs b/src/read.rs index 051b11b6..cc95558c 100644 --- a/src/read.rs +++ b/src/read.rs @@ -14,7 +14,6 @@ use std::borrow::Cow; use std::collections::HashMap; use std::io::{self, prelude::*}; use std::path::Path; -use std::rc::Rc; use std::sync::Arc; #[cfg(any( @@ -551,7 +550,7 @@ impl ZipArchive { } /// Search for a file entry by name - pub fn by_name<'a>(&'a mut self, name: &str) -> ZipResult> { + pub fn by_name(&mut self, name: &str) -> ZipResult { Ok(self.by_name_with_optional_password(name, None)?.unwrap()) } @@ -582,11 +581,11 @@ impl ZipArchive { /// There are many passwords out there that will also pass the validity checks /// we are able to perform. This is a weakness of the ZipCrypto algorithm, /// due to its fairly primitive approach to cryptography. - pub fn by_index_decrypt<'a>( - &'a mut self, + pub fn by_index_decrypt( + &mut self, file_number: usize, password: &[u8], - ) -> ZipResult, InvalidPassword>> { + ) -> ZipResult> { self.by_index_with_optional_password(file_number, Some(password)) } @@ -613,11 +612,11 @@ impl ZipArchive { }) } - fn by_index_with_optional_password<'a>( - &'a mut self, + fn by_index_with_optional_password( + &mut self, file_number: usize, mut password: Option<&[u8]>, - ) -> ZipResult, InvalidPassword>> { + ) -> ZipResult> { let data = self .shared .files @@ -738,8 +737,8 @@ fn central_header_to_zip_file_inner( uncompressed_size: uncompressed_size as u64, file_name, file_name_raw, - extra_field: Rc::new(extra_field), - central_extra_field: Rc::new(vec![]), + extra_field: Arc::new(extra_field), + central_extra_field: Arc::new(vec![]), file_comment, header_start: offset, central_header_start, @@ -1099,8 +1098,8 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult, /// Extra field usually used for storage expansion - pub extra_field: Rc>, + pub extra_field: Arc>, /// Extra field only written to central directory - pub central_extra_field: Rc>, + pub central_extra_field: Arc>, /// File comment pub file_comment: String, /// Specifies where the local header of the file starts @@ -531,8 +531,8 @@ mod test { uncompressed_size: 0, file_name: file_name.clone(), file_name_raw: file_name.into_bytes(), - extra_field: Rc::new(vec![]), - central_extra_field: Rc::new(vec![]), + extra_field: Arc::new(vec![]), + central_extra_field: Arc::new(vec![]), file_comment: String::new(), header_start: 0, data_start: AtomicU64::new(0), @@ -581,6 +581,7 @@ mod test { assert!(DateTime::from_date_and_time(2107, 12, 32, 0, 0, 0).is_err()); } + use std::sync::Arc; #[cfg(feature = "time")] use time::{format_description::well_known::Rfc3339, OffsetDateTime}; diff --git a/src/write.rs b/src/write.rs index 7eaa8d27..ea9921bf 100644 --- a/src/write.rs +++ b/src/write.rs @@ -14,7 +14,7 @@ use std::io; use std::io::prelude::*; use std::io::{BufReader, SeekFrom}; use std::mem; -use std::rc::Rc; +use std::sync::Arc; #[cfg(any( feature = "deflate", @@ -134,8 +134,8 @@ pub struct FileOptions { pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, - extra_data: Rc>, - central_extra_data: Rc>, + extra_data: Arc>, + central_extra_data: Arc>, alignment: u16, } @@ -149,8 +149,8 @@ impl arbitrary::Arbitrary<'_> for FileOptions { permissions: Option::::arbitrary(u)?, large_file: bool::arbitrary(u)?, encrypt_with: Option::::arbitrary(u)?, - extra_data: Rc::new(vec![]), - central_extra_data: Rc::new(vec![]), + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: u16::arbitrary(u)?, }; u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { @@ -252,12 +252,12 @@ impl FileOptions { } else { &mut self.extra_data }; - let vec = Rc::get_mut(field); + let vec = Arc::get_mut(field); let vec = match vec { Some(exclusive) => exclusive, None => { - *field = Rc::new(field.to_vec()); - Rc::get_mut(field).unwrap() + *field = Arc::new(field.to_vec()); + Arc::get_mut(field).unwrap() } }; vec.reserve_exact(data.len() + 4); @@ -272,10 +272,10 @@ impl FileOptions { #[must_use] pub fn clear_extra_data(mut self) -> FileOptions { if self.extra_data.len() > 0 { - self.extra_data = Rc::new(vec![]); + self.extra_data = Arc::new(vec![]); } if self.central_extra_data.len() > 0 { - self.central_extra_data = Rc::new(vec![]); + self.central_extra_data = Arc::new(vec![]); } self } @@ -305,8 +305,8 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, - extra_data: Rc::new(vec![]), - central_extra_data: Rc::new(vec![]), + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 1, } } @@ -1398,7 +1398,7 @@ mod test { use crate::ZipArchive; use std::io; use std::io::{Read, Write}; - use std::rc::Rc; + use std::sync::Arc; #[test] fn write_empty_zip() { @@ -1518,8 +1518,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Rc::new(vec![]), - central_extra_data: Rc::new(vec![]), + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 1, }; writer.start_file("mimetype", options).unwrap(); @@ -1558,8 +1558,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Rc::new(vec![]), - central_extra_data: Rc::new(vec![]), + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1608,8 +1608,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Rc::new(vec![]), - central_extra_data: Rc::new(vec![]), + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); From 2b07fd1e1096942fa4f5b62d5caa86a43436f10c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 10:52:23 -0700 Subject: [PATCH 201/281] Remove unused import --- src/types.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index 4d70ac65..661cff4e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -581,7 +581,6 @@ mod test { assert!(DateTime::from_date_and_time(2107, 12, 32, 0, 0, 0).is_err()); } - use std::sync::Arc; #[cfg(feature = "time")] use time::{format_description::well_known::Rfc3339, OffsetDateTime}; From d03ac6a75bc468d5a1a25acacd8e6d00c4f13610 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 12:05:23 -0700 Subject: [PATCH 202/281] Bump version to 0.8.1 --- CHANGELOG.md | 4 +++- Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b000d5d1..55badc44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,4 +146,6 @@ ## [0.8.1] - - `ZipWriter` now implements `Send` if the underlying writer does. \ No newline at end of file +### Fixed + + - `ZipWriter` now once again implements `Send` if the underlying writer does. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index f537a4e2..3139a90e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.8.0" +version = "0.8.1" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index cdcadb09..c9c7ba4d 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.8.0" +zip_next = "0.8.1" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.8.0", default-features = false } +zip_next = { version = "0.8.1", default-features = false } ``` The features available are: From 0594f5ef3209f0d111af6f5ad4a6f8c32fd04815 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:21:29 -0700 Subject: [PATCH 203/281] Add missing alignment setter --- src/write.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/write.rs b/src/write.rs index ea9921bf..009da226 100644 --- a/src/write.rs +++ b/src/write.rs @@ -279,6 +279,13 @@ impl FileOptions { } self } + + /// Sets the alignment to the given number of bytes. + #[must_use] + pub fn with_alignment(mut self, alignment: u16) -> FileOptions { + self.alignment = alignment; + self + } } impl Default for FileOptions { @@ -1399,6 +1406,7 @@ mod test { use std::io; use std::io::{Read, Write}; use std::sync::Arc; + use crate::result::ZipResult; #[test] fn write_empty_zip() { @@ -1739,6 +1747,30 @@ mod test { println!("{:02x?}", zip.get_ref()); let _ = ZipArchive::new(zip).unwrap(); } + + #[test] + fn test_filename_looks_like_zip64_locator_5() -> ZipResult<()> { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .add_directory("", FileOptions::default().with_alignment(21)) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer.shallow_copy_file("/", "").unwrap(); + writer.shallow_copy_file("", "\0").unwrap(); + writer.shallow_copy_file("\0", "PK\u{6}\u{6}").unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file("\0\0\0\0\0\0", FileOptions::default()) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file("#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", FileOptions::default()) + .unwrap(); + let zip = writer.finish().unwrap(); + println!("{:02x?}", zip.get_ref()); + let _ = ZipArchive::new(zip).unwrap(); + Ok(()) + } } #[cfg(not(feature = "unreserved"))] From 3e87a376b9e557ad025cbd71128f266b3f9ba121 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:21:55 -0700 Subject: [PATCH 204/281] Handle special naming convention for directories --- fuzz/fuzz_targets/fuzz_write.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 6290460a..c7085ccc 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -28,13 +28,23 @@ pub struct FileOperation { reopen: bool, } +impl FileOperation { + fn referenceable_name(&self) -> &str { + if let WriteDirectory(_) = self.basic { + self.name + "/" + } else { + self.name + } + } +} + fn do_operation(writer: &mut RefCell>, operation: FileOperation) -> Result<(), Box> where T: Read + Write + Seek { let name = operation.name; match operation.basic { BasicFileOperation::WriteNormalFile {contents, mut options, ..} => { - if contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { + if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } writer.borrow_mut().start_file(name, options)?; @@ -49,12 +59,12 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; } BasicFileOperation::ShallowCopy(base) => { - let base_name = base.name.to_owned(); + let base_name = base.referenceable_name().to_owned(); do_operation(writer, *base)?; writer.borrow_mut().shallow_copy_file(&base_name, &name)?; } BasicFileOperation::DeepCopy(base) => { - let base_name = base.name.to_owned(); + let base_name = base.referenceable_name().to_owned(); do_operation(writer, *base)?; writer.borrow_mut().deep_copy_file(&base_name, &name)?; } From 5c51ddc49e636ce6c9c50778361e98791c0e2680 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:24:01 -0700 Subject: [PATCH 205/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index c7085ccc..a068ff07 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -30,7 +30,8 @@ pub struct FileOperation { impl FileOperation { fn referenceable_name(&self) -> &str { - if let WriteDirectory(_) = self.basic { + if let WriteDirectory(_) = self.basic && !self.name.ends_with('\\') + && !self.name.ends_with('/') { self.name + "/" } else { self.name From 382770a947bd3383531440440528a1099d8d989c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:25:35 -0700 Subject: [PATCH 206/281] Reformat --- src/write.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 009da226..f7db7441 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1401,12 +1401,12 @@ fn path_to_string(path: &std::path::Path) -> String { mod test { use super::{FileOptions, ZipWriter}; use crate::compression::CompressionMethod; + use crate::result::ZipResult; use crate::types::DateTime; use crate::ZipArchive; use std::io; use std::io::{Read, Write}; use std::sync::Arc; - use crate::result::ZipResult; #[test] fn write_empty_zip() { @@ -1764,7 +1764,10 @@ mod test { .unwrap(); let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); writer - .start_file("#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", FileOptions::default()) + .start_file( + "#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + FileOptions::default(), + ) .unwrap(); let zip = writer.finish().unwrap(); println!("{:02x?}", zip.get_ref()); From 8680df6f1f7edf90f55bb5eff512ad249b49db07 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:33:29 -0700 Subject: [PATCH 207/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index a068ff07..4dd02a87 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -29,12 +29,11 @@ pub struct FileOperation { } impl FileOperation { - fn referenceable_name(&self) -> &str { - if let WriteDirectory(_) = self.basic && !self.name.ends_with('\\') - && !self.name.ends_with('/') { - self.name + "/" - } else { - self.name + fn referenceable_name(&self) -> String { + if let BasicFileOperation::WriteDirectory(_) = self.basic { + if !self.name.ends_with('\\') && !self.name.ends_with('/') { + return self.name.to_owned() + "/"; + } } } } @@ -45,7 +44,7 @@ fn do_operation(writer: &mut RefCell>, let name = operation.name; match operation.basic { BasicFileOperation::WriteNormalFile {contents, mut options, ..} => { - if file.contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { + if contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { options = options.large_file(true); } writer.borrow_mut().start_file(name, options)?; @@ -60,12 +59,12 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; } BasicFileOperation::ShallowCopy(base) => { - let base_name = base.referenceable_name().to_owned(); + let base_name = base.referenceable_name(); do_operation(writer, *base)?; writer.borrow_mut().shallow_copy_file(&base_name, &name)?; } BasicFileOperation::DeepCopy(base) => { - let base_name = base.referenceable_name().to_owned(); + let base_name = base.referenceable_name(); do_operation(writer, *base)?; writer.borrow_mut().deep_copy_file(&base_name, &name)?; } From 51ce58751ecdd409d30ffb791f757cb136da9de5 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:36:31 -0700 Subject: [PATCH 208/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 4dd02a87..5df9e50d 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -34,6 +34,8 @@ impl FileOperation { if !self.name.ends_with('\\') && !self.name.ends_with('/') { return self.name.to_owned() + "/"; } + } else { + self.name.to_owned() } } } From c31d85930babd1867940b06b6d02a3a1a489dbc5 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:41:49 -0700 Subject: [PATCH 209/281] Refactor: add subdirectory-after-writing-parent option --- fuzz/fuzz_targets/fuzz_write.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 5df9e50d..ce9f213c 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -13,6 +13,9 @@ pub enum BasicFileOperation { options: zip_next::write::FileOptions, }, WriteDirectory(zip_next::write::FileOptions), + WriteSubdirectory { + parent_dir: BasicFileOperation::WriteSubdirectory + }, WriteSymlinkWithTarget { target: Box, options: zip_next::write::FileOptions, @@ -57,6 +60,10 @@ fn do_operation(writer: &mut RefCell>, BasicFileOperation::WriteDirectory(options) => { writer.borrow_mut().add_directory(name, options)?; } + BasicFileOperation::WriteSubdirectory(parent_dir) => { + do_operation(writer, parent_dir)?; + writer.borrow_mut().add_directory(parent_dir.referenceable_name() + name, options)?; + } BasicFileOperation::WriteSymlinkWithTarget {target, options} => { writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; } From 7bb1d9d190dca45b08b2e429b5f1fb8e635184ba Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:43:19 -0700 Subject: [PATCH 210/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index ce9f213c..d0c3aa65 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -33,10 +33,9 @@ pub struct FileOperation { impl FileOperation { fn referenceable_name(&self) -> String { - if let BasicFileOperation::WriteDirectory(_) = self.basic { - if !self.name.ends_with('\\') && !self.name.ends_with('/') { - return self.name.to_owned() + "/"; - } + if let BasicFileOperation::WriteDirectory(_) = self.basic + && !self.name.ends_with('\\') && !self.name.ends_with('/') { + return self.name.to_owned() + "/"; } else { self.name.to_owned() } From 70cf9514bbbce2f96ed994c44efecc2651c613ab Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:55:11 -0700 Subject: [PATCH 211/281] Revert "Bug fix" This reverts commit 7bb1d9d190dca45b08b2e429b5f1fb8e635184ba. --- fuzz/fuzz_targets/fuzz_write.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index d0c3aa65..ce9f213c 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -33,9 +33,10 @@ pub struct FileOperation { impl FileOperation { fn referenceable_name(&self) -> String { - if let BasicFileOperation::WriteDirectory(_) = self.basic - && !self.name.ends_with('\\') && !self.name.ends_with('/') { - return self.name.to_owned() + "/"; + if let BasicFileOperation::WriteDirectory(_) = self.basic { + if !self.name.ends_with('\\') && !self.name.ends_with('/') { + return self.name.to_owned() + "/"; + } } else { self.name.to_owned() } From e3b0da188b966341e8ce7f4a28b36090ef8a3144 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:55:12 -0700 Subject: [PATCH 212/281] Revert "Refactor: add subdirectory-after-writing-parent option" This reverts commit c31d85930babd1867940b06b6d02a3a1a489dbc5. --- fuzz/fuzz_targets/fuzz_write.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index ce9f213c..5df9e50d 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -13,9 +13,6 @@ pub enum BasicFileOperation { options: zip_next::write::FileOptions, }, WriteDirectory(zip_next::write::FileOptions), - WriteSubdirectory { - parent_dir: BasicFileOperation::WriteSubdirectory - }, WriteSymlinkWithTarget { target: Box, options: zip_next::write::FileOptions, @@ -60,10 +57,6 @@ fn do_operation(writer: &mut RefCell>, BasicFileOperation::WriteDirectory(options) => { writer.borrow_mut().add_directory(name, options)?; } - BasicFileOperation::WriteSubdirectory(parent_dir) => { - do_operation(writer, parent_dir)?; - writer.borrow_mut().add_directory(parent_dir.referenceable_name() + name, options)?; - } BasicFileOperation::WriteSymlinkWithTarget {target, options} => { writer.borrow_mut().add_symlink(name, target.to_string_lossy(), options)?; } From b6514e5c7804e4ea385a8a021556b7c2b602ca0a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 18:57:57 -0700 Subject: [PATCH 213/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 5df9e50d..2e36069e 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -34,9 +34,8 @@ impl FileOperation { if !self.name.ends_with('\\') && !self.name.ends_with('/') { return self.name.to_owned() + "/"; } - } else { - self.name.to_owned() } + self.name.to_owned() } } From 7370d63f2d5ec0826835d6d9dedb1195c6d836c4 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 14 May 2023 19:03:59 -0700 Subject: [PATCH 214/281] Bump version to 0.8.2 --- CHANGELOG.md | 12 +++++++++++- Cargo.toml | 2 +- README.md | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55badc44..3cb979d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -148,4 +148,14 @@ ### Fixed - - `ZipWriter` now once again implements `Send` if the underlying writer does. \ No newline at end of file + - `ZipWriter` now once again implements `Send` if the underlying writer does. + +## [0.8.2] + +### Fixed + + - Fixed an issue where code might spuriously fail during write fuzzing. + +### Added + + - New method `with_alignment` on `FileOptions`. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 3139a90e..22b3bc46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.8.1" +version = "0.8.2" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index c9c7ba4d..be2cc8b5 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.8.1" +zip_next = "0.8.2" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.8.1", default-features = false } +zip_next = { version = "0.8.2", default-features = false } ``` The features available are: From 1df9186527ec6f6964f0b671e791472a477a7c9f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 15 May 2023 20:23:46 -0700 Subject: [PATCH 215/281] Wrap extra-data field in an Option, to save Arc overhead when empty --- CHANGELOG.md | 8 +++- src/read.rs | 130 +++++++++++++++++++++++++++------------------------ src/types.rs | 8 ++-- src/write.rs | 86 ++++++++++++++++++++++------------ 4 files changed, 136 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb979d8..6979583a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,4 +158,10 @@ ### Added - - New method `with_alignment` on `FileOptions`. \ No newline at end of file + - New method `with_alignment` on `FileOptions`. + +## [0.8.3] + +### Changed + + - Improved performance for files with no extra data (should mainly affect multithreaded Mutex-guarded access). \ No newline at end of file diff --git a/src/read.rs b/src/read.rs index cc95558c..074b0af5 100644 --- a/src/read.rs +++ b/src/read.rs @@ -737,8 +737,8 @@ fn central_header_to_zip_file_inner( uncompressed_size: uncompressed_size as u64, file_name, file_name_raw, - extra_field: Arc::new(extra_field), - central_extra_field: Arc::new(vec![]), + extra_field: Some(Arc::new(extra_field)), + central_extra_field: None, file_comment, header_start: offset, central_header_start, @@ -770,69 +770,73 @@ fn central_header_to_zip_file_inner( } fn parse_extra_field(file: &mut ZipFileData) -> ZipResult<()> { - let mut reader = io::Cursor::new(file.extra_field.as_ref()); + if let Some(extra_data) = &file.extra_field { + let mut reader = io::Cursor::new(extra_data.as_ref()); - while (reader.position() as usize) < file.extra_field.len() { - let kind = reader.read_u16::()?; - let len = reader.read_u16::()?; - let mut len_left = len as i64; - match kind { - // Zip64 extended information extra field - 0x0001 => { - if file.uncompressed_size == spec::ZIP64_BYTES_THR { - file.large_file = true; - file.uncompressed_size = reader.read_u64::()?; - len_left -= 8; + while (reader.position() as usize) < extra_data.len() { + let kind = reader.read_u16::()?; + let len = reader.read_u16::()?; + let mut len_left = len as i64; + match kind { + // Zip64 extended information extra field + 0x0001 => { + if file.uncompressed_size == spec::ZIP64_BYTES_THR { + file.large_file = true; + file.uncompressed_size = reader.read_u64::()?; + len_left -= 8; + } + if file.compressed_size == spec::ZIP64_BYTES_THR { + file.large_file = true; + file.compressed_size = reader.read_u64::()?; + len_left -= 8; + } + if file.header_start == spec::ZIP64_BYTES_THR { + file.header_start = reader.read_u64::()?; + len_left -= 8; + } } - if file.compressed_size == spec::ZIP64_BYTES_THR { - file.large_file = true; - file.compressed_size = reader.read_u64::()?; - len_left -= 8; + 0x9901 => { + // AES + if len != 7 { + return Err(ZipError::UnsupportedArchive( + "AES extra data field has an unsupported length", + )); + } + let vendor_version = reader.read_u16::()?; + let vendor_id = reader.read_u16::()?; + let aes_mode = reader.read_u8()?; + let compression_method = reader.read_u16::()?; + + if vendor_id != 0x4541 { + return Err(ZipError::InvalidArchive("Invalid AES vendor")); + } + let vendor_version = match vendor_version { + 0x0001 => AesVendorVersion::Ae1, + 0x0002 => AesVendorVersion::Ae2, + _ => return Err(ZipError::InvalidArchive("Invalid AES vendor version")), + }; + match aes_mode { + 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version)), + 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version)), + 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version)), + _ => { + return Err(ZipError::InvalidArchive("Invalid AES encryption strength")) + } + }; + file.compression_method = { + #[allow(deprecated)] + CompressionMethod::from_u16(compression_method) + }; } - if file.header_start == spec::ZIP64_BYTES_THR { - file.header_start = reader.read_u64::()?; - len_left -= 8; + _ => { + // Other fields are ignored } } - 0x9901 => { - // AES - if len != 7 { - return Err(ZipError::UnsupportedArchive( - "AES extra data field has an unsupported length", - )); - } - let vendor_version = reader.read_u16::()?; - let vendor_id = reader.read_u16::()?; - let aes_mode = reader.read_u8()?; - let compression_method = reader.read_u16::()?; - if vendor_id != 0x4541 { - return Err(ZipError::InvalidArchive("Invalid AES vendor")); - } - let vendor_version = match vendor_version { - 0x0001 => AesVendorVersion::Ae1, - 0x0002 => AesVendorVersion::Ae2, - _ => return Err(ZipError::InvalidArchive("Invalid AES vendor version")), - }; - match aes_mode { - 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version)), - 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version)), - 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version)), - _ => return Err(ZipError::InvalidArchive("Invalid AES encryption strength")), - }; - file.compression_method = { - #[allow(deprecated)] - CompressionMethod::from_u16(compression_method) - }; + // We could also check for < 0 to check for errors + if len_left > 0 { + reader.seek(io::SeekFrom::Current(len_left))?; } - _ => { - // Other fields are ignored - } - } - - // We could also check for < 0 to check for errors - if len_left > 0 { - reader.seek(io::SeekFrom::Current(len_left))?; } } Ok(()) @@ -979,7 +983,11 @@ impl<'a> ZipFile<'a> { /// Get the extra data of the zip header for this file pub fn extra_data(&self) -> &[u8] { - &self.data.extra_field + if let Some(extra_data) = &self.data.extra_field { + extra_data + } else { + &[] + } } /// Get the starting offset of the data of the compressed file @@ -1098,8 +1106,8 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult, /// Extra field usually used for storage expansion - pub extra_field: Arc>, + pub extra_field: Option>>, /// Extra field only written to central directory - pub central_extra_field: Arc>, + pub central_extra_field: Option>>, /// File comment pub file_comment: String, /// Specifies where the local header of the file starts @@ -531,8 +531,8 @@ mod test { uncompressed_size: 0, file_name: file_name.clone(), file_name_raw: file_name.into_bytes(), - extra_field: Arc::new(vec![]), - central_extra_field: Arc::new(vec![]), + extra_field: None, + central_extra_field: None, file_comment: String::new(), header_start: 0, data_start: AtomicU64::new(0), diff --git a/src/write.rs b/src/write.rs index f7db7441..ca6b89ec 100644 --- a/src/write.rs +++ b/src/write.rs @@ -134,8 +134,8 @@ pub struct FileOptions { pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, - extra_data: Arc>, - central_extra_data: Arc>, + extra_data: Option>>, + central_extra_data: Option>>, alignment: u16, } @@ -149,8 +149,8 @@ impl arbitrary::Arbitrary<'_> for FileOptions { permissions: Option::::arbitrary(u)?, large_file: bool::arbitrary(u)?, encrypt_with: Option::::arbitrary(u)?, - extra_data: Arc::new(vec![]), - central_extra_data: Arc::new(vec![]), + extra_data: None, + central_extra_data: None, alignment: u16::arbitrary(u)?, }; u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { @@ -242,7 +242,14 @@ impl FileOptions { ) -> ZipResult<()> { validate_extra_data(header_id, data)?; let len = data.len() + 4; - if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { + if self.extra_data.as_ref().map_or(0, |data| data.len()) + + self + .central_extra_data + .as_ref() + .map_or(0, |data| data.len()) + + len + > u16::MAX as usize + { Err(InvalidArchive( "Extra data field would be longer than allowed", )) @@ -252,12 +259,17 @@ impl FileOptions { } else { &mut self.extra_data }; - let vec = Arc::get_mut(field); - let vec = match vec { - Some(exclusive) => exclusive, + let vec = match field { + Some(arc) => match Arc::get_mut(arc) { + Some(exclusive) => exclusive, + None => { + *field = Some(Arc::new(arc.to_vec())); + Arc::get_mut(field.as_mut().unwrap()).unwrap() + } + }, None => { - *field = Arc::new(field.to_vec()); - Arc::get_mut(field).unwrap() + *field = Some(Arc::new(Vec::new())); + Arc::get_mut(field.as_mut().unwrap()).unwrap() } }; vec.reserve_exact(data.len() + 4); @@ -271,11 +283,11 @@ impl FileOptions { /// Removes the extra data fields. #[must_use] pub fn clear_extra_data(mut self) -> FileOptions { - if self.extra_data.len() > 0 { - self.extra_data = Arc::new(vec![]); + if self.extra_data.is_some() { + self.extra_data = None; } - if self.central_extra_data.len() > 0 { - self.central_extra_data = Arc::new(vec![]); + if self.central_extra_data.is_some() { + self.central_extra_data = None; } self } @@ -312,8 +324,8 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, - extra_data: Arc::new(vec![]), - central_extra_data: Arc::new(vec![]), + extra_data: None, + central_extra_data: None, alignment: 1, } } @@ -582,11 +594,17 @@ impl ZipWriter { // file name length writer.write_u16::(file.file_name.as_bytes().len() as u16)?; // extra field length - let mut extra_field_length = file.extra_field.len(); + let mut extra_field_length = file.extra_field.as_ref().map_or(0, |data| data.len()); if file.large_file { extra_field_length += 20; } - if extra_field_length + file.central_extra_field.len() > u16::MAX as usize { + if extra_field_length + + file + .central_extra_field + .as_ref() + .map_or(0, |data| data.len()) + > u16::MAX as usize + { let _ = self.abort_file(); return Err(InvalidArchive("Extra data field is too large")); } @@ -598,7 +616,9 @@ impl ZipWriter { if file.large_file { write_local_zip64_extra_field(writer, file)?; } - writer.write_all(&file.extra_field)?; + if let Some(extra_data) = &file.extra_field { + writer.write_all(extra_data)?; + } let mut header_end = writer.stream_position()?; if options.alignment > 1 { let align = options.alignment as u64; @@ -1262,8 +1282,11 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // extra field length writer.write_u16::( zip64_extra_field_length - + file.extra_field.len() as u16 - + file.central_extra_field.len() as u16, + + file.extra_field.as_ref().map_or(0, |data| data.len()) as u16 + + file + .central_extra_field + .as_ref() + .map_or(0, |data| data.len()) as u16, )?; // file comment length writer.write_u16::(0)?; @@ -1280,8 +1303,12 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // zip64 extra field writer.write_all(&zip64_extra_field[..zip64_extra_field_length as usize])?; // extra field - writer.write_all(&file.extra_field)?; - writer.write_all(&file.central_extra_field)?; + if let Some(extra_data) = &file.extra_field { + writer.write_all(extra_data)?; + } + if let Some(extra_data) = &file.central_extra_field { + writer.write_all(extra_data)?; + } // file comment // @@ -1406,7 +1433,6 @@ mod test { use crate::ZipArchive; use std::io; use std::io::{Read, Write}; - use std::sync::Arc; #[test] fn write_empty_zip() { @@ -1526,8 +1552,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Arc::new(vec![]), - central_extra_data: Arc::new(vec![]), + extra_data: None, + central_extra_data: None, alignment: 1, }; writer.start_file("mimetype", options).unwrap(); @@ -1566,8 +1592,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Arc::new(vec![]), - central_extra_data: Arc::new(vec![]), + extra_data: None, + central_extra_data: None, alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1616,8 +1642,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: Arc::new(vec![]), - central_extra_data: Arc::new(vec![]), + extra_data: None, + central_extra_data: None, alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); From 5d16fa250c524c0a373a4aac75cc2782f47e42c1 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Mon, 15 May 2023 21:25:34 -0700 Subject: [PATCH 216/281] Revert "Wrap extra-data field in an Option, to save Arc overhead when empty" This reverts commit 1df9186527ec6f6964f0b671e791472a477a7c9f. --- CHANGELOG.md | 8 +--- src/read.rs | 130 ++++++++++++++++++++++++--------------------------- src/types.rs | 8 ++-- src/write.rs | 86 ++++++++++++---------------------- 4 files changed, 96 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6979583a..3cb979d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,10 +158,4 @@ ### Added - - New method `with_alignment` on `FileOptions`. - -## [0.8.3] - -### Changed - - - Improved performance for files with no extra data (should mainly affect multithreaded Mutex-guarded access). \ No newline at end of file + - New method `with_alignment` on `FileOptions`. \ No newline at end of file diff --git a/src/read.rs b/src/read.rs index 074b0af5..cc95558c 100644 --- a/src/read.rs +++ b/src/read.rs @@ -737,8 +737,8 @@ fn central_header_to_zip_file_inner( uncompressed_size: uncompressed_size as u64, file_name, file_name_raw, - extra_field: Some(Arc::new(extra_field)), - central_extra_field: None, + extra_field: Arc::new(extra_field), + central_extra_field: Arc::new(vec![]), file_comment, header_start: offset, central_header_start, @@ -770,73 +770,69 @@ fn central_header_to_zip_file_inner( } fn parse_extra_field(file: &mut ZipFileData) -> ZipResult<()> { - if let Some(extra_data) = &file.extra_field { - let mut reader = io::Cursor::new(extra_data.as_ref()); + let mut reader = io::Cursor::new(file.extra_field.as_ref()); - while (reader.position() as usize) < extra_data.len() { - let kind = reader.read_u16::()?; - let len = reader.read_u16::()?; - let mut len_left = len as i64; - match kind { - // Zip64 extended information extra field - 0x0001 => { - if file.uncompressed_size == spec::ZIP64_BYTES_THR { - file.large_file = true; - file.uncompressed_size = reader.read_u64::()?; - len_left -= 8; - } - if file.compressed_size == spec::ZIP64_BYTES_THR { - file.large_file = true; - file.compressed_size = reader.read_u64::()?; - len_left -= 8; - } - if file.header_start == spec::ZIP64_BYTES_THR { - file.header_start = reader.read_u64::()?; - len_left -= 8; - } + while (reader.position() as usize) < file.extra_field.len() { + let kind = reader.read_u16::()?; + let len = reader.read_u16::()?; + let mut len_left = len as i64; + match kind { + // Zip64 extended information extra field + 0x0001 => { + if file.uncompressed_size == spec::ZIP64_BYTES_THR { + file.large_file = true; + file.uncompressed_size = reader.read_u64::()?; + len_left -= 8; } - 0x9901 => { - // AES - if len != 7 { - return Err(ZipError::UnsupportedArchive( - "AES extra data field has an unsupported length", - )); - } - let vendor_version = reader.read_u16::()?; - let vendor_id = reader.read_u16::()?; - let aes_mode = reader.read_u8()?; - let compression_method = reader.read_u16::()?; - - if vendor_id != 0x4541 { - return Err(ZipError::InvalidArchive("Invalid AES vendor")); - } - let vendor_version = match vendor_version { - 0x0001 => AesVendorVersion::Ae1, - 0x0002 => AesVendorVersion::Ae2, - _ => return Err(ZipError::InvalidArchive("Invalid AES vendor version")), - }; - match aes_mode { - 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version)), - 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version)), - 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version)), - _ => { - return Err(ZipError::InvalidArchive("Invalid AES encryption strength")) - } - }; - file.compression_method = { - #[allow(deprecated)] - CompressionMethod::from_u16(compression_method) - }; + if file.compressed_size == spec::ZIP64_BYTES_THR { + file.large_file = true; + file.compressed_size = reader.read_u64::()?; + len_left -= 8; } - _ => { - // Other fields are ignored + if file.header_start == spec::ZIP64_BYTES_THR { + file.header_start = reader.read_u64::()?; + len_left -= 8; } } + 0x9901 => { + // AES + if len != 7 { + return Err(ZipError::UnsupportedArchive( + "AES extra data field has an unsupported length", + )); + } + let vendor_version = reader.read_u16::()?; + let vendor_id = reader.read_u16::()?; + let aes_mode = reader.read_u8()?; + let compression_method = reader.read_u16::()?; - // We could also check for < 0 to check for errors - if len_left > 0 { - reader.seek(io::SeekFrom::Current(len_left))?; + if vendor_id != 0x4541 { + return Err(ZipError::InvalidArchive("Invalid AES vendor")); + } + let vendor_version = match vendor_version { + 0x0001 => AesVendorVersion::Ae1, + 0x0002 => AesVendorVersion::Ae2, + _ => return Err(ZipError::InvalidArchive("Invalid AES vendor version")), + }; + match aes_mode { + 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version)), + 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version)), + 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version)), + _ => return Err(ZipError::InvalidArchive("Invalid AES encryption strength")), + }; + file.compression_method = { + #[allow(deprecated)] + CompressionMethod::from_u16(compression_method) + }; } + _ => { + // Other fields are ignored + } + } + + // We could also check for < 0 to check for errors + if len_left > 0 { + reader.seek(io::SeekFrom::Current(len_left))?; } } Ok(()) @@ -983,11 +979,7 @@ impl<'a> ZipFile<'a> { /// Get the extra data of the zip header for this file pub fn extra_data(&self) -> &[u8] { - if let Some(extra_data) = &self.data.extra_field { - extra_data - } else { - &[] - } + &self.data.extra_field } /// Get the starting offset of the data of the compressed file @@ -1106,8 +1098,8 @@ pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult, /// Extra field usually used for storage expansion - pub extra_field: Option>>, + pub extra_field: Arc>, /// Extra field only written to central directory - pub central_extra_field: Option>>, + pub central_extra_field: Arc>, /// File comment pub file_comment: String, /// Specifies where the local header of the file starts @@ -531,8 +531,8 @@ mod test { uncompressed_size: 0, file_name: file_name.clone(), file_name_raw: file_name.into_bytes(), - extra_field: None, - central_extra_field: None, + extra_field: Arc::new(vec![]), + central_extra_field: Arc::new(vec![]), file_comment: String::new(), header_start: 0, data_start: AtomicU64::new(0), diff --git a/src/write.rs b/src/write.rs index ca6b89ec..f7db7441 100644 --- a/src/write.rs +++ b/src/write.rs @@ -134,8 +134,8 @@ pub struct FileOptions { pub(crate) permissions: Option, pub(crate) large_file: bool, encrypt_with: Option, - extra_data: Option>>, - central_extra_data: Option>>, + extra_data: Arc>, + central_extra_data: Arc>, alignment: u16, } @@ -149,8 +149,8 @@ impl arbitrary::Arbitrary<'_> for FileOptions { permissions: Option::::arbitrary(u)?, large_file: bool::arbitrary(u)?, encrypt_with: Option::::arbitrary(u)?, - extra_data: None, - central_extra_data: None, + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: u16::arbitrary(u)?, }; u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { @@ -242,14 +242,7 @@ impl FileOptions { ) -> ZipResult<()> { validate_extra_data(header_id, data)?; let len = data.len() + 4; - if self.extra_data.as_ref().map_or(0, |data| data.len()) - + self - .central_extra_data - .as_ref() - .map_or(0, |data| data.len()) - + len - > u16::MAX as usize - { + if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { Err(InvalidArchive( "Extra data field would be longer than allowed", )) @@ -259,17 +252,12 @@ impl FileOptions { } else { &mut self.extra_data }; - let vec = match field { - Some(arc) => match Arc::get_mut(arc) { - Some(exclusive) => exclusive, - None => { - *field = Some(Arc::new(arc.to_vec())); - Arc::get_mut(field.as_mut().unwrap()).unwrap() - } - }, + let vec = Arc::get_mut(field); + let vec = match vec { + Some(exclusive) => exclusive, None => { - *field = Some(Arc::new(Vec::new())); - Arc::get_mut(field.as_mut().unwrap()).unwrap() + *field = Arc::new(field.to_vec()); + Arc::get_mut(field).unwrap() } }; vec.reserve_exact(data.len() + 4); @@ -283,11 +271,11 @@ impl FileOptions { /// Removes the extra data fields. #[must_use] pub fn clear_extra_data(mut self) -> FileOptions { - if self.extra_data.is_some() { - self.extra_data = None; + if self.extra_data.len() > 0 { + self.extra_data = Arc::new(vec![]); } - if self.central_extra_data.is_some() { - self.central_extra_data = None; + if self.central_extra_data.len() > 0 { + self.central_extra_data = Arc::new(vec![]); } self } @@ -324,8 +312,8 @@ impl Default for FileOptions { permissions: None, large_file: false, encrypt_with: None, - extra_data: None, - central_extra_data: None, + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 1, } } @@ -594,17 +582,11 @@ impl ZipWriter { // file name length writer.write_u16::(file.file_name.as_bytes().len() as u16)?; // extra field length - let mut extra_field_length = file.extra_field.as_ref().map_or(0, |data| data.len()); + let mut extra_field_length = file.extra_field.len(); if file.large_file { extra_field_length += 20; } - if extra_field_length - + file - .central_extra_field - .as_ref() - .map_or(0, |data| data.len()) - > u16::MAX as usize - { + if extra_field_length + file.central_extra_field.len() > u16::MAX as usize { let _ = self.abort_file(); return Err(InvalidArchive("Extra data field is too large")); } @@ -616,9 +598,7 @@ impl ZipWriter { if file.large_file { write_local_zip64_extra_field(writer, file)?; } - if let Some(extra_data) = &file.extra_field { - writer.write_all(extra_data)?; - } + writer.write_all(&file.extra_field)?; let mut header_end = writer.stream_position()?; if options.alignment > 1 { let align = options.alignment as u64; @@ -1282,11 +1262,8 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // extra field length writer.write_u16::( zip64_extra_field_length - + file.extra_field.as_ref().map_or(0, |data| data.len()) as u16 - + file - .central_extra_field - .as_ref() - .map_or(0, |data| data.len()) as u16, + + file.extra_field.len() as u16 + + file.central_extra_field.len() as u16, )?; // file comment length writer.write_u16::(0)?; @@ -1303,12 +1280,8 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // zip64 extra field writer.write_all(&zip64_extra_field[..zip64_extra_field_length as usize])?; // extra field - if let Some(extra_data) = &file.extra_field { - writer.write_all(extra_data)?; - } - if let Some(extra_data) = &file.central_extra_field { - writer.write_all(extra_data)?; - } + writer.write_all(&file.extra_field)?; + writer.write_all(&file.central_extra_field)?; // file comment // @@ -1433,6 +1406,7 @@ mod test { use crate::ZipArchive; use std::io; use std::io::{Read, Write}; + use std::sync::Arc; #[test] fn write_empty_zip() { @@ -1552,8 +1526,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: None, - central_extra_data: None, + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 1, }; writer.start_file("mimetype", options).unwrap(); @@ -1592,8 +1566,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: None, - central_extra_data: None, + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); @@ -1642,8 +1616,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, - extra_data: None, - central_extra_data: None, + extra_data: Arc::new(vec![]), + central_extra_data: Arc::new(vec![]), alignment: 0, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); From 9ff418e373245f4e995a1569fe3c71eb63e97686 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 16 May 2023 08:55:14 -0700 Subject: [PATCH 217/281] Reclassify `arbitrary` as a dev dependency --- CHANGELOG.md | 8 +++++++- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb979d8..b0a2f738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,4 +158,10 @@ ### Added - - New method `with_alignment` on `FileOptions`. \ No newline at end of file + - New method `with_alignment` on `FileOptions`. + +## [0.8.3] + +### Changed + + - Reclassify `arbitrary` as a dev dependency. diff --git a/Cargo.toml b/Cargo.toml index 22b3bc46..542d429a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ zstd = { version = "0.12.3", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" -[target.'cfg(fuzzing)'.dependencies] +[target.'cfg(fuzzing)'.dev-dependencies] arbitrary = { version = "1.3.0", features = ["derive"] } [dev-dependencies] From fc5c0ccd7d71bee72e6ea6334be5502071d59cba Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 16 May 2023 09:14:50 -0700 Subject: [PATCH 218/281] Revert "Reclassify `arbitrary` as a dev dependency" This reverts commit 9ff418e373245f4e995a1569fe3c71eb63e97686. --- CHANGELOG.md | 8 +------- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0a2f738..3cb979d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,10 +158,4 @@ ### Added - - New method `with_alignment` on `FileOptions`. - -## [0.8.3] - -### Changed - - - Reclassify `arbitrary` as a dev dependency. + - New method `with_alignment` on `FileOptions`. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 542d429a..22b3bc46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ zstd = { version = "0.12.3", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" -[target.'cfg(fuzzing)'.dev-dependencies] +[target.'cfg(fuzzing)'.dependencies] arbitrary = { version = "1.3.0", features = ["derive"] } [dev-dependencies] From b37dac5d79437f6b20cb8615c889cab2820d1833 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 16 May 2023 18:51:13 -0700 Subject: [PATCH 219/281] Update documentation link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be2cc8b5..48ddfa70 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.6.6/zip_next/) +[Documentation](https://docs.rs/zip_next/0.8.2/zip_next/) Info ---- From 4753b6ecb9864933875105c2e02e4dbba30bd850 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 16 May 2023 20:40:57 -0700 Subject: [PATCH 220/281] Remove unused import --- src/aes_ctr.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/aes_ctr.rs b/src/aes_ctr.rs index ae6ca24c..a1636b4b 100644 --- a/src/aes_ctr.rs +++ b/src/aes_ctr.rs @@ -4,7 +4,6 @@ //! different byte order (little endian) than NIST (big endian). //! See [AesCtrZipKeyStream] for more information. -use aes::cipher; use aes::cipher::generic_array::GenericArray; use aes::cipher::{BlockEncrypt, KeyInit}; use byteorder::WriteBytesExt; From fa045ad4c54c2b82532217447d290f5b796952fc Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 11:26:33 -0700 Subject: [PATCH 221/281] Bug fix for abort_file when deleting an entry that isn't the last --- CHANGELOG.md | 6 ++++++ fuzz/fuzz_targets/fuzz_write.rs | 4 ++++ src/write.rs | 35 ++++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f46256d..a4f5f049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,3 +165,9 @@ ### Merged from upstream - Uses the `aes::cipher::KeyInit` trait from `aes` 0.8.2 where appropriate. + +### Fixed + + - Calling `abort_file()` no longer corrupts the archive if called on a + shallow copy of a remaining file, or on an archive whose CDR entries are out + of sequence. However, it may leave an unused entry in the archive. \ No newline at end of file diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 2e36069e..13b536b6 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -25,6 +25,7 @@ pub enum BasicFileOperation { pub struct FileOperation { basic: BasicFileOperation, name: String, + abort: bool, reopen: bool, } @@ -70,6 +71,9 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().deep_copy_file(&base_name, &name)?; } } + if operation.abort { + writer.abort_file().unwrap(); + } if operation.reopen { let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); *writer = new_writer.into(); diff --git a/src/write.rs b/src/write.rs index f7db7441..5557c185 100644 --- a/src/write.rs +++ b/src/write.rs @@ -713,9 +713,19 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; - self.inner - .get_plain() - .seek(SeekFrom::Start(last_file.header_start))?; + + // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd + // overwrite a valid file and corrupt the archive + if !self.writing_to_file + && self + .files + .iter() + .all(|file| file.data_start.load() < last_file.data_start.load()) + { + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; + } self.writing_to_file = false; Ok(()) } @@ -1774,6 +1784,25 @@ mod test { let _ = ZipArchive::new(zip).unwrap(); Ok(()) } + + #[test] + fn remove_shallow_copy_keeps_original() -> ZipResult<()> { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer + .start_file("original", FileOptions::default()) + .unwrap(); + writer.write_all(RT_TEST_TEXT.as_bytes()).unwrap(); + writer + .shallow_copy_file("original", "shallow_copy") + .unwrap(); + writer.abort_file().unwrap(); + let mut zip = ZipArchive::new(writer.finish().unwrap()).unwrap(); + let mut file = zip.by_name("original").unwrap(); + let mut contents = Vec::new(); + file.read_to_end(&mut contents).unwrap(); + assert_eq!(RT_TEST_TEXT.as_bytes(), contents); + Ok(()) + } } #[cfg(not(feature = "unreserved"))] From a86a1f99a3a4c77e8db327f209e86fb73469e693 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 11:37:59 -0700 Subject: [PATCH 222/281] Bug fix for fuzz_write.rs --- fuzz/fuzz_targets/fuzz_write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 13b536b6..9ff43a47 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -72,7 +72,7 @@ fn do_operation(writer: &mut RefCell>, } } if operation.abort { - writer.abort_file().unwrap(); + writer.borrow_mut().abort_file().unwrap(); } if operation.reopen { let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); From 46333b2d61c093381f86bdb87f32a367480ff19a Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 11:39:22 -0700 Subject: [PATCH 223/281] Fix Clippy warning: `rev().next()` -> `next_back()` --- src/read.rs | 3 +-- src/read/stream.rs | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/read.rs b/src/read.rs index cc95558c..5cbba0ec 100644 --- a/src/read.rs +++ b/src/read.rs @@ -957,8 +957,7 @@ impl<'a> ZipFile<'a> { pub fn is_dir(&self) -> bool { self.name() .chars() - .rev() - .next() + .next_back() .map_or(false, |c| c == '/' || c == '\\') } diff --git a/src/read/stream.rs b/src/read/stream.rs index 5a01b23f..1075c423 100644 --- a/src/read/stream.rs +++ b/src/read/stream.rs @@ -1,6 +1,6 @@ use std::fs; use std::io::{self, Read}; -use std::path::Path; +use std::path::{Path, PathBuf}; use super::{ central_header_to_zip_file_inner, read_zipfile_from_stream, spec, ZipError, ZipFile, @@ -162,7 +162,7 @@ impl ZipStreamFileMetadata { /// [`ZipFile::enclosed_name`] is the better option in most scenarios. /// /// [`ParentDir`]: `Component::ParentDir` - pub fn mangled_name(&self) -> ::std::path::PathBuf { + pub fn mangled_name(&self) -> PathBuf { self.0.file_name_sanitized() } @@ -184,8 +184,7 @@ impl ZipStreamFileMetadata { pub fn is_dir(&self) -> bool { self.name() .chars() - .rev() - .next() + .next_back() .map_or(false, |c| c == '/' || c == '\\') } From d8f4d1aaa4bf81e6e8f505ec32046a74064249ae Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 11:45:59 -0700 Subject: [PATCH 224/281] Bug fix: fuzz_write tried to copy aborted files --- fuzz/fuzz_targets/fuzz_write.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 9ff43a47..e443a5b5 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -25,8 +25,8 @@ pub enum BasicFileOperation { pub struct FileOperation { basic: BasicFileOperation, name: String, - abort: bool, reopen: bool, + // 'abort' flag is separate, to prevent trying to copy an aborted file } impl FileOperation { @@ -41,7 +41,8 @@ impl FileOperation { } fn do_operation(writer: &mut RefCell>, - operation: FileOperation) -> Result<(), Box> + operation: FileOperation, + abort: bool) -> Result<(), Box> where T: Read + Write + Seek { let name = operation.name; match operation.basic { @@ -62,12 +63,12 @@ fn do_operation(writer: &mut RefCell>, } BasicFileOperation::ShallowCopy(base) => { let base_name = base.referenceable_name(); - do_operation(writer, *base)?; + do_operation(writer, *base, false)?; writer.borrow_mut().shallow_copy_file(&base_name, &name)?; } BasicFileOperation::DeepCopy(base) => { let base_name = base.referenceable_name(); - do_operation(writer, *base)?; + do_operation(writer, *base, false)?; writer.borrow_mut().deep_copy_file(&base_name, &name)?; } } @@ -81,10 +82,10 @@ fn do_operation(writer: &mut RefCell>, Ok(()) } -fuzz_target!(|data: Vec| { +fuzz_target!(|data: Vec<(FileOperation, bool)>| { let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()))); - for operation in data { - let _ = do_operation(&mut writer, operation); + for (operation, abort) in data { + let _ = do_operation(&mut writer, operation, abort); } let _ = zip_next::ZipArchive::new(writer.borrow_mut().finish().unwrap()); }); \ No newline at end of file From 4e3ecb9252b284c4f1d4535e01e8fdba18e42444 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 11:48:12 -0700 Subject: [PATCH 225/281] Bug fix --- fuzz/fuzz_targets/fuzz_write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index e443a5b5..294cb51a 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -72,7 +72,7 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().deep_copy_file(&base_name, &name)?; } } - if operation.abort { + if abort { writer.borrow_mut().abort_file().unwrap(); } if operation.reopen { From 710695219fb533a9998babbc859e5163ece37fe7 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 12:16:48 -0700 Subject: [PATCH 226/281] Bug fix: disable file encryption in abort_file --- src/write.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/write.rs b/src/write.rs index 5557c185..74f3b11a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -713,6 +713,7 @@ impl ZipWriter { .inner .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd // overwrite a valid file and corrupt the archive @@ -1803,8 +1804,23 @@ mod test { assert_eq!(RT_TEST_TEXT.as_bytes(), contents); Ok(()) } + + #[test] + fn remove_encrypted_file() -> ZipResult<()> { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let first_file_options = FileOptions::default() + .with_alignment(65535) + .with_deprecated_encryption(b"Password"); + writer.start_file("", first_file_options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new(zip); + writer.start_file("", FileOptions::default()).unwrap(); + Ok(()) + } } + #[cfg(not(feature = "unreserved"))] const EXTRA_FIELD_MAPPING: [u16; 49] = [ 0x0001, 0x0007, 0x0008, 0x0009, 0x000a, 0x000c, 0x000d, 0x000e, 0x000f, 0x0014, 0x0015, 0x0016, From 468ed36f1be909db83d0a989dda098af0ff95d35 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 12:17:20 -0700 Subject: [PATCH 227/281] Reformat --- src/write.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 74f3b11a..73337477 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1820,7 +1820,6 @@ mod test { } } - #[cfg(not(feature = "unreserved"))] const EXTRA_FIELD_MAPPING: [u16; 49] = [ 0x0001, 0x0007, 0x0008, 0x0009, 0x000a, 0x000c, 0x000d, 0x000e, 0x000f, 0x0014, 0x0015, 0x0016, From 2c897b52b111304c314fc14c1584e62ad43fe43e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 12:18:35 -0700 Subject: [PATCH 228/281] Update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4f5f049..2fd48d0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,4 +170,6 @@ - Calling `abort_file()` no longer corrupts the archive if called on a shallow copy of a remaining file, or on an archive whose CDR entries are out - of sequence. However, it may leave an unused entry in the archive. \ No newline at end of file + of sequence. However, it may leave an unused entry in the archive. + - Calling `abort_file()` while writing a ZipCrypto-encrypted file no longer + causes a crash. \ No newline at end of file From bef9fce30ab0877033676dbde6e12e9f12737e6b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 15:24:00 -0700 Subject: [PATCH 229/281] Bug fix: create a valid archive even when last file was aborted with content --- CHANGELOG.md | 8 +- fuzz/fuzz_targets/fuzz_write.rs | 3 +- src/write.rs | 140 +++++++++++++++++++++----------- 3 files changed, 100 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd48d0b..ac7b7924 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,4 +172,10 @@ shallow copy of a remaining file, or on an archive whose CDR entries are out of sequence. However, it may leave an unused entry in the archive. - Calling `abort_file()` while writing a ZipCrypto-encrypted file no longer - causes a crash. \ No newline at end of file + causes a crash. + - Calling `abort_file()` on the last file before `finish()` no longer produces + an invalid ZIP file or garbage in the comment. + + ### Added + + - `ZipWriter` methods `get_comment()` and `get_raw_comment()`. \ No newline at end of file diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 294cb51a..1f81a2ec 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -76,7 +76,8 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().abort_file().unwrap(); } if operation.reopen { - let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); + let mut new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); + assert_eq!(Ok(""), new_writer.get_comment()); *writer = new_writer.into(); } Ok(()) diff --git a/src/write.rs b/src/write.rs index 73337477..d6076065 100644 --- a/src/write.rs +++ b/src/write.rs @@ -14,6 +14,7 @@ use std::io; use std::io::prelude::*; use std::io::{BufReader, SeekFrom}; use std::mem; +use std::str::{from_utf8, Utf8Error}; use std::sync::Arc; #[cfg(any( @@ -504,6 +505,19 @@ impl ZipWriter { self.comment = comment; } + /// Get ZIP archive comment. + pub fn get_comment(&mut self) -> Result<&str, Utf8Error> { + from_utf8(self.get_raw_comment()) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8 + pub fn get_raw_comment(&self) -> &Vec { + &self.comment + } + /// Start a new file for with the requested options. fn start_entry( &mut self, @@ -714,14 +728,12 @@ impl ZipWriter { .prepare_next_writer(CompressionMethod::Stored, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; - // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd // overwrite a valid file and corrupt the archive - if !self.writing_to_file - && self - .files - .iter() - .all(|file| file.data_start.load() < last_file.data_start.load()) + if self + .files + .iter() + .all(|file| file.data_start.load() < last_file.data_start.load()) { self.inner .get_plain() @@ -957,56 +969,71 @@ impl ZipWriter { self.finish_file()?; { + let central_start = self.write_central_and_footer()?; let writer = self.inner.get_plain(); - - let central_start = writer.stream_position()?; - for file in self.files.iter() { - write_central_directory_header(writer, file)?; + let footer_end = writer.stream_position()?; + let file_end = writer.seek(SeekFrom::End(0))?; + if footer_end < file_end { + // Data from an aborted file is past the end of the footer, so rewrite the footer at + // the actual end. + let central_and_footer_size = footer_end - central_start; + writer.seek(SeekFrom::End(-(central_and_footer_size as i64)))?; + self.write_central_and_footer()?; } - let central_size = writer.stream_position()? - central_start; - - if self.files.len() > spec::ZIP64_ENTRY_THR - || central_size.max(central_start) > spec::ZIP64_BYTES_THR - { - let zip64_footer = spec::Zip64CentralDirectoryEnd { - version_made_by: DEFAULT_VERSION as u16, - version_needed_to_extract: DEFAULT_VERSION as u16, - disk_number: 0, - disk_with_central_directory: 0, - number_of_files_on_this_disk: self.files.len() as u64, - number_of_files: self.files.len() as u64, - central_directory_size: central_size, - central_directory_offset: central_start, - }; - - zip64_footer.write(writer)?; - - let zip64_footer = spec::Zip64CentralDirectoryEndLocator { - disk_with_central_directory: 0, - end_of_central_directory_offset: central_start + central_size, - number_of_disks: 1, - }; - - zip64_footer.write(writer)?; - } - - let number_of_files = self.files.len().min(spec::ZIP64_ENTRY_THR) as u16; - let footer = spec::CentralDirectoryEnd { - disk_number: 0, - disk_with_central_directory: 0, - zip_file_comment: self.comment.clone(), - number_of_files_on_this_disk: number_of_files, - number_of_files, - central_directory_size: central_size.min(spec::ZIP64_BYTES_THR) as u32, - central_directory_offset: central_start.min(spec::ZIP64_BYTES_THR) as u32, - }; - - footer.write(writer)?; } Ok(()) } + fn write_central_and_footer(&mut self) -> Result { + let writer = self.inner.get_plain(); + + let central_start = writer.stream_position()?; + for file in self.files.iter() { + write_central_directory_header(writer, file)?; + } + let central_size = writer.stream_position()? - central_start; + + if self.files.len() > spec::ZIP64_ENTRY_THR + || central_size.max(central_start) > spec::ZIP64_BYTES_THR + { + let zip64_footer = spec::Zip64CentralDirectoryEnd { + version_made_by: DEFAULT_VERSION as u16, + version_needed_to_extract: DEFAULT_VERSION as u16, + disk_number: 0, + disk_with_central_directory: 0, + number_of_files_on_this_disk: self.files.len() as u64, + number_of_files: self.files.len() as u64, + central_directory_size: central_size, + central_directory_offset: central_start, + }; + + zip64_footer.write(writer)?; + + let zip64_footer = spec::Zip64CentralDirectoryEndLocator { + disk_with_central_directory: 0, + end_of_central_directory_offset: central_start + central_size, + number_of_disks: 1, + }; + + zip64_footer.write(writer)?; + } + + let number_of_files = self.files.len().min(spec::ZIP64_ENTRY_THR) as u16; + let footer = spec::CentralDirectoryEnd { + disk_number: 0, + disk_with_central_directory: 0, + zip_file_comment: self.comment.clone(), + number_of_files_on_this_disk: number_of_files, + number_of_files, + central_directory_size: central_size.min(spec::ZIP64_BYTES_THR) as u32, + central_directory_offset: central_start.min(spec::ZIP64_BYTES_THR) as u32, + }; + + footer.write(writer)?; + Ok(central_start) + } + fn index_by_name(&self, name: &str) -> ZipResult { Ok(*self.files_by_name.get(name).ok_or(ZipError::FileNotFound)?) } @@ -1818,6 +1845,21 @@ mod test { writer.start_file("", FileOptions::default()).unwrap(); Ok(()) } + + #[test] + fn remove_encrypted_aligned_symlink() -> ZipResult<()> { + let mut options = FileOptions::default(); + options = options.with_deprecated_encryption(b"Password"); + options.alignment = 65535; + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + writer.add_symlink("", "s\t\0\0ggggg\0\0", options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + println!("{:0>2x?}", zip.get_ref()); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer.start_file("", FileOptions::default()).unwrap(); + Ok(()) + } } #[cfg(not(feature = "unreserved"))] From 235d2da74530e863c413305bc6efd3bec16dbc3c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 15:34:22 -0700 Subject: [PATCH 230/281] Include comments in fuzz_write --- fuzz/fuzz_targets/fuzz_write.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 1f81a2ec..580105f6 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -76,16 +76,19 @@ fn do_operation(writer: &mut RefCell>, writer.borrow_mut().abort_file().unwrap(); } if operation.reopen { - let mut new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); - assert_eq!(Ok(""), new_writer.get_comment()); + let old_comment = writer.borrow().get_raw_comment().to_owned(); + let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); + assert_eq!(&old_comment, new_writer.get_raw_comment()); *writer = new_writer.into(); } Ok(()) } -fuzz_target!(|data: Vec<(FileOperation, bool)>| { +fuzz_target!(|data: (Vec, Vec<(FileOperation, bool)>)| { + let (comment, operations) = data; let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()))); - for (operation, abort) in data { + writer.borrow_mut().set_raw_comment(comment); + for (operation, abort) in operations { let _ = do_operation(&mut writer, operation, abort); } let _ = zip_next::ZipArchive::new(writer.borrow_mut().finish().unwrap()); From dae16c431fcd14a1eed16aaec78d4c76b0d20b25 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 21 May 2023 17:58:20 -0700 Subject: [PATCH 231/281] Bump version to 0.8.3 --- Cargo.toml | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 22b3bc46..94037d24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.8.2" +version = "0.8.3" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 48ddfa70..6446e45f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.8.2/zip_next/) +[Documentation](https://docs.rs/zip_next/0.8.3/zip_next/) Info ---- @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.8.2" +zip_next = "0.8.3" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.8.2", default-features = false } +zip_next = { version = "0.8.3", default-features = false } ``` The features available are: From 829f7935cb45d46625111118d573cc439476540a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 11:07:01 +0000 Subject: [PATCH 232/281] Bump dependabot/fetch-metadata from 1.4.0 to 1.5.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/dependabot_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot_automation.yml b/.github/workflows/dependabot_automation.yml index 4104bafd..0d5c25fe 100644 --- a/.github/workflows/dependabot_automation.yml +++ b/.github/workflows/dependabot_automation.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.4.0 + uses: dependabot/fetch-metadata@v1.5.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Approve From 877be556891ff04ec59ead467c523b6346568d28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 11:14:18 +0000 Subject: [PATCH 233/281] Bump dependabot/fetch-metadata from 1.5.0 to 1.5.1 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/dependabot_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot_automation.yml b/.github/workflows/dependabot_automation.yml index 0d5c25fe..67e9e10e 100644 --- a/.github/workflows/dependabot_automation.yml +++ b/.github/workflows/dependabot_automation.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.5.0 + uses: dependabot/fetch-metadata@v1.5.1 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Approve From 255cfaf26105de0d647410c6fd31d4fb6457d66c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Fri, 26 May 2023 17:22:45 -0700 Subject: [PATCH 234/281] Add flush_on_finish_file parameter --- CHANGELOG.md | 8 +++- Cargo.toml | 2 +- README.md | 6 +-- benches/read_entry.rs | 2 +- benches/read_metadata.rs | 2 +- examples/write_dir.rs | 2 +- examples/write_sample.rs | 2 +- fuzz/fuzz_targets/fuzz_write.rs | 32 ++++++++------ src/write.rs | 76 ++++++++++++++++++++------------- tests/end_to_end.rs | 6 +-- tests/zip_crypto.rs | 2 +- 11 files changed, 86 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac7b7924..f033712d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -178,4 +178,10 @@ ### Added - - `ZipWriter` methods `get_comment()` and `get_raw_comment()`. \ No newline at end of file + - `ZipWriter` methods `get_comment()` and `get_raw_comment()`. + +## [0.9.0] + +### Added + + - `flush_on_finish_file` parameter for `ZipWriter`. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 94037d24..12395874 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.8.3" +version = "0.9.0" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 6446e45f..f8aa7671 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.8.3/zip_next/) +[Documentation](https://docs.rs/zip_next/0.9.0/zip_next/) Info ---- @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.8.3" +zip_next = "0.9.0" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.8.3", default-features = false } +zip_next = { version = "0.9.0", default-features = false } ``` The features available are: diff --git a/benches/read_entry.rs b/benches/read_entry.rs index 4ee20b02..dd4bced7 100644 --- a/benches/read_entry.rs +++ b/benches/read_entry.rs @@ -8,7 +8,7 @@ use zip_next::{ZipArchive, ZipWriter}; fn generate_random_archive(size: usize) -> Vec { let data = Vec::new(); - let mut writer = ZipWriter::new(Cursor::new(data)); + let mut writer = ZipWriter::new(Cursor::new(data), false); let options = zip_next::write::FileOptions::default() .compression_method(zip_next::CompressionMethod::Stored); diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index f9be2ec3..eee2d713 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -11,7 +11,7 @@ const FILE_SIZE: usize = 1024; fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { let data = Vec::new(); - let mut writer = ZipWriter::new(Cursor::new(data)); + let mut writer = ZipWriter::new(Cursor::new(data), false); let options = FileOptions::default().compression_method(CompressionMethod::Stored); let bytes = vec![0u8; file_size]; diff --git a/examples/write_dir.rs b/examples/write_dir.rs index f0d9efcd..61535c34 100644 --- a/examples/write_dir.rs +++ b/examples/write_dir.rs @@ -73,7 +73,7 @@ fn zip_dir( where T: Write + Seek, { - let mut zip = zip_next::ZipWriter::new(writer); + let mut zip = zip_next::ZipWriter::new(writer, false); let options = FileOptions::default() .compression_method(method) .unix_permissions(0o755); diff --git a/examples/write_sample.rs b/examples/write_sample.rs index bb9739d0..0834d473 100644 --- a/examples/write_sample.rs +++ b/examples/write_sample.rs @@ -25,7 +25,7 @@ fn doit(filename: &str) -> zip_next::result::ZipResult<()> { let path = std::path::Path::new(filename); let file = std::fs::File::create(path).unwrap(); - let mut zip = zip_next::ZipWriter::new(file); + let mut zip = zip_next::ZipWriter::new(file, false); zip.add_directory("test/", Default::default())?; diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 580105f6..6961e9df 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -6,7 +6,7 @@ use arbitrary::Arbitrary; use std::io::{Cursor, Read, Seek, Write}; use std::path::{PathBuf}; -#[derive(Arbitrary,Debug)] +#[derive(Arbitrary,Clone,Debug)] pub enum BasicFileOperation { WriteNormalFile { contents: Vec>, @@ -21,7 +21,7 @@ pub enum BasicFileOperation { DeepCopy(Box), } -#[derive(Arbitrary,Debug)] +#[derive(Arbitrary,Clone,Debug)] pub struct FileOperation { basic: BasicFileOperation, name: String, @@ -29,6 +29,13 @@ pub struct FileOperation { // 'abort' flag is separate, to prevent trying to copy an aborted file } +#[derive(Arbitrary,Clone,Debug)] +pub struct FuzzTestCase { + comment: Vec, + operations: Vec<(FileOperation, bool)>, + flush_on_finish_file: bool, +} + impl FileOperation { fn referenceable_name(&self) -> String { if let BasicFileOperation::WriteDirectory(_) = self.basic { @@ -42,7 +49,7 @@ impl FileOperation { fn do_operation(writer: &mut RefCell>, operation: FileOperation, - abort: bool) -> Result<(), Box> + abort: bool, flush_on_finish_file: bool) -> Result<(), Box> where T: Read + Write + Seek { let name = operation.name; match operation.basic { @@ -63,12 +70,12 @@ fn do_operation(writer: &mut RefCell>, } BasicFileOperation::ShallowCopy(base) => { let base_name = base.referenceable_name(); - do_operation(writer, *base, false)?; + do_operation(writer, *base, false, flush_on_finish_file)?; writer.borrow_mut().shallow_copy_file(&base_name, &name)?; } BasicFileOperation::DeepCopy(base) => { let base_name = base.referenceable_name(); - do_operation(writer, *base, false)?; + do_operation(writer, *base, false, flush_on_finish_file)?; writer.borrow_mut().deep_copy_file(&base_name, &name)?; } } @@ -77,19 +84,20 @@ fn do_operation(writer: &mut RefCell>, } if operation.reopen { let old_comment = writer.borrow().get_raw_comment().to_owned(); - let new_writer = zip_next::ZipWriter::new_append(writer.borrow_mut().finish().unwrap()).unwrap(); + let new_writer = zip_next::ZipWriter::new_append( + writer.borrow_mut().finish().unwrap(), flush_on_finish_file).unwrap(); assert_eq!(&old_comment, new_writer.get_raw_comment()); *writer = new_writer.into(); } Ok(()) } -fuzz_target!(|data: (Vec, Vec<(FileOperation, bool)>)| { - let (comment, operations) = data; - let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()))); - writer.borrow_mut().set_raw_comment(comment); - for (operation, abort) in operations { - let _ = do_operation(&mut writer, operation, abort); +fuzz_target!(|test_case: FuzzTestCase| { + let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()), + test_case.flush_on_finish_file)); + writer.borrow_mut().set_raw_comment(test_case.comment); + for (operation, abort) in test_case.operations { + let _ = do_operation(&mut writer, operation, abort, test_case.flush_on_finish_file); } let _ = zip_next::ZipArchive::new(writer.borrow_mut().finish().unwrap()); }); \ No newline at end of file diff --git a/src/write.rs b/src/write.rs index d6076065..ccd359d6 100644 --- a/src/write.rs +++ b/src/write.rs @@ -84,7 +84,7 @@ pub(crate) mod zip_writer { /// /// // We use a buffer here, though you'd normally use a `File` /// let mut buf = [0; 65536]; - /// let mut zip = ZipWriter::new(std::io::Cursor::new(&mut buf[..])); + /// let mut zip = ZipWriter::new(std::io::Cursor::new(&mut buf[..]), false); /// /// let options = FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); /// zip.start_file("hello_world.txt", options)?; @@ -106,6 +106,7 @@ pub(crate) mod zip_writer { pub(super) writing_to_file: bool, pub(super) writing_raw: bool, pub(super) comment: Vec, + pub(super) flush_on_finish_file: bool, } } use crate::result::ZipError::InvalidArchive; @@ -372,7 +373,9 @@ impl ZipWriterStats { impl ZipWriter { /// Initializes the archive from an existing ZIP archive, making it ready for append. - pub fn new_append(mut readwriter: A) -> ZipResult> { + /// + /// See [`ZipWriter::new`] for the caveats that apply when `flush_on_finish_file` is set. + pub fn new_append(mut readwriter: A, flush_on_finish_file: bool) -> ZipResult> { let (footer, cde_start_pos) = spec::CentralDirectoryEnd::find_and_parse(&mut readwriter)?; if footer.disk_number != footer.disk_with_central_directory { @@ -409,6 +412,7 @@ impl ZipWriter { writing_to_file: false, comment: footer.zip_file_comment, writing_raw: true, // avoid recomputing the last file's header + flush_on_finish_file, }) } } @@ -472,7 +476,14 @@ impl ZipWriter { /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. /// After a successful write, the file remains open for writing. After a failed write, call /// [`ZipWriter::is_writing_file`] to determine if the file remains open. - pub fn new(inner: W) -> ZipWriter { + /// + /// `flush_on_finish_file` is designed to support a streaming `inner` that may unload flushed + /// bytes. This ZipWriter will not try to seek further back than the last flushed byte unless + /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns + /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to + /// read flushed bytes and not overwrite them. Do not enable this with a [BufWriter], because + /// that implicitly calls [`Writer::flush`] whenever [`Seek::seek`] is called. + pub fn new(inner: W, flush_on_finish_file: bool) -> ZipWriter { ZipWriter { inner: Storer(MaybeEncrypted::Unencrypted(inner)), files: Vec::new(), @@ -481,6 +492,7 @@ impl ZipWriter { writing_to_file: false, writing_raw: false, comment: Vec::new(), + flush_on_finish_file, } } @@ -699,6 +711,12 @@ impl ZipWriter { update_local_file_header(writer, file)?; writer.seek(SeekFrom::Start(file_end))?; } + if self.flush_on_finish_file { + if let Err(e) = writer.flush() { + self.abort_file()?; + return Err(e.into()); + } + } self.writing_to_file = false; Ok(()) @@ -1276,7 +1294,7 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) writer.write_u16::(version_made_by)?; // version needed to extract writer.write_u16::(file.version_needed())?; - // general puprose bit flag + // general purpose bit flag let flag = if !file.file_name.is_ascii() { 1u16 << 11 } else { @@ -1307,7 +1325,7 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) writer.write_u16::(0)?; // disk number start writer.write_u16::(0)?; - // internal file attribytes + // internal file attributes writer.write_u16::(0)?; // external file attributes writer.write_u32::(file.external_attributes)?; @@ -1448,7 +1466,7 @@ mod test { #[test] fn write_empty_zip() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer.set_comment("ZIP"); let result = writer.finish().unwrap(); assert_eq!(result.get_ref().len(), 25); @@ -1467,7 +1485,7 @@ mod test { #[test] fn write_zip_dir() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .add_directory( "test", @@ -1495,7 +1513,7 @@ mod test { #[test] fn write_symlink_simple() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .add_symlink( "name", @@ -1524,7 +1542,7 @@ mod test { #[test] fn write_symlink_wonky_paths() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .add_symlink( "directory\\link", @@ -1556,7 +1574,7 @@ mod test { #[test] fn write_mimetype_zip() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); let options = FileOptions { compression_method: CompressionMethod::Stored, compression_level: None, @@ -1596,7 +1614,7 @@ mod test { #[test] fn test_shallow_copy() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); let options = FileOptions { compression_method: CompressionMethod::Deflated, compression_level: Some(9), @@ -1617,7 +1635,7 @@ mod test { .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .expect_err("Duplicate filename"); let zip = writer.finish().unwrap(); - let mut writer = ZipWriter::new_append(zip).unwrap(); + let mut writer = ZipWriter::new_append(zip, false).unwrap(); writer .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) .expect_err("Duplicate filename"); @@ -1646,7 +1664,7 @@ mod test { #[test] fn test_deep_copy() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); let options = FileOptions { compression_method: CompressionMethod::Deflated, compression_level: Some(9), @@ -1664,7 +1682,7 @@ mod test { .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .unwrap(); let zip = writer.finish().unwrap(); - let mut writer = ZipWriter::new_append(zip).unwrap(); + let mut writer = ZipWriter::new_append(zip, false).unwrap(); writer .deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME) .unwrap(); @@ -1693,7 +1711,7 @@ mod test { #[test] fn duplicate_filenames() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file("foo/bar/test", FileOptions::default()) .unwrap(); @@ -1707,7 +1725,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file( "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1720,7 +1738,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_2() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file( "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1734,7 +1752,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_2a() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file( "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1748,7 +1766,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_3() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file("\0PK\u{6}\u{6}", FileOptions::default()) .unwrap(); @@ -1765,7 +1783,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_4() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file("PK\u{6}\u{6}", FileOptions::default()) .unwrap(); @@ -1788,19 +1806,19 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_5() -> ZipResult<()> { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .add_directory("", FileOptions::default().with_alignment(21)) .unwrap(); - let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap(), false).unwrap(); writer.shallow_copy_file("/", "").unwrap(); writer.shallow_copy_file("", "\0").unwrap(); writer.shallow_copy_file("\0", "PK\u{6}\u{6}").unwrap(); - let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap(), false).unwrap(); writer .start_file("\0\0\0\0\0\0", FileOptions::default()) .unwrap(); - let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap(), false).unwrap(); writer .start_file( "#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1815,7 +1833,7 @@ mod test { #[test] fn remove_shallow_copy_keeps_original() -> ZipResult<()> { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer .start_file("original", FileOptions::default()) .unwrap(); @@ -1834,14 +1852,14 @@ mod test { #[test] fn remove_encrypted_file() -> ZipResult<()> { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); let first_file_options = FileOptions::default() .with_alignment(65535) .with_deprecated_encryption(b"Password"); writer.start_file("", first_file_options).unwrap(); writer.abort_file().unwrap(); let zip = writer.finish().unwrap(); - let mut writer = ZipWriter::new(zip); + let mut writer = ZipWriter::new(zip, false); writer.start_file("", FileOptions::default()).unwrap(); Ok(()) } @@ -1851,12 +1869,12 @@ mod test { let mut options = FileOptions::default(); options = options.with_deprecated_encryption(b"Password"); options.alignment = 65535; - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer.add_symlink("", "s\t\0\0ggggg\0\0", options).unwrap(); writer.abort_file().unwrap(); let zip = writer.finish().unwrap(); println!("{:0>2x?}", zip.get_ref()); - let mut writer = ZipWriter::new_append(zip).unwrap(); + let mut writer = ZipWriter::new_append(zip, false).unwrap(); writer.start_file("", FileOptions::default()).unwrap(); Ok(()) } diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 70e59f61..30f48506 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -35,7 +35,7 @@ fn copy() { { let mut src_archive = zip_next::ZipArchive::new(src_file).unwrap(); - let mut zip = ZipWriter::new(&mut tgt_file); + let mut zip = ZipWriter::new(&mut tgt_file, false); { let file = src_archive @@ -73,7 +73,7 @@ fn append() { write_test_archive(file, method, *shallow_copy); { - let mut zip = ZipWriter::new_append(&mut file).unwrap(); + let mut zip = ZipWriter::new_append(&mut file, false).unwrap(); zip.start_file( COPY_ENTRY_NAME, FileOptions::default() @@ -95,7 +95,7 @@ fn append() { // Write a test zip archive to buffer. fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, shallow_copy: bool) { - let mut zip = ZipWriter::new(file); + let mut zip = ZipWriter::new(file, false); zip.add_directory("test/", Default::default()).unwrap(); diff --git a/tests/zip_crypto.rs b/tests/zip_crypto.rs index 45612aac..4b25de8d 100644 --- a/tests/zip_crypto.rs +++ b/tests/zip_crypto.rs @@ -25,7 +25,7 @@ fn encrypting_file() { use std::io::{Read, Write}; use zip_next::unstable::write::FileOptionsExt; let mut buf = vec![0; 2048]; - let mut archive = zip_next::write::ZipWriter::new(Cursor::new(&mut buf)); + let mut archive = zip_next::write::ZipWriter::new(Cursor::new(&mut buf), false); archive .start_file( "name", From 87291bc4294f00d887bdbc727fd74c0a98a27583 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:02:14 -0700 Subject: [PATCH 235/281] Add Zopfli --- Cargo.toml | 4 +- src/compression.rs | 18 ++++++--- src/write.rs | 99 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 108 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 12395874..29ac853f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.21", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true } +zopfli = { version = "0.7.4", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] crossbeam-utils = "0.8.15" @@ -42,8 +43,9 @@ aes-crypto = [ "aes", "constant_time_eq", "hmac", "pbkdf2", "sha1" ] deflate = ["flate2/rust_backend"] deflate-miniz = ["flate2/default"] deflate-zlib = ["flate2/zlib"] +deflate-zopfli = ["zopfli"] unreserved = [] -default = ["aes-crypto", "bzip2", "deflate", "time", "zstd"] +default = ["aes-crypto", "bzip2", "deflate", "deflate-zopfli", "time", "zstd"] [[bench]] name = "read_entry" diff --git a/src/compression.rs b/src/compression.rs index 1e9ae18c..4d568bf0 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -20,7 +20,8 @@ pub enum CompressionMethod { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] Deflated, /// Compress the file using BZIP2 @@ -55,13 +56,15 @@ impl CompressionMethod { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] pub const DEFLATE: Self = CompressionMethod::Deflated; #[cfg(not(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" )))] pub const DEFLATE: Self = CompressionMethod::Unsupported(8); pub const DEFLATE64: Self = CompressionMethod::Unsupported(9); @@ -101,7 +104,8 @@ impl CompressionMethod { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] 8 => CompressionMethod::Deflated, #[cfg(feature = "bzip2")] @@ -127,7 +131,8 @@ impl CompressionMethod { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] CompressionMethod::Deflated => 8, #[cfg(feature = "bzip2")] @@ -155,7 +160,8 @@ pub const SUPPORTED_COMPRESSION_METHODS: &[CompressionMethod] = &[ #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] CompressionMethod::Deflated, #[cfg(feature = "bzip2")] diff --git a/src/write.rs b/src/write.rs index ccd359d6..13ef3811 100644 --- a/src/write.rs +++ b/src/write.rs @@ -14,6 +14,7 @@ use std::io; use std::io::prelude::*; use std::io::{BufReader, SeekFrom}; use std::mem; +use std::num::NonZeroU8; use std::str::{from_utf8, Utf8Error}; use std::sync::Arc; @@ -26,9 +27,11 @@ use flate2::write::DeflateEncoder; #[cfg(feature = "bzip2")] use bzip2::write::BzEncoder; +use flate2::Compression; #[cfg(feature = "time")] use time::OffsetDateTime; +use zopfli::Options; #[cfg(feature = "zstd")] use zstd::stream::write::Encoder as ZstdEncoder; @@ -60,6 +63,8 @@ enum GenericZipWriter { feature = "deflate-zlib" ))] Deflater(DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(zopfli::DeflateEncoder>), #[cfg(feature = "bzip2")] Bzip2(BzEncoder>), #[cfg(feature = "zstd")] @@ -185,7 +190,8 @@ impl FileOptions { /// `None` value specifies default compression level. /// /// Range of values depends on compression method: - /// * `Deflated`: 0 - 9. Default is 6 + /// * `Deflated`: 10 - 264 for Zopfli, 0 - 9 for other encoders. Default is 24 if Zopfli is the + /// only encoder, or 6 otherwise. /// * `Bzip2`: 0 - 9. Default is 6 /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 /// * others: only `None` is allowed @@ -1110,21 +1116,78 @@ impl GenericZipWriter { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] CompressionMethod::Deflated => { + #[cfg(all( + not(feature = "deflate"), + not(feature = "deflate-miniz"), + not(feature = "deflate-zlib"), + feature = "deflate-zopfli" + ))] + let default = 24; + + #[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib"))] + let default = flate2::Compression::default().level() as i32; + let level = clamp_opt( - compression_level.unwrap_or(flate2::Compression::default().level() as i32), + compression_level.unwrap_or(default), deflate_compression_level_range(), ) .ok_or(ZipError::UnsupportedArchive( "Unsupported compression level", ))? as u32; - Ok(Box::new(move |bare| { + + #[cfg(not(feature = "deflate-zopfli"))] + return Ok(Box::new(move |bare| { GenericZipWriter::Deflater(DeflateEncoder::new( bare, flate2::Compression::new(level), )) + })); + + #[cfg(all( + not(feature = "deflate"), + not(feature = "deflate-miniz"), + not(feature = "deflate-zlib"), + feature = "deflate-zopfli" + ))] + return Ok(Box::new(move |bare| { + let mut options = Options::default(); + options.iteration_count = NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); + GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( + options, + Default::default(), + bare + )) + })); + + #[cfg(all( + any(feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib"), + feature = "deflate-zopfli") + )] + Ok(Box::new(move |bare| { + let best_non_zopfli = Compression::best().level(); + if level > best_non_zopfli { + let mut options = Options::default(); + options.iteration_count = NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); + GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( + options, + Default::default(), + bare + )) + } else { + GenericZipWriter::Deflater(DeflateEncoder::new( + bare, + flate2::Compression::new(level), + )) + } })) } #[cfg(feature = "bzip2")] @@ -1175,6 +1238,8 @@ impl GenericZipWriter { feature = "deflate-zlib" ))] GenericZipWriter::Deflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + GenericZipWriter::ZopfliDeflater(w) => w.finish()?, #[cfg(feature = "bzip2")] GenericZipWriter::Bzip2(w) => w.finish()?, #[cfg(feature = "zstd")] @@ -1192,7 +1257,7 @@ impl GenericZipWriter { } fn ref_mut(&mut self) -> Option<&mut dyn Write> { - match *self { + match self { Storer(ref mut w) => Some(w as &mut dyn Write), #[cfg(any( feature = "deflate", @@ -1200,6 +1265,8 @@ impl GenericZipWriter { feature = "deflate-zlib" ))] GenericZipWriter::Deflater(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + GenericZipWriter::ZopfliDeflater(w) => Some(w as &mut dyn Write), #[cfg(feature = "bzip2")] GenericZipWriter::Bzip2(ref mut w) => Some(w as &mut dyn Write), #[cfg(feature = "zstd")] @@ -1230,11 +1297,30 @@ impl GenericZipWriter { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" ))] fn deflate_compression_level_range() -> std::ops::RangeInclusive { + #[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + ))] let min = flate2::Compression::none().level() as i32; + + #[cfg(not(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + )))] + let min = flate2::Compression::best().level() + 1 as i32; + + #[cfg(not(feature = "deflate-zopfli"))] let max = flate2::Compression::best().level() as i32; + + #[cfg(feature = "deflate-zopfli")] + let max = flate2::Compression::best().level() as i32 + u8::MAX as i32; + min..=max } @@ -1249,6 +1335,7 @@ fn bzip2_compression_level_range() -> std::ops::RangeInclusive { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zopfli", feature = "bzip2", feature = "zstd" ))] From 3161de632e8b3d4aac4184240e5b2e7db41ef966 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:06:33 -0700 Subject: [PATCH 236/281] Rewrite with new functional-update syntax --- src/write.rs | 59 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/write.rs b/src/write.rs index 13ef3811..8b2c5981 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1121,18 +1121,19 @@ impl GenericZipWriter { ))] CompressionMethod::Deflated => { #[cfg(all( - not(feature = "deflate"), - not(feature = "deflate-miniz"), - not(feature = "deflate-zlib"), - feature = "deflate-zopfli" + not(feature = "deflate"), + not(feature = "deflate-miniz"), + not(feature = "deflate-zlib"), + feature = "deflate-zopfli" ))] let default = 24; #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib"))] - let default = flate2::Compression::default().level() as i32; + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" + ))] + let default = Compression::default().level() as i32; let level = clamp_opt( compression_level.unwrap_or(default), @@ -1151,41 +1152,49 @@ impl GenericZipWriter { })); #[cfg(all( - not(feature = "deflate"), - not(feature = "deflate-miniz"), - not(feature = "deflate-zlib"), - feature = "deflate-zopfli" + not(feature = "deflate"), + not(feature = "deflate-miniz"), + not(feature = "deflate-zlib"), + feature = "deflate-zopfli" ))] return Ok(Box::new(move |bare| { let mut options = Options::default(); - options.iteration_count = NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); + options.iteration_count = + NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( options, Default::default(), - bare + bare, )) })); #[cfg(all( - any(feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib"), - feature = "deflate-zopfli") - )] + any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib" + ), + feature = "deflate-zopfli" + ))] Ok(Box::new(move |bare| { let best_non_zopfli = Compression::best().level(); if level > best_non_zopfli { - let mut options = Options::default(); - options.iteration_count = NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); + let options = Options { + iteration_count: NonZeroU8::try_from( + (level - best_non_zopfli) as u8, + ) + .unwrap(), + ..Default::default() + }; GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( options, Default::default(), - bare + bare, )) } else { GenericZipWriter::Deflater(DeflateEncoder::new( bare, - flate2::Compression::new(level), + Compression::new(level), )) } })) @@ -1306,7 +1315,7 @@ fn deflate_compression_level_range() -> std::ops::RangeInclusive { feature = "deflate-miniz", feature = "deflate-zlib", ))] - let min = flate2::Compression::none().level() as i32; + let min = Compression::none().level() as i32; #[cfg(not(any( feature = "deflate", @@ -1319,7 +1328,7 @@ fn deflate_compression_level_range() -> std::ops::RangeInclusive { let max = flate2::Compression::best().level() as i32; #[cfg(feature = "deflate-zopfli")] - let max = flate2::Compression::best().level() as i32 + u8::MAX as i32; + let max = Compression::best().level() as i32 + u8::MAX as i32; min..=max } From 291212a99c034a9ac5462a80e488f19cf04141db Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:07:14 -0700 Subject: [PATCH 237/281] Bump version to 0.9.1 --- CHANGELOG.md | 8 +++++++- Cargo.toml | 2 +- README.md | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f033712d..b4404d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -184,4 +184,10 @@ ### Added - - `flush_on_finish_file` parameter for `ZipWriter`. \ No newline at end of file + - `flush_on_finish_file` parameter for `ZipWriter`. + +## [0.9.1] + +### Added + + - Zopfli for aggressive Deflate compression. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 29ac853f..d7f95351 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.9.0" +version = "0.9.1" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index f8aa7671..76e83ad6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.9.0/zip_next/) +[Documentation](https://docs.rs/zip_next/0.9.1/zip_next/) Info ---- @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.9.0" +zip_next = "0.9.1" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.9.0", default-features = false } +zip_next = { version = "0.9.1", default-features = false } ``` The features available are: From 72a2f73496b9819934232e99789eb65b1436e404 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:18:01 -0700 Subject: [PATCH 238/281] Adjust timeout and job count for fuzz_write --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b2fa83ac..16ae3c05 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,7 +116,7 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=1s -jobs=100 -workers=2 -runs=100000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=10s -jobs=100 -workers=2 -runs=20000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 From b3988b2aecb8a2efb72393c3be2daee9efedc197 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:42:23 -0700 Subject: [PATCH 239/281] Refactor: short-circuit empty writes --- src/write.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/write.rs b/src/write.rs index 8b2c5981..eaff39d5 100644 --- a/src/write.rs +++ b/src/write.rs @@ -335,6 +335,9 @@ impl Write for ZipWriter { "No file has been started", )); } + if buf.len() == 0 { + return Ok(0); + } match self.inner.ref_mut() { Some(ref mut w) => { let write_result = w.write(buf); @@ -1559,6 +1562,7 @@ mod test { use std::io; use std::io::{Read, Write}; use std::sync::Arc; + use crate::CompressionMethod::Deflated; #[test] fn write_empty_zip() { @@ -1974,6 +1978,18 @@ mod test { writer.start_file("", FileOptions::default()).unwrap(); Ok(()) } + + #[cfg(feature = "deflate-zopfli")] + #[test] + fn zopfli_empty_write() -> ZipResult<()> { + let mut options = FileOptions::default(); + options = options.compression_method(Deflated).compression_level(Some(264)); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + writer.start_file("", options).unwrap(); + writer.write_all(&[]).unwrap(); + writer.write_all(&[]).unwrap(); + Ok(()) + } } #[cfg(not(feature = "unreserved"))] From 8804a46cab2f138b981d896a4086c2957296a24f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:45:01 -0700 Subject: [PATCH 240/281] Fix unnecessary path qualification --- src/write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index eaff39d5..65c29e51 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1716,7 +1716,7 @@ mod test { fn test_shallow_copy() { let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); let options = FileOptions { - compression_method: CompressionMethod::Deflated, + compression_method: Deflated, compression_level: Some(9), last_modified_time: DateTime::default(), permissions: Some(33188), @@ -1766,7 +1766,7 @@ mod test { fn test_deep_copy() { let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); let options = FileOptions { - compression_method: CompressionMethod::Deflated, + compression_method: Deflated, compression_level: Some(9), last_modified_time: DateTime::default(), permissions: Some(33188), From 8f49b0bb58dd7a3c58fd1fd1abd52a70637237d2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 14:47:36 -0700 Subject: [PATCH 241/281] Fix Clippy warning and formatting --- src/write.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/write.rs b/src/write.rs index 65c29e51..58c2b616 100644 --- a/src/write.rs +++ b/src/write.rs @@ -335,7 +335,7 @@ impl Write for ZipWriter { "No file has been started", )); } - if buf.len() == 0 { + if buf.is_empty() { return Ok(0); } match self.inner.ref_mut() { @@ -1558,11 +1558,11 @@ mod test { use crate::compression::CompressionMethod; use crate::result::ZipResult; use crate::types::DateTime; + use crate::CompressionMethod::Deflated; use crate::ZipArchive; use std::io; use std::io::{Read, Write}; use std::sync::Arc; - use crate::CompressionMethod::Deflated; #[test] fn write_empty_zip() { @@ -1983,7 +1983,9 @@ mod test { #[test] fn zopfli_empty_write() -> ZipResult<()> { let mut options = FileOptions::default(); - options = options.compression_method(Deflated).compression_level(Some(264)); + options = options + .compression_method(Deflated) + .compression_level(Some(264)); let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); writer.start_file("", options).unwrap(); writer.write_all(&[]).unwrap(); From 130ca38cf6fb83c5af1f49b0e5c41e097f315dbd Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 15:44:43 -0700 Subject: [PATCH 242/281] Implement adjustable buffer size for Zopfli --- .github/workflows/ci.yaml | 2 +- src/result.rs | 7 +++ src/write.rs | 97 ++++++++++++++++++++++++++++++--------- 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 16ae3c05..a676dc03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,7 +116,7 @@ jobs: cargo fuzz build fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=10s -jobs=100 -workers=2 -runs=20000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=10000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 diff --git a/src/result.rs b/src/result.rs index 11b10ad0..749f3354 100644 --- a/src/result.rs +++ b/src/result.rs @@ -3,6 +3,7 @@ use std::error::Error; use std::fmt; use std::io; +use std::io::IntoInnerError; /// Generic result type with ZipError as its error variant pub type ZipResult = Result; @@ -41,6 +42,12 @@ impl From for ZipError { } } +impl From> for ZipError { + fn from(value: IntoInnerError) -> Self { + ZipError::Io(value.into_error()) + } +} + impl fmt::Display for ZipError { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self { diff --git a/src/write.rs b/src/write.rs index 58c2b616..ce978a87 100644 --- a/src/write.rs +++ b/src/write.rs @@ -12,7 +12,7 @@ use std::convert::TryInto; use std::default::Default; use std::io; use std::io::prelude::*; -use std::io::{BufReader, SeekFrom}; +use std::io::{BufReader, BufWriter, SeekFrom}; use std::mem; use std::num::NonZeroU8; use std::str::{from_utf8, Utf8Error}; @@ -65,6 +65,8 @@ enum GenericZipWriter { Deflater(DeflateEncoder>), #[cfg(feature = "deflate-zopfli")] ZopfliDeflater(zopfli::DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(BufWriter>>), #[cfg(feature = "bzip2")] Bzip2(BzEncoder>), #[cfg(feature = "zstd")] @@ -144,6 +146,8 @@ pub struct FileOptions { extra_data: Arc>, central_extra_data: Arc>, alignment: u16, + #[cfg(feature = "deflate-zopfli")] + pub(super) zopfli_buffer_size: Option, } #[cfg(fuzzing)] @@ -159,6 +163,8 @@ impl arbitrary::Arbitrary<'_> for FileOptions { extra_data: Arc::new(vec![]), central_extra_data: Arc::new(vec![]), alignment: u16::arbitrary(u)?, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << u.int_in_range(10..=30)?), }; u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { options @@ -294,6 +300,17 @@ impl FileOptions { self.alignment = alignment; self } + + /// Sets the size of the buffer used to hold the next block that Zopfli will compress. The + /// larger the buffer, the more effective the compression, but the more memory is required. + /// A value of `None` indicates no buffer, which is recommended only when all non-empty writes + /// are larger than about 32 KiB. + #[must_use] + #[cfg(feature = "deflate-zopfli")] + pub fn with_zopfli_buffer(mut self, size: Option) -> FileOptions { + self.zopfli_buffer_size = size; + self + } } impl Default for FileOptions { @@ -323,6 +340,8 @@ impl Default for FileOptions { extra_data: Arc::new(vec![]), central_extra_data: Arc::new(vec![]), alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), } } } @@ -448,6 +467,8 @@ impl ZipWriter { extra_data: src_data.extra_field.clone(), central_extra_data: src_data.central_extra_field.clone(), alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, }; if let Some(perms) = src_data.unix_mode() { options = options.unix_permissions(perms); @@ -491,7 +512,8 @@ impl ZipWriter { /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to /// read flushed bytes and not overwrite them. Do not enable this with a [BufWriter], because - /// that implicitly calls [`Writer::flush`] whenever [`Seek::seek`] is called. + /// that implicitly calls [`Writer::flush`] whenever [`Seek::seek`] is called. Likewise, when + /// using Deflate compression, set [] pub fn new(inner: W, flush_on_finish_file: bool) -> ZipWriter { ZipWriter { inner: Storer(MaybeEncrypted::Unencrypted(inner)), @@ -698,9 +720,9 @@ impl ZipWriter { return Ok(()); } - let make_plain_writer = self - .inner - .prepare_next_writer(CompressionMethod::Stored, None)?; + let make_plain_writer = + self.inner + .prepare_next_writer(CompressionMethod::Stored, None, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; let writer = self.inner.get_plain(); @@ -750,9 +772,9 @@ impl ZipWriter { pub fn abort_file(&mut self) -> ZipResult<()> { let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; self.files_by_name.remove(&last_file.file_name); - let make_plain_writer = self - .inner - .prepare_next_writer(CompressionMethod::Stored, None)?; + let make_plain_writer = + self.inner + .prepare_next_writer(CompressionMethod::Stored, None, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd @@ -779,9 +801,12 @@ impl ZipWriter { S: Into, { Self::normalize_options(&mut options); - let make_new_self = self - .inner - .prepare_next_writer(options.compression_method, options.compression_level)?; + let make_new_self = self.inner.prepare_next_writer( + options.compression_method, + options.compression_level, + #[cfg(feature = "deflate-zopfli")] + options.zopfli_buffer_size, + )?; self.start_entry(name, options, None)?; if let Err(e) = self.inner.switch_to(make_new_self) { self.abort_file().unwrap(); @@ -1097,6 +1122,8 @@ impl GenericZipWriter { &self, compression: CompressionMethod, compression_level: Option, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Option, ) -> ZipResult> { if let Closed = self { return Err( @@ -1164,11 +1191,19 @@ impl GenericZipWriter { let mut options = Options::default(); options.iteration_count = NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); - GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( - options, - Default::default(), - bare, - )) + match deflate_buffer_size { + Some(size) => { + GenericZipWriter::BufferedZopfliDeflater(BufWriter::with_capacity( + size, + zopfli::DeflateEncoder::new(options, Default::default(), bare), + )) + } + None => GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + )), + } })); #[cfg(all( @@ -1189,11 +1224,21 @@ impl GenericZipWriter { .unwrap(), ..Default::default() }; - GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( - options, - Default::default(), - bare, - )) + match zopfli_buffer_size { + Some(size) => GenericZipWriter::BufferedZopfliDeflater( + BufWriter::with_capacity( + size, + zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + ), + ), + ), + None => GenericZipWriter::ZopfliDeflater( + zopfli::DeflateEncoder::new(options, Default::default(), bare), + ), + } } else { GenericZipWriter::Deflater(DeflateEncoder::new( bare, @@ -1252,6 +1297,8 @@ impl GenericZipWriter { GenericZipWriter::Deflater(w) => w.finish()?, #[cfg(feature = "deflate-zopfli")] GenericZipWriter::ZopfliDeflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + GenericZipWriter::BufferedZopfliDeflater(w) => w.into_inner()?.finish()?, #[cfg(feature = "bzip2")] GenericZipWriter::Bzip2(w) => w.finish()?, #[cfg(feature = "zstd")] @@ -1279,6 +1326,8 @@ impl GenericZipWriter { GenericZipWriter::Deflater(ref mut w) => Some(w as &mut dyn Write), #[cfg(feature = "deflate-zopfli")] GenericZipWriter::ZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + GenericZipWriter::BufferedZopfliDeflater(w) => Some(w as &mut dyn Write), #[cfg(feature = "bzip2")] GenericZipWriter::Bzip2(ref mut w) => Some(w as &mut dyn Write), #[cfg(feature = "zstd")] @@ -1685,6 +1734,8 @@ mod test { extra_data: Arc::new(vec![]), central_extra_data: Arc::new(vec![]), alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, }; writer.start_file("mimetype", options).unwrap(); writer @@ -1725,6 +1776,8 @@ mod test { extra_data: Arc::new(vec![]), central_extra_data: Arc::new(vec![]), alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); @@ -1775,6 +1828,8 @@ mod test { extra_data: Arc::new(vec![]), central_extra_data: Arc::new(vec![]), alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, }; writer.start_file(RT_TEST_FILENAME, options).unwrap(); writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); From ed922b060a39f44c5c1c990c6db5b1dee85bbba2 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 15:45:17 -0700 Subject: [PATCH 243/281] Reformat --- src/write.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index ce978a87..8af489ce 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1122,8 +1122,7 @@ impl GenericZipWriter { &self, compression: CompressionMethod, compression_level: Option, - #[cfg(feature = "deflate-zopfli")] - zopfli_buffer_size: Option, + #[cfg(feature = "deflate-zopfli")] zopfli_buffer_size: Option, ) -> ZipResult> { if let Closed = self { return Err( From 543f469277ca8b56524687cfaf31d0fd9c90fd01 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 16:47:44 -0700 Subject: [PATCH 244/281] Refactor: zopfli_buffer_size doesn't matter when not using Zopfli --- src/write.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8af489ce..985946f1 100644 --- a/src/write.rs +++ b/src/write.rs @@ -120,6 +120,7 @@ use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; use crate::zipcrypto::ZipCryptoKeys; pub use zip_writer::ZipWriter; +use crate::CompressionMethod::Deflated; #[derive(Default)] struct ZipWriterStats { @@ -164,8 +165,13 @@ impl arbitrary::Arbitrary<'_> for FileOptions { central_extra_data: Arc::new(vec![]), alignment: u16::arbitrary(u)?, #[cfg(feature = "deflate-zopfli")] - zopfli_buffer_size: Some(1 << u.int_in_range(10..=30)?), + zopfli_buffer_size: None, }; + #[cfg(feature = "deflate-zopfli")] + if options.compression_method == Deflated + && options.compression_level > Compression::best().level() { + options.zopfli_buffer_size = Some(1 << u.int_in_range(10..=30)?); + } u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { options .add_extra_data( @@ -322,11 +328,12 @@ impl Default for FileOptions { feature = "deflate-miniz", feature = "deflate-zlib" ))] - compression_method: CompressionMethod::Deflated, + compression_method: Deflated, #[cfg(not(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zopfli" )))] compression_method: CompressionMethod::Stored, compression_level: None, @@ -1148,7 +1155,7 @@ impl GenericZipWriter { feature = "deflate-zlib", feature = "deflate-zopfli" ))] - CompressionMethod::Deflated => { + Deflated => { #[cfg(all( not(feature = "deflate"), not(feature = "deflate-miniz"), From 7e6472c761b3c559292606d6f44ee08d11cb914e Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 16:55:21 -0700 Subject: [PATCH 245/281] Refactor: fuzz with a compression level that matches the method or is just past the edge --- src/write.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/write.rs b/src/write.rs index 985946f1..de2238a7 100644 --- a/src/write.rs +++ b/src/write.rs @@ -156,7 +156,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { let mut options = FileOptions { compression_method: CompressionMethod::arbitrary(u)?, - compression_level: Option::::arbitrary(u)?, + compression_level: None, last_modified_time: DateTime::arbitrary(u)?, permissions: Option::::arbitrary(u)?, large_file: bool::arbitrary(u)?, @@ -167,10 +167,15 @@ impl arbitrary::Arbitrary<'_> for FileOptions { #[cfg(feature = "deflate-zopfli")] zopfli_buffer_size: None, }; - #[cfg(feature = "deflate-zopfli")] - if options.compression_method == Deflated - && options.compression_level > Compression::best().level() { - options.zopfli_buffer_size = Some(1 << u.int_in_range(10..=30)?); + if options.compression_method == Deflated { + options.compression_level = Some(u.int_in_range(0..=265)?); + if options.compression_level > Compression::best().level() { + options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); + } + } else if options.compression_method != Stored { + options.compression_level = Some(u.int_in_range(0..=10)?); + } else if bool::arbitrary(u) { + options.compression_level = Some(1); } u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { options From 744068b1173e6ada100c15b9abc791b86eeeebdf Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 16:56:23 -0700 Subject: [PATCH 246/281] Reformat --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index de2238a7..575135d3 100644 --- a/src/write.rs +++ b/src/write.rs @@ -119,8 +119,8 @@ pub(crate) mod zip_writer { use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; use crate::zipcrypto::ZipCryptoKeys; -pub use zip_writer::ZipWriter; use crate::CompressionMethod::Deflated; +pub use zip_writer::ZipWriter; #[derive(Default)] struct ZipWriterStats { From 3414e0d6fb56f0f24552a57cf788a7bed080f0cb Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 16:59:07 -0700 Subject: [PATCH 247/281] Bug fix --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 575135d3..8555658a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -174,7 +174,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { } } else if options.compression_method != Stored { options.compression_level = Some(u.int_in_range(0..=10)?); - } else if bool::arbitrary(u) { + } else if bool::arbitrary(u)? { options.compression_level = Some(1); } u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { From f46a0055ade0a045f4fe4b8a71ad1c7fe019a171 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:04:27 -0700 Subject: [PATCH 248/281] Bug fixes and refactors for Arbitrary impl --- src/write.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8555658a..8fe8d8ee 100644 --- a/src/write.rs +++ b/src/write.rs @@ -167,15 +167,26 @@ impl arbitrary::Arbitrary<'_> for FileOptions { #[cfg(feature = "deflate-zopfli")] zopfli_buffer_size: None, }; - if options.compression_method == Deflated { - options.compression_level = Some(u.int_in_range(0..=265)?); - if options.compression_level > Compression::best().level() { - options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); + match options.compression_method { + Deflated => { + if bool::arbitrary(u)? { + let level = u.int_in_range(0..=265)?; + options.compression_level = Some(level); + if level > Compression::best().level() { + options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); + } + } + }, + Stored => { + if bool::arbitrary(u)? { + options.compression_level = Some(1); + } + }, + _ => { + if bool::arbitrary(u)? { + options.compression_level = Some(u.int_in_range(0..=10)?); + } } - } else if options.compression_method != Stored { - options.compression_level = Some(u.int_in_range(0..=10)?); - } else if bool::arbitrary(u)? { - options.compression_level = Some(1); } u.arbitrary_loop(Some(0), Some((u16::MAX / 4) as u32), |u| { options From d2e2b546255bac8d3844973d644525dcc00d5cee Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:05:49 -0700 Subject: [PATCH 249/281] Reformat --- src/write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8fe8d8ee..374f237c 100644 --- a/src/write.rs +++ b/src/write.rs @@ -176,12 +176,12 @@ impl arbitrary::Arbitrary<'_> for FileOptions { options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); } } - }, + } Stored => { if bool::arbitrary(u)? { options.compression_level = Some(1); } - }, + } _ => { if bool::arbitrary(u)? { options.compression_level = Some(u.int_in_range(0..=10)?); From fd081c9f8c1b0090943fdb55f0239e01a7ff76e6 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:06:37 -0700 Subject: [PATCH 250/281] Bug fix for fuzzing with deflate-zopfli disabled --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index 374f237c..9e9571a5 100644 --- a/src/write.rs +++ b/src/write.rs @@ -172,6 +172,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { if bool::arbitrary(u)? { let level = u.int_in_range(0..=265)?; options.compression_level = Some(level); + #[cfg(feature = "deflate-zopfli")] if level > Compression::best().level() { options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); } From e30f57f344479240733f7008d0f17c4424eed09b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:28:30 -0700 Subject: [PATCH 251/281] Fix up merge of https://github.com/Pr0methean/zip-next/pull/8 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d7f95351..fd7d97bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ hmac = { version = "0.12.1", optional = true, features = ["reset"] } pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } time = { version = "0.3.21", optional = true, default-features = false, features = ["std"] } -zstd = { version = "0.12.3", optional = true } +zstd = { version = "0.12.3", optional = true, default-features = false } zopfli = { version = "0.7.4", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] From 6ec619d78ae1cb4eb50863d0cd7b9a0b60f956e9 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:30:50 -0700 Subject: [PATCH 252/281] Bug fix --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 9e9571a5..8f55cd72 100644 --- a/src/write.rs +++ b/src/write.rs @@ -173,7 +173,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { let level = u.int_in_range(0..=265)?; options.compression_level = Some(level); #[cfg(feature = "deflate-zopfli")] - if level > Compression::best().level() { + if level > Compression::best().level().try_into().unwrap() { options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); } } From dd9ea0689afbd65d1ae9a9482c2e88f9800735df Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:36:31 -0700 Subject: [PATCH 253/281] Bug fix --- src/write.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8f55cd72..2227bdef 100644 --- a/src/write.rs +++ b/src/write.rs @@ -119,7 +119,7 @@ pub(crate) mod zip_writer { use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; use crate::zipcrypto::ZipCryptoKeys; -use crate::CompressionMethod::Deflated; +use crate::CompressionMethod::{Deflated, Stored}; pub use zip_writer::ZipWriter; #[derive(Default)] @@ -352,7 +352,7 @@ impl Default for FileOptions { feature = "deflate-zlib", feature = "deflate-zopfli" )))] - compression_method: CompressionMethod::Stored, + compression_method: Stored, compression_level: None, #[cfg(feature = "time")] last_modified_time: OffsetDateTime::now_utc().try_into().unwrap_or_default(), @@ -746,7 +746,7 @@ impl ZipWriter { let make_plain_writer = self.inner - .prepare_next_writer(CompressionMethod::Stored, None, None)?; + .prepare_next_writer(Stored, None, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; let writer = self.inner.get_plain(); @@ -798,7 +798,7 @@ impl ZipWriter { self.files_by_name.remove(&last_file.file_name); let make_plain_writer = self.inner - .prepare_next_writer(CompressionMethod::Stored, None, None)?; + .prepare_next_writer(Stored, None, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd @@ -959,7 +959,7 @@ impl ZipWriter { options.permissions = Some(0o755); } *options.permissions.as_mut().unwrap() |= 0o40000; - options.compression_method = CompressionMethod::Stored; + options.compression_method = Stored; let name_as_string = name.into(); // Append a slash to the filename if it does not end with it. @@ -1028,7 +1028,7 @@ impl ZipWriter { *options.permissions.as_mut().unwrap() |= 0o120000; // The symlink target is stored as file content. And compressing the target path // likely wastes space. So always store. - options.compression_method = CompressionMethod::Stored; + options.compression_method = Stored; self.start_entry(name, options, None)?; self.writing_to_file = true; @@ -1157,7 +1157,7 @@ impl GenericZipWriter { { #[allow(deprecated)] match compression { - CompressionMethod::Stored => { + Stored => { if compression_level.is_some() { Err(ZipError::UnsupportedArchive( "Unsupported compression level", From 26eeabe488587e1fd13f0af5aacb74c667159d0f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 27 May 2023 17:39:14 -0700 Subject: [PATCH 254/281] Reformat --- src/write.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/write.rs b/src/write.rs index 2227bdef..e1815aa0 100644 --- a/src/write.rs +++ b/src/write.rs @@ -744,9 +744,7 @@ impl ZipWriter { return Ok(()); } - let make_plain_writer = - self.inner - .prepare_next_writer(Stored, None, None)?; + let make_plain_writer = self.inner.prepare_next_writer(Stored, None, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; let writer = self.inner.get_plain(); @@ -796,9 +794,7 @@ impl ZipWriter { pub fn abort_file(&mut self) -> ZipResult<()> { let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; self.files_by_name.remove(&last_file.file_name); - let make_plain_writer = - self.inner - .prepare_next_writer(Stored, None, None)?; + let make_plain_writer = self.inner.prepare_next_writer(Stored, None, None)?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd From d9c20c55c412ff7f73ce06cf39ab7cf5317be34c Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 28 May 2023 10:11:51 -0700 Subject: [PATCH 255/281] Limit Zopfli iterations in write fuzzing --- fuzz/fuzz_targets/fuzz_write.rs | 3 ++- src/write.rs | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 6961e9df..5aa0ab4b 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -54,7 +54,8 @@ fn do_operation(writer: &mut RefCell>, let name = operation.name; match operation.basic { BasicFileOperation::WriteNormalFile {contents, mut options, ..} => { - if contents.iter().map(Vec::len).sum::() >= u32::MAX as usize { + let uncompressed_size = contents.iter().map(Vec::len).sum::(); + if uncompressed_size >= u32::MAX as usize { options = options.large_file(true); } writer.borrow_mut().start_file(name, options)?; diff --git a/src/write.rs b/src/write.rs index e1815aa0..e9eb2a15 100644 --- a/src/write.rs +++ b/src/write.rs @@ -170,7 +170,7 @@ impl arbitrary::Arbitrary<'_> for FileOptions { match options.compression_method { Deflated => { if bool::arbitrary(u)? { - let level = u.int_in_range(0..=265)?; + let level = u.int_in_range(0..=24)?; options.compression_level = Some(level); #[cfg(feature = "deflate-zopfli")] if level > Compression::best().level().try_into().unwrap() { @@ -334,6 +334,11 @@ impl FileOptions { self.zopfli_buffer_size = size; self } + + /// Returns the compression level currently set. + pub fn get_compression_level(&self) -> Option { + self.compression_level + } } impl Default for FileOptions { From 9ec0ddc5ca09bb2a73374941bef48615267297fb Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 09:18:26 -0700 Subject: [PATCH 256/281] Add zlib-ng --- CHANGELOG.md | 8 ++- Cargo.toml | 5 +- README.md | 16 +++-- examples/write_dir.rs | 7 ++- src/compression.rs | 6 ++ src/read.rs | 15 +++-- src/write.rs | 142 +++++++++++++++++------------------------- 7 files changed, 100 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4404d33..ebf0d55f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,4 +190,10 @@ ### Added - - Zopfli for aggressive Deflate compression. \ No newline at end of file + - Zopfli for aggressive Deflate compression. + +## [0.9.2] + +### Added + + - `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index fd7d97bc..63b0c17d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.9.1" +version = "0.9.2" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" @@ -43,9 +43,10 @@ aes-crypto = [ "aes", "constant_time_eq", "hmac", "pbkdf2", "sha1" ] deflate = ["flate2/rust_backend"] deflate-miniz = ["flate2/default"] deflate-zlib = ["flate2/zlib"] +deflate-zlib-ng = ["flate2/zlib-ng"] deflate-zopfli = ["zopfli"] unreserved = [] -default = ["aes-crypto", "bzip2", "deflate", "deflate-zopfli", "time", "zstd"] +default = ["aes-crypto", "bzip2", "deflate", "deflate-zlib-ng", "deflate-zopfli", "time", "zstd"] [[bench]] name = "read_entry" diff --git a/README.md b/README.md index 76e83ad6..9c747911 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.9.1/zip_next/) +[Documentation](https://docs.rs/zip_next/0.9.2/zip_next/) Info ---- @@ -32,25 +32,31 @@ With all default features: ```toml [dependencies] -zip_next = "0.9.1" +zip_next = "0.9.2" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.9.1", default-features = false } +zip_next = { version = "0.9.2", default-features = false } ``` The features available are: * `aes-crypto`: Enables decryption of files which were encrypted with AES. Supports AE-1 and AE-2 methods. -* `deflate`: Enables the deflate compression algorithm, which is the default for zip files. +* `deflate`: Enables decompressing the deflate compression algorithm, which is the default for zip files. +* `deflate-miniz`: Enables deflating files with the `miniz_oxide` library (used when compression quality is 0..=9). +* `deflate-zlib`: Enables deflating files with the `zlib` library (used when compression quality is 0..=9). +* `deflate-zlib-ng`: Enables deflating files with the `zlib-ng` library (used when compression quality is 0..=9). + This is the fastest `deflate` implementation available. +* `deflate-zopfli`: Enables deflating files with the `zopfli` library (used when compression quality is 10..=264). This + is the most effective `deflate` implementation available. * `bzip2`: Enables the BZip2 compression algorithm. * `time`: Enables features using the [time](https://github.com/rust-lang-deprecated/time) crate. * `zstd`: Enables the Zstandard compression algorithm. -All of these are enabled by default. +By default `aes-crypto`, `deflate`, `deflate-zlib-ng`, `deflate-zopfli`, `bzip2`, `time` and `zstd` are enabled. MSRV ---- diff --git a/examples/write_dir.rs b/examples/write_dir.rs index 61535c34..a7f14e35 100644 --- a/examples/write_dir.rs +++ b/examples/write_dir.rs @@ -18,14 +18,17 @@ const METHOD_STORED: Option = #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] const METHOD_DEFLATED: Option = Some(zip_next::CompressionMethod::Deflated); #[cfg(not(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli" )))] const METHOD_DEFLATED: Option = None; diff --git a/src/compression.rs b/src/compression.rs index 4d568bf0..e23d743f 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -21,6 +21,7 @@ pub enum CompressionMethod { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] Deflated, @@ -57,6 +58,7 @@ impl CompressionMethod { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] pub const DEFLATE: Self = CompressionMethod::Deflated; @@ -64,6 +66,7 @@ impl CompressionMethod { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" )))] pub const DEFLATE: Self = CompressionMethod::Unsupported(8); @@ -105,6 +108,7 @@ impl CompressionMethod { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] 8 => CompressionMethod::Deflated, @@ -132,6 +136,7 @@ impl CompressionMethod { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] CompressionMethod::Deflated => 8, @@ -161,6 +166,7 @@ pub const SUPPORTED_COMPRESSION_METHODS: &[CompressionMethod] = &[ feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] CompressionMethod::Deflated, diff --git a/src/read.rs b/src/read.rs index 5cbba0ec..f20d4662 100644 --- a/src/read.rs +++ b/src/read.rs @@ -19,7 +19,8 @@ use std::sync::Arc; #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] use flate2::read::DeflateDecoder; @@ -126,7 +127,8 @@ pub(crate) enum ZipFileReader<'a> { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] Deflated(Crc32Reader>>), #[cfg(feature = "bzip2")] @@ -144,7 +146,8 @@ impl<'a> Read for ZipFileReader<'a> { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] ZipFileReader::Deflated(r) => r.read(buf), #[cfg(feature = "bzip2")] @@ -165,7 +168,8 @@ impl<'a> ZipFileReader<'a> { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] ZipFileReader::Deflated(r) => r.into_inner().into_inner().into_inner(), #[cfg(feature = "bzip2")] @@ -271,7 +275,8 @@ pub(crate) fn make_reader( #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] CompressionMethod::Deflated => { let deflate_reader = DeflateDecoder::new(reader); diff --git a/src/write.rs b/src/write.rs index e9eb2a15..5ed7c78a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -21,7 +21,8 @@ use std::sync::Arc; #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] use flate2::write::DeflateEncoder; @@ -60,7 +61,8 @@ enum GenericZipWriter { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] Deflater(DeflateEncoder>), #[cfg(feature = "deflate-zopfli")] @@ -348,13 +350,16 @@ impl Default for FileOptions { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli" ))] compression_method: Deflated, #[cfg(not(any( feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" )))] compression_method: Stored, @@ -1157,6 +1162,7 @@ impl GenericZipWriter { { #[allow(deprecated)] + #[allow(unreachable_code)] match compression { Stored => { if compression_level.is_some() { @@ -1171,23 +1177,19 @@ impl GenericZipWriter { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] Deflated => { - #[cfg(all( - not(feature = "deflate"), - not(feature = "deflate-miniz"), - not(feature = "deflate-zlib"), - feature = "deflate-zopfli" - ))] - let default = 24; - - #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" - ))] - let default = Compression::default().level() as i32; + let default = if cfg!(feature = "deflate") + || cfg!(feature = "deflate-miniz") + || cfg!(feature = "deflate-zlib") + || cfg!(feature = "deflate-zlib-ng") + { + Compression::default().level() as i32 + } else { + 24 + }; let level = clamp_opt( compression_level.unwrap_or(default), @@ -1197,48 +1199,8 @@ impl GenericZipWriter { "Unsupported compression level", ))? as u32; - #[cfg(not(feature = "deflate-zopfli"))] - return Ok(Box::new(move |bare| { - GenericZipWriter::Deflater(DeflateEncoder::new( - bare, - flate2::Compression::new(level), - )) - })); - - #[cfg(all( - not(feature = "deflate"), - not(feature = "deflate-miniz"), - not(feature = "deflate-zlib"), - feature = "deflate-zopfli" - ))] - return Ok(Box::new(move |bare| { - let mut options = Options::default(); - options.iteration_count = - NonZeroU8::try_from((level - best_non_zopfli) as u8).unwrap(); - match deflate_buffer_size { - Some(size) => { - GenericZipWriter::BufferedZopfliDeflater(BufWriter::with_capacity( - size, - zopfli::DeflateEncoder::new(options, Default::default(), bare), - )) - } - None => GenericZipWriter::ZopfliDeflater(zopfli::DeflateEncoder::new( - options, - Default::default(), - bare, - )), - } - })); - - #[cfg(all( - any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib" - ), - feature = "deflate-zopfli" - ))] - Ok(Box::new(move |bare| { + #[cfg(feature = "deflate-zopfli")] + { let best_non_zopfli = Compression::best().level(); if level > best_non_zopfli { let options = Options { @@ -1248,7 +1210,7 @@ impl GenericZipWriter { .unwrap(), ..Default::default() }; - match zopfli_buffer_size { + return Ok(Box::new(move |bare| match zopfli_buffer_size { Some(size) => GenericZipWriter::BufferedZopfliDeflater( BufWriter::with_capacity( size, @@ -1262,14 +1224,25 @@ impl GenericZipWriter { None => GenericZipWriter::ZopfliDeflater( zopfli::DeflateEncoder::new(options, Default::default(), bare), ), - } - } else { + })); + } + } + + #[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + ))] + { + return Ok(Box::new(move |bare| { GenericZipWriter::Deflater(DeflateEncoder::new( bare, Compression::new(level), )) - } - })) + })); + } + unreachable!() } #[cfg(feature = "bzip2")] CompressionMethod::Bzip2 => { @@ -1316,7 +1289,8 @@ impl GenericZipWriter { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] GenericZipWriter::Deflater(w) => w.finish()?, #[cfg(feature = "deflate-zopfli")] @@ -1345,7 +1319,8 @@ impl GenericZipWriter { #[cfg(any( feature = "deflate", feature = "deflate-miniz", - feature = "deflate-zlib" + feature = "deflate-zlib", + feature = "deflate-zlib-ng" ))] GenericZipWriter::Deflater(ref mut w) => Some(w as &mut dyn Write), #[cfg(feature = "deflate-zopfli")] @@ -1383,28 +1358,26 @@ impl GenericZipWriter { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] fn deflate_compression_level_range() -> std::ops::RangeInclusive { - #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib", - ))] - let min = Compression::none().level() as i32; + let min = if cfg!(feature = "deflate") + || cfg!(feature = "deflate-miniz") + || cfg!(feature = "deflate-zlib") + || cfg!(feature = "deflate-zlib-ng") + { + Compression::none().level() as i32 + } else { + Compression::best().level() as i32 + 1 + }; - #[cfg(not(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib", - )))] - let min = flate2::Compression::best().level() + 1 as i32; - - #[cfg(not(feature = "deflate-zopfli"))] - let max = flate2::Compression::best().level() as i32; - - #[cfg(feature = "deflate-zopfli")] - let max = Compression::best().level() as i32 + u8::MAX as i32; + let max = Compression::best().level() as i32 + + if cfg!(feature = "deflate-zopfli") { + u8::MAX as i32 + } else { + 0 + }; min..=max } @@ -1420,6 +1393,7 @@ fn bzip2_compression_level_range() -> std::ops::RangeInclusive { feature = "deflate", feature = "deflate-miniz", feature = "deflate-zlib", + feature = "deflate-zlib-ng", feature = "deflate-zopfli", feature = "bzip2", feature = "zstd" From b47d6419b8909586d38ab841c2a19698ad0e8d97 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 09:48:39 -0700 Subject: [PATCH 257/281] Add chrono integration --- CHANGELOG.md | 3 ++- Cargo.toml | 2 ++ README.md | 2 ++ src/types.rs | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf0d55f..a0d7010b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -196,4 +196,5 @@ ### Added - - `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. \ No newline at end of file + - `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. + - `chrono` to convert zip_next::DateTime to and from chrono::NaiveDateTime \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 63b0c17d..84204c3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ edition = "2021" aes = { version = "0.8.2", optional = true } byteorder = "1.4.3" bzip2 = { version = "0.4.4", optional = true } +chrono = { version = "0.4.25", optional = true } constant_time_eq = { version = "0.2.5", optional = true } crc32fast = "1.3.2" flate2 = { version = "1.0.26", default-features = false, optional = true } @@ -40,6 +41,7 @@ time = { version = "0.3.21", features = ["formatting", "macros"] } [features] aes-crypto = [ "aes", "constant_time_eq", "hmac", "pbkdf2", "sha1" ] +chrono = ["chrono/default"] deflate = ["flate2/rust_backend"] deflate-miniz = ["flate2/default"] deflate-zlib = ["flate2/zlib"] diff --git a/README.md b/README.md index 9c747911..566fb09e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ The features available are: is the most effective `deflate` implementation available. * `bzip2`: Enables the BZip2 compression algorithm. * `time`: Enables features using the [time](https://github.com/rust-lang-deprecated/time) crate. +* `chrono`: Enables converting last-modified `zip_next::DateTime` to and from `chrono::NaiveDateTime` and from + `chrono::DateTime`. * `zstd`: Enables the Zstandard compression algorithm. By default `aes-crypto`, `deflate`, `deflate-zlib-ng`, `deflate-zopfli`, `bzip2`, `time` and `zstd` are enabled. diff --git a/src/types.rs b/src/types.rs index 661cff4e..3ab8a02f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -3,6 +3,8 @@ use path::{Component, Path, PathBuf}; use std::path; use std::sync::Arc; +#[cfg(feature = "chrono")] +use chrono::{Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; #[cfg(not(any( all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", @@ -116,6 +118,39 @@ impl arbitrary::Arbitrary<'_> for DateTime { } } +#[cfg(feature = "chrono")] +#[allow(clippy::result_unit_err)] +impl TryFrom for DateTime +where + T: Datelike + Timelike, +{ + type Error = (); + + fn try_from(value: T) -> Result { + Ok(DateTime::from_date_and_time( + value.year().try_into()?, + value.month().try_into()?, + value.day().try_into()?, + value.hour().try_into()?, + value.minute().try_into()?, + value.second().try_into()?, + )?) + } +} + +#[cfg(feature = "chrono")] +#[allow(clippy::result_unit_err)] +impl TryInto for DateTime { + type Error = (); + + fn try_into(self) -> Result { + Ok(NaiveDateTime::new( + NaiveDate::from_ymd_opt(self.year.into(), self.month.into(), self.day.into())?, + NaiveTime::from_hms_opt(self.hour.into(), self.minute.into(), self.second.into())?, + )) + } +} + impl Default for DateTime { /// Constructs an 'default' datetime of 1980-01-01 00:00:00 fn default() -> DateTime { From 630ca3fa0f23a469f0c1c77feb24979d57c0e4aa Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 10:06:20 -0700 Subject: [PATCH 258/281] Bug fixes for chrono integration --- README.md | 3 +-- src/result.rs | 8 ++++++++ src/types.rs | 13 +++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 566fb09e..38b93134 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,7 @@ The features available are: is the most effective `deflate` implementation available. * `bzip2`: Enables the BZip2 compression algorithm. * `time`: Enables features using the [time](https://github.com/rust-lang-deprecated/time) crate. -* `chrono`: Enables converting last-modified `zip_next::DateTime` to and from `chrono::NaiveDateTime` and from - `chrono::DateTime`. +* `chrono`: Enables converting last-modified `zip_next::DateTime` to and from `chrono::NaiveDateTime`. * `zstd`: Enables the Zstandard compression algorithm. By default `aes-crypto`, `deflate`, `deflate-zlib-ng`, `deflate-zopfli`, `bzip2`, `time` and `zstd` are enabled. diff --git a/src/result.rs b/src/result.rs index 749f3354..0c4776eb 100644 --- a/src/result.rs +++ b/src/result.rs @@ -4,6 +4,7 @@ use std::error::Error; use std::fmt; use std::io; use std::io::IntoInnerError; +use std::num::TryFromIntError; /// Generic result type with ZipError as its error variant pub type ZipResult = Result; @@ -93,6 +94,13 @@ impl From for io::Error { #[derive(Debug)] pub struct DateTimeRangeError; +// TryFromIntError is also an out-of-range error. +impl From for DateTimeRangeError { + fn from(_value: TryFromIntError) -> Self { + DateTimeRangeError + } +} + impl fmt::Display for DateTimeRangeError { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!( diff --git a/src/types.rs b/src/types.rs index 3ab8a02f..86c41e13 100644 --- a/src/types.rs +++ b/src/types.rs @@ -120,13 +120,10 @@ impl arbitrary::Arbitrary<'_> for DateTime { #[cfg(feature = "chrono")] #[allow(clippy::result_unit_err)] -impl TryFrom for DateTime -where - T: Datelike + Timelike, -{ - type Error = (); +impl TryFrom for DateTime { + type Error = DateTimeRangeError; - fn try_from(value: T) -> Result { + fn try_from(value: NaiveDateTime) -> Result { Ok(DateTime::from_date_and_time( value.year().try_into()?, value.month().try_into()?, @@ -202,7 +199,7 @@ impl DateTime { hour: u8, minute: u8, second: u8, - ) -> Result { + ) -> Result { if (1980..=2107).contains(&year) && (1..=12).contains(&month) && (1..=31).contains(&day) @@ -219,7 +216,7 @@ impl DateTime { second, }) } else { - Err(()) + Err(DateTimeRangeError) } } From bd0cc62afac0e60e5e4c9efd05369ebd36e45f88 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 10:14:06 -0700 Subject: [PATCH 259/281] Bug fix --- src/types.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/types.rs b/src/types.rs index 86c41e13..efad0021 100644 --- a/src/types.rs +++ b/src/types.rs @@ -141,10 +141,11 @@ impl TryInto for DateTime { type Error = (); fn try_into(self) -> Result { - Ok(NaiveDateTime::new( - NaiveDate::from_ymd_opt(self.year.into(), self.month.into(), self.day.into())?, - NaiveTime::from_hms_opt(self.hour.into(), self.minute.into(), self.second.into())?, - )) + let date = NaiveDate::from_ymd_opt(self.year.into(), self.month.into(), self.day.into()) + .ok_or(())?; + let time = NaiveTime::from_hms_opt(self.hour.into(), self.minute.into(), self.second.into()) + .ok_or(())?; + Ok(NaiveDateTime::new(date, time)) } } From e596efd5e055c19e5d8d7b586086fc26482e4dfb Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 10:28:09 -0700 Subject: [PATCH 260/281] Update version number in doc comment --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 16f050e3..93b31e35 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,6 @@ mod zipcrypto; /// /// ```toml /// [dependencies] -/// zip_next = "=0.8.3" +/// zip_next = "=0.9.2" /// ``` pub mod unstable; From eb8709afdfaa31e6aa7e89442023dc39490b66ca Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 10:41:45 -0700 Subject: [PATCH 261/281] Fix Clippy and fmt warnings --- src/types.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/types.rs b/src/types.rs index efad0021..5ee22af3 100644 --- a/src/types.rs +++ b/src/types.rs @@ -124,14 +124,14 @@ impl TryFrom for DateTime { type Error = DateTimeRangeError; fn try_from(value: NaiveDateTime) -> Result { - Ok(DateTime::from_date_and_time( + DateTime::from_date_and_time( value.year().try_into()?, value.month().try_into()?, value.day().try_into()?, value.hour().try_into()?, value.minute().try_into()?, value.second().try_into()?, - )?) + ) } } @@ -143,8 +143,9 @@ impl TryInto for DateTime { fn try_into(self) -> Result { let date = NaiveDate::from_ymd_opt(self.year.into(), self.month.into(), self.day.into()) .ok_or(())?; - let time = NaiveTime::from_hms_opt(self.hour.into(), self.minute.into(), self.second.into()) - .ok_or(())?; + let time = + NaiveTime::from_hms_opt(self.hour.into(), self.minute.into(), self.second.into()) + .ok_or(())?; Ok(NaiveDateTime::new(date, time)) } } From 2407ef95c6db06d119e424f70afd28864f9f9bf8 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 18:17:59 -0700 Subject: [PATCH 262/281] Fixes and refactors for no-features build --- .github/workflows/ci.yaml | 36 +++++- CHANGELOG.md | 14 ++- Cargo.toml | 4 +- README.md | 4 +- benches/read_entry.rs | 2 +- benches/read_metadata.rs | 2 +- examples/write_dir.rs | 2 +- examples/write_sample.rs | 2 +- fuzz/fuzz_targets/fuzz_write.rs | 6 +- src/compression.rs | 49 ++++++++ src/lib.rs | 2 +- src/read.rs | 120 ++++++++++++-------- src/spec.rs | 12 -- src/types.rs | 27 ++--- src/write.rs | 190 +++++++++++++++++++------------- tests/end_to_end.rs | 6 +- tests/zip_crypto.rs | 2 +- 17 files changed, 310 insertions(+), 170 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a676dc03..f837e6b5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,18 +25,24 @@ jobs: toolchain: ${{ matrix.rust }} override: true - - name: check + - name: Check uses: actions-rs/cargo@v1 with: command: check args: --all --bins --examples - - name: tests + - name: Tests uses: actions-rs/cargo@v1 with: command: test args: --all + - name: Tests (no features) + uses: actions-rs/cargo@v1 + with: + command: test + args: --all --no-default-features + clippy: runs-on: ubuntu-latest @@ -117,6 +123,32 @@ jobs: - name: run fuzz run: | cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=10000 -max_len=5000000000 + - name: Upload any failure inputs + if: always() + uses: actions/upload-artifact@v3 + with: + name: fuzz_write_bad_inputs + path: fuzz/artifacts/fuzz_write/crash-* + if-no-files-found: ignore + + fuzz_write_with_no_features: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + + - run: cargo install cargo-fuzz + - name: compile fuzz + run: | + cargo fuzz build --no-default-features fuzz_write + - name: run fuzz + run: | + cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=1000000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index a0d7010b..1f48ba12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -197,4 +197,16 @@ ### Added - `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. - - `chrono` to convert zip_next::DateTime to and from chrono::NaiveDateTime \ No newline at end of file + - `chrono` to convert zip_next::DateTime to and from chrono::NaiveDateTime + +## [0.10.0] + +### Changed + + - Replaces the `flush_on_finish_file` parameter of `ZipWriter::new` and `ZipWriter::Append` with + a `set_flush_on_finish_file` method. + +### Fixed + + - Fixes build errors that occur when all default features are disabled. + - Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 84204c3f..86256105 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.9.2" +version = "0.10.0" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" @@ -16,7 +16,7 @@ edition = "2021" aes = { version = "0.8.2", optional = true } byteorder = "1.4.3" bzip2 = { version = "0.4.4", optional = true } -chrono = { version = "0.4.25", optional = true } +chrono = { version = "0.4.26", optional = true } constant_time_eq = { version = "0.2.5", optional = true } crc32fast = "1.3.2" flate2 = { version = "1.0.26", default-features = false, optional = true } diff --git a/README.md b/README.md index 38b93134..12902363 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.9.2" +zip_next = "0.10.0" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.9.2", default-features = false } +zip_next = { version = "0.10.0", default-features = false } ``` The features available are: diff --git a/benches/read_entry.rs b/benches/read_entry.rs index dd4bced7..4ee20b02 100644 --- a/benches/read_entry.rs +++ b/benches/read_entry.rs @@ -8,7 +8,7 @@ use zip_next::{ZipArchive, ZipWriter}; fn generate_random_archive(size: usize) -> Vec { let data = Vec::new(); - let mut writer = ZipWriter::new(Cursor::new(data), false); + let mut writer = ZipWriter::new(Cursor::new(data)); let options = zip_next::write::FileOptions::default() .compression_method(zip_next::CompressionMethod::Stored); diff --git a/benches/read_metadata.rs b/benches/read_metadata.rs index eee2d713..f9be2ec3 100644 --- a/benches/read_metadata.rs +++ b/benches/read_metadata.rs @@ -11,7 +11,7 @@ const FILE_SIZE: usize = 1024; fn generate_random_archive(count_files: usize, file_size: usize) -> Vec { let data = Vec::new(); - let mut writer = ZipWriter::new(Cursor::new(data), false); + let mut writer = ZipWriter::new(Cursor::new(data)); let options = FileOptions::default().compression_method(CompressionMethod::Stored); let bytes = vec![0u8; file_size]; diff --git a/examples/write_dir.rs b/examples/write_dir.rs index a7f14e35..6d92d161 100644 --- a/examples/write_dir.rs +++ b/examples/write_dir.rs @@ -76,7 +76,7 @@ fn zip_dir( where T: Write + Seek, { - let mut zip = zip_next::ZipWriter::new(writer, false); + let mut zip = zip_next::ZipWriter::new(writer); let options = FileOptions::default() .compression_method(method) .unix_permissions(0o755); diff --git a/examples/write_sample.rs b/examples/write_sample.rs index 0834d473..bb9739d0 100644 --- a/examples/write_sample.rs +++ b/examples/write_sample.rs @@ -25,7 +25,7 @@ fn doit(filename: &str) -> zip_next::result::ZipResult<()> { let path = std::path::Path::new(filename); let file = std::fs::File::create(path).unwrap(); - let mut zip = zip_next::ZipWriter::new(file, false); + let mut zip = zip_next::ZipWriter::new(file); zip.add_directory("test/", Default::default())?; diff --git a/fuzz/fuzz_targets/fuzz_write.rs b/fuzz/fuzz_targets/fuzz_write.rs index 5aa0ab4b..b09b7ef9 100644 --- a/fuzz/fuzz_targets/fuzz_write.rs +++ b/fuzz/fuzz_targets/fuzz_write.rs @@ -51,6 +51,7 @@ fn do_operation(writer: &mut RefCell>, operation: FileOperation, abort: bool, flush_on_finish_file: bool) -> Result<(), Box> where T: Read + Write + Seek { + writer.borrow_mut().set_flush_on_finish_file(flush_on_finish_file); let name = operation.name; match operation.basic { BasicFileOperation::WriteNormalFile {contents, mut options, ..} => { @@ -86,7 +87,7 @@ fn do_operation(writer: &mut RefCell>, if operation.reopen { let old_comment = writer.borrow().get_raw_comment().to_owned(); let new_writer = zip_next::ZipWriter::new_append( - writer.borrow_mut().finish().unwrap(), flush_on_finish_file).unwrap(); + writer.borrow_mut().finish().unwrap()).unwrap(); assert_eq!(&old_comment, new_writer.get_raw_comment()); *writer = new_writer.into(); } @@ -94,8 +95,7 @@ fn do_operation(writer: &mut RefCell>, } fuzz_target!(|test_case: FuzzTestCase| { - let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()), - test_case.flush_on_finish_file)); + let mut writer = RefCell::new(zip_next::ZipWriter::new(Cursor::new(Vec::new()))); writer.borrow_mut().set_raw_comment(test_case.comment); for (operation, abort) in test_case.operations { let _ = do_operation(&mut writer, operation, abort, test_case.flush_on_finish_file); diff --git a/src/compression.rs b/src/compression.rs index e23d743f..9abd34af 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -152,6 +152,55 @@ impl CompressionMethod { } } +impl Default for CompressionMethod { + fn default() -> Self { + #[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli" + ))] + return CompressionMethod::Deflated; + + #[cfg(all( + not(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli" + )), + feature = "bzip2" + ))] + return CompressionMethod::Bzip2; + + #[cfg(all( + not(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli", + feature = "bzip2" + )), + feature = "zstd" + ))] + return CompressionMethod::Zstd; + + #[cfg(not(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli", + feature = "bzip2", + feature = "zstd" + )))] + return CompressionMethod::Stored; + } +} + impl fmt::Display for CompressionMethod { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Just duplicate what the Debug format looks like, i.e, the enum key: diff --git a/src/lib.rs b/src/lib.rs index 93b31e35..c18edfe4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,6 @@ mod zipcrypto; /// /// ```toml /// [dependencies] -/// zip_next = "=0.9.2" +/// zip_next = "=0.10.0" /// ``` pub mod unstable; diff --git a/src/read.rs b/src/read.rs index f20d4662..5c8d818d 100644 --- a/src/read.rs +++ b/src/read.rs @@ -72,6 +72,7 @@ pub(crate) mod zip_archive { } pub use zip_archive::ZipArchive; + #[allow(clippy::large_enum_variant)] pub(crate) enum CryptoReader<'a> { Plaintext(io::Take<&'a mut dyn Read>), @@ -296,11 +297,19 @@ pub(crate) fn make_reader( } } +pub(crate) struct DirectoryCounts { + pub(crate) archive_offset: u64, + pub(crate) directory_start: u64, + pub(crate) number_of_files: usize, + pub(crate) disk_number: u32, + pub(crate) disk_with_central_directory: u32, +} + impl ZipArchive { fn get_directory_counts_zip32( footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, - ) -> ZipResult<(u64, u64, usize)> { + ) -> ZipResult { // Some zip files have data prepended to them, resulting in the // offsets all being too small. Get the amount of error by comparing // the actual file position we found the CDE at with the offset @@ -314,14 +323,20 @@ impl ZipArchive { let directory_start = footer.central_directory_offset as u64 + archive_offset; let number_of_files = footer.number_of_files_on_this_disk as usize; - Ok((archive_offset, directory_start, number_of_files)) + Ok(DirectoryCounts { + archive_offset, + directory_start, + number_of_files, + disk_number: footer.disk_number as u32, + disk_with_central_directory: footer.disk_with_central_directory as u32, + }) } fn get_directory_counts_zip64( reader: &mut R, footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, - ) -> ZipResult<(u64, u64, usize, ZipResult<()>)> { + ) -> ZipResult { // See if there's a ZIP64 footer. The ZIP64 locator if present will // have its signature 20 bytes in front of the standard footer. The // standard footer, in turn, is 22+N bytes large, where N is the @@ -373,21 +388,13 @@ impl ZipArchive { )); } - let supported = if (footer64.disk_number != footer64.disk_with_central_directory) - || (!footer.record_too_small() - && footer.disk_number as u32 != locator64.disk_with_central_directory) - { - unsupported_zip_error("Support for multi-disk files is not implemented") - } else { - Ok(()) - }; - - Ok(( + Ok(DirectoryCounts { archive_offset, directory_start, - footer64.number_of_files as usize, - supported, - )) + number_of_files: footer64.number_of_files as usize, + disk_number: footer64.disk_number, + disk_with_central_directory: footer64.disk_with_central_directory, + }) } /// Get the directory start offset and number of files. This is done in a @@ -396,29 +403,48 @@ impl ZipArchive { reader: &mut R, footer: &spec::CentralDirectoryEnd, cde_start_pos: u64, - ) -> ZipResult<(u64, u64, usize)> { + ) -> ZipResult { // Check if file has a zip64 footer - let (archive_offset_64, directory_start_64, number_of_files_64, supported_64) = - match Self::get_directory_counts_zip64(reader, footer, cde_start_pos) { - Ok(result) => result, - Err(_) => return Self::get_directory_counts_zip32(footer, cde_start_pos), - }; - // Check if it also has a zip32 footer - let (archive_offset_32, directory_start_32, number_of_files_32) = - match Self::get_directory_counts_zip32(footer, cde_start_pos) { - Ok(result) => result, - Err(_) => { - supported_64?; - return Ok((archive_offset_64, directory_start_64, number_of_files_64)); + let counts_64 = Self::get_directory_counts_zip64(reader, footer, cde_start_pos); + let counts_32 = Self::get_directory_counts_zip32(footer, cde_start_pos); + match counts_64 { + Err(_) => match counts_32 { + Err(e) => Err(e), + Ok(counts) => { + if counts.disk_number != counts.disk_with_central_directory { + return unsupported_zip_error( + "Support for multi-disk files is not implemented", + ); + } + Ok(counts) } - }; - // It has both, so check if the zip64 footer is valid; if not, assume zip32 - if number_of_files_64 != number_of_files_32 && number_of_files_32 != u16::MAX as usize { - return Ok((archive_offset_32, directory_start_32, number_of_files_32)); + }, + Ok(counts_64) => { + match counts_32 { + Err(_) => Ok(counts_64), + Ok(counts_32) => { + // Both zip32 and zip64 footers exist, so check if the zip64 footer is valid; if not, try zip32 + if counts_64.number_of_files != counts_32.number_of_files + && counts_32.number_of_files != u16::MAX as usize + { + return Ok(counts_32); + } + if counts_64.disk_number != counts_32.disk_number + && counts_32.disk_number != u16::MAX as u32 + { + return Ok(counts_32); + } + if counts_64.disk_with_central_directory + != counts_32.disk_with_central_directory + && counts_32.disk_with_central_directory != u16::MAX as u32 + { + return Ok(counts_32); + } + Ok(counts_64) + } + } + } } - // It is, so we assume a zip64 - supported_64?; - Ok((archive_offset_64, directory_start_64, number_of_files_64)) } /// Read a ZIP archive, collecting the files it contains @@ -427,32 +453,34 @@ impl ZipArchive { pub fn new(mut reader: R) -> ZipResult> { let (footer, cde_start_pos) = spec::CentralDirectoryEnd::find_and_parse(&mut reader)?; - if !footer.record_too_small() && footer.disk_number != footer.disk_with_central_directory { + let counts = Self::get_directory_counts(&mut reader, &footer, cde_start_pos)?; + + if counts.disk_number != counts.disk_with_central_directory { return unsupported_zip_error("Support for multi-disk files is not implemented"); } - let (archive_offset, directory_start, number_of_files) = - Self::get_directory_counts(&mut reader, &footer, cde_start_pos)?; - // If the parsed number of files is greater than the offset then // something fishy is going on and we shouldn't trust number_of_files. - let file_capacity = if number_of_files > cde_start_pos as usize { + let file_capacity = if counts.number_of_files > cde_start_pos as usize { 0 } else { - number_of_files + counts.number_of_files }; let mut files = Vec::with_capacity(file_capacity); let mut names_map = HashMap::with_capacity(file_capacity); - if reader.seek(io::SeekFrom::Start(directory_start)).is_err() { + if reader + .seek(io::SeekFrom::Start(counts.directory_start)) + .is_err() + { return Err(ZipError::InvalidArchive( "Could not seek to start of central directory", )); } - for _ in 0..number_of_files { - let file = central_header_to_zip_file(&mut reader, archive_offset)?; + for _ in 0..counts.number_of_files { + let file = central_header_to_zip_file(&mut reader, counts.archive_offset)?; names_map.insert(file.file_name.clone(), files.len()); files.push(file); } @@ -460,7 +488,7 @@ impl ZipArchive { let shared = Arc::new(zip_archive::Shared { files, names_map, - offset: archive_offset, + offset: counts.archive_offset, comment: footer.zip_file_comment, }); diff --git a/src/spec.rs b/src/spec.rs index 1ecd2cce..691f2cab 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -23,18 +23,6 @@ pub struct CentralDirectoryEnd { } impl CentralDirectoryEnd { - // Per spec 4.4.1.4 - a CentralDirectoryEnd field might be insufficient to hold the - // required data. In this case the file SHOULD contain a ZIP64 format record - // and the field of this record will be set to -1 - pub(crate) fn record_too_small(&self) -> bool { - self.disk_number == 0xFFFF - || self.disk_with_central_directory == 0xFFFF - || self.number_of_files_on_this_disk == 0xFFFF - || self.number_of_files == 0xFFFF - || self.central_directory_size == 0xFFFFFFFF - || self.central_directory_offset == 0xFFFFFFFF - } - pub fn parse(reader: &mut T) -> ZipResult { let magic = reader.read_u32::()?; if magic != CENTRAL_DIRECTORY_END_SIGNATURE { diff --git a/src/types.rs b/src/types.rs index 5ee22af3..8af87d0b 100644 --- a/src/types.rs +++ b/src/types.rs @@ -11,8 +11,6 @@ use chrono::{Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; target_arch = "powerpc" )))] use std::sync::atomic; -#[cfg(not(feature = "time"))] -use std::time::SystemTime; #[cfg(doc)] use {crate::read::ZipFile, crate::write::FileOptions}; @@ -53,8 +51,6 @@ mod atomic { } } -#[cfg(feature = "time")] -use crate::result::DateTimeRangeError; #[cfg(feature = "time")] use time::{error::ComponentRange, Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; @@ -121,16 +117,16 @@ impl arbitrary::Arbitrary<'_> for DateTime { #[cfg(feature = "chrono")] #[allow(clippy::result_unit_err)] impl TryFrom for DateTime { - type Error = DateTimeRangeError; + type Error = (); fn try_from(value: NaiveDateTime) -> Result { DateTime::from_date_and_time( - value.year().try_into()?, - value.month().try_into()?, - value.day().try_into()?, - value.hour().try_into()?, - value.minute().try_into()?, - value.second().try_into()?, + value.year().try_into().map_err(|_| ())?, + value.month().try_into().map_err(|_| ())?, + value.day().try_into().map_err(|_| ())?, + value.hour().try_into().map_err(|_| ())?, + value.minute().try_into().map_err(|_| ())?, + value.second().try_into().map_err(|_| ())?, ) } } @@ -201,7 +197,7 @@ impl DateTime { hour: u8, minute: u8, second: u8, - ) -> Result { + ) -> Result { if (1980..=2107).contains(&year) && (1..=12).contains(&month) && (1..=31).contains(&day) @@ -218,7 +214,7 @@ impl DateTime { second, }) } else { - Err(DateTimeRangeError) + Err(()) } } @@ -316,8 +312,9 @@ impl DateTime { } #[cfg(feature = "time")] +#[allow(clippy::result_unit_err)] impl TryFrom for DateTime { - type Error = DateTimeRangeError; + type Error = (); fn try_from(dt: OffsetDateTime) -> Result { if dt.year() >= 1980 && dt.year() <= 2107 { @@ -330,7 +327,7 @@ impl TryFrom for DateTime { second: dt.second(), }) } else { - Err(DateTimeRangeError) + Err(()) } } } diff --git a/src/write.rs b/src/write.rs index 5ed7c78a..b882a326 100644 --- a/src/write.rs +++ b/src/write.rs @@ -8,12 +8,31 @@ use crate::types::{ffi, AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSIO use byteorder::{LittleEndian, WriteBytesExt}; use crc32fast::Hasher; use std::collections::HashMap; +#[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-zopfli", + feature = "bzip2", + feature = "zstd", + feature = "time" +))] use std::convert::TryInto; use std::default::Default; use std::io; use std::io::prelude::*; -use std::io::{BufReader, BufWriter, SeekFrom}; +use std::io::{BufReader, SeekFrom}; use std::mem; +#[cfg(any( + feature = "deflate", + feature = "deflate-miniz", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "zopfli", + feature = "bzip2", + feature = "zstd", +))] use std::num::NonZeroU8; use std::str::{from_utf8, Utf8Error}; use std::sync::Arc; @@ -24,16 +43,20 @@ use std::sync::Arc; feature = "deflate-zlib", feature = "deflate-zlib-ng" ))] -use flate2::write::DeflateEncoder; +use flate2::{write::DeflateEncoder, Compression}; #[cfg(feature = "bzip2")] use bzip2::write::BzEncoder; -use flate2::Compression; #[cfg(feature = "time")] use time::OffsetDateTime; + +#[cfg(feature = "deflate-zopfli")] use zopfli::Options; +#[cfg(feature = "deflate-zopfli")] +use std::io::BufWriter; + #[cfg(feature = "zstd")] use zstd::stream::write::Encoder as ZstdEncoder; @@ -93,7 +116,7 @@ pub(crate) mod zip_writer { /// /// // We use a buffer here, though you'd normally use a `File` /// let mut buf = [0; 65536]; - /// let mut zip = ZipWriter::new(std::io::Cursor::new(&mut buf[..]), false); + /// let mut zip = ZipWriter::new(std::io::Cursor::new(&mut buf[..])); /// /// let options = FileOptions::default().compression_method(zip_next::CompressionMethod::Stored); /// zip.start_file("hello_world.txt", options)?; @@ -121,7 +144,7 @@ pub(crate) mod zip_writer { use crate::result::ZipError::InvalidArchive; use crate::write::GenericZipWriter::{Closed, Storer}; use crate::zipcrypto::ZipCryptoKeys; -use crate::CompressionMethod::{Deflated, Stored}; +use crate::CompressionMethod::Stored; pub use zip_writer::ZipWriter; #[derive(Default)] @@ -170,11 +193,11 @@ impl arbitrary::Arbitrary<'_> for FileOptions { zopfli_buffer_size: None, }; match options.compression_method { - Deflated => { + #[cfg(feature = "deflate-zopfli")] + CompressionMethod::Deflated => { if bool::arbitrary(u)? { let level = u.int_in_range(0..=24)?; options.compression_level = Some(level); - #[cfg(feature = "deflate-zopfli")] if level > Compression::best().level().try_into().unwrap() { options.zopfli_buffer_size = Some(1 << u.int_in_range(9..=30)?); } @@ -347,22 +370,7 @@ impl Default for FileOptions { /// Construct a new FileOptions object fn default() -> Self { Self { - #[cfg(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib", - feature = "deflate-zlib-ng", - feature = "deflate-zopfli" - ))] - compression_method: Deflated, - #[cfg(not(any( - feature = "deflate", - feature = "deflate-miniz", - feature = "deflate-zlib", - feature = "deflate-zlib-ng", - feature = "deflate-zopfli" - )))] - compression_method: Stored, + compression_method: Default::default(), compression_level: None, #[cfg(feature = "time")] last_modified_time: OffsetDateTime::now_utc().try_into().unwrap_or_default(), @@ -435,28 +443,28 @@ impl ZipWriterStats { impl ZipWriter { /// Initializes the archive from an existing ZIP archive, making it ready for append. - /// - /// See [`ZipWriter::new`] for the caveats that apply when `flush_on_finish_file` is set. - pub fn new_append(mut readwriter: A, flush_on_finish_file: bool) -> ZipResult> { + pub fn new_append(mut readwriter: A) -> ZipResult> { let (footer, cde_start_pos) = spec::CentralDirectoryEnd::find_and_parse(&mut readwriter)?; - if footer.disk_number != footer.disk_with_central_directory { + let counts = ZipArchive::get_directory_counts(&mut readwriter, &footer, cde_start_pos)?; + + if counts.disk_number != counts.disk_with_central_directory { return Err(ZipError::UnsupportedArchive( "Support for multi-disk files is not implemented", )); } - let (archive_offset, directory_start, number_of_files) = - ZipArchive::get_directory_counts(&mut readwriter, &footer, cde_start_pos)?; - - if readwriter.seek(SeekFrom::Start(directory_start)).is_err() { + if readwriter + .seek(SeekFrom::Start(counts.directory_start)) + .is_err() + { return Err(InvalidArchive( "Could not seek to start of central directory", )); } - let files = (0..number_of_files) - .map(|_| central_header_to_zip_file(&mut readwriter, archive_offset)) + let files = (0..counts.number_of_files) + .map(|_| central_header_to_zip_file(&mut readwriter, counts.archive_offset)) .collect::, _>>()?; let mut files_by_name = HashMap::new(); @@ -464,7 +472,7 @@ impl ZipWriter { files_by_name.insert(file.file_name.to_owned(), index); } - let _ = readwriter.seek(SeekFrom::Start(directory_start)); // seek directory_start to overwrite it + let _ = readwriter.seek(SeekFrom::Start(counts.directory_start)); // seek directory_start to overwrite it Ok(ZipWriter { inner: Storer(MaybeEncrypted::Unencrypted(readwriter)), @@ -474,9 +482,26 @@ impl ZipWriter { writing_to_file: false, comment: footer.zip_file_comment, writing_raw: true, // avoid recomputing the last file's header - flush_on_finish_file, + flush_on_finish_file: false, }) } + + /// `flush_on_finish_file` is designed to support a streaming `inner` that may unload flushed + /// bytes. It flushes a file's header and body once it starts writing another file. A ZipWriter + /// will not try to seek back into where a previous file was written unless + /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns + /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to + /// read previously-written files and not overwrite them. + /// + /// Note: when using an `inner` that cannot overwrite flushed bytes, do not wrap it in a + /// [std::io::BufWriter], because that has a [seek] method that implicitly calls [flush], and + /// ZipWriter needs to seek backward to update each file's header with the size and checksum + /// after writing the body. + /// + /// This setting is false by default. + pub fn set_flush_on_finish_file(&mut self, flush_on_finish_file: bool) { + self.flush_on_finish_file = flush_on_finish_file; + } } impl ZipWriter { @@ -540,15 +565,7 @@ impl ZipWriter { /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. /// After a successful write, the file remains open for writing. After a failed write, call /// [`ZipWriter::is_writing_file`] to determine if the file remains open. - /// - /// `flush_on_finish_file` is designed to support a streaming `inner` that may unload flushed - /// bytes. This ZipWriter will not try to seek further back than the last flushed byte unless - /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns - /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to - /// read flushed bytes and not overwrite them. Do not enable this with a [BufWriter], because - /// that implicitly calls [`Writer::flush`] whenever [`Seek::seek`] is called. Likewise, when - /// using Deflate compression, set [] - pub fn new(inner: W, flush_on_finish_file: bool) -> ZipWriter { + pub fn new(inner: W) -> ZipWriter { ZipWriter { inner: Storer(MaybeEncrypted::Unencrypted(inner)), files: Vec::new(), @@ -557,7 +574,7 @@ impl ZipWriter { writing_to_file: false, writing_raw: false, comment: Vec::new(), - flush_on_finish_file, + flush_on_finish_file: false, } } @@ -754,7 +771,12 @@ impl ZipWriter { return Ok(()); } - let make_plain_writer = self.inner.prepare_next_writer(Stored, None, None)?; + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; let writer = self.inner.get_plain(); @@ -804,7 +826,12 @@ impl ZipWriter { pub fn abort_file(&mut self) -> ZipResult<()> { let last_file = self.files.pop().ok_or(ZipError::FileNotFound)?; self.files_by_name.remove(&last_file.file_name); - let make_plain_writer = self.inner.prepare_next_writer(Stored, None, None)?; + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; self.inner.switch_to(make_plain_writer)?; self.switch_to_non_encrypting_writer()?; // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd @@ -1180,7 +1207,7 @@ impl GenericZipWriter { feature = "deflate-zlib-ng", feature = "deflate-zopfli" ))] - Deflated => { + CompressionMethod::Deflated => { let default = if cfg!(feature = "deflate") || cfg!(feature = "deflate-miniz") || cfg!(feature = "deflate-zlib") @@ -1605,6 +1632,13 @@ mod test { use crate::compression::CompressionMethod; use crate::result::ZipResult; use crate::types::DateTime; + #[cfg(any( + feature = "deflate", + feature = "deflate-zlib", + feature = "deflate-zlib-ng", + feature = "deflate-miniz", + feature = "deflate-zopfli" + ))] use crate::CompressionMethod::Deflated; use crate::ZipArchive; use std::io; @@ -1613,7 +1647,7 @@ mod test { #[test] fn write_empty_zip() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer.set_comment("ZIP"); let result = writer.finish().unwrap(); assert_eq!(result.get_ref().len(), 25); @@ -1632,7 +1666,7 @@ mod test { #[test] fn write_zip_dir() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .add_directory( "test", @@ -1660,7 +1694,7 @@ mod test { #[test] fn write_symlink_simple() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .add_symlink( "name", @@ -1689,7 +1723,7 @@ mod test { #[test] fn write_symlink_wonky_paths() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .add_symlink( "directory\\link", @@ -1721,7 +1755,7 @@ mod test { #[test] fn write_mimetype_zip() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); let options = FileOptions { compression_method: CompressionMethod::Stored, compression_level: None, @@ -1763,10 +1797,10 @@ mod test { #[test] fn test_shallow_copy() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); let options = FileOptions { - compression_method: Deflated, - compression_level: Some(9), + compression_method: CompressionMethod::default(), + compression_level: None, last_modified_time: DateTime::default(), permissions: Some(33188), large_file: false, @@ -1786,7 +1820,7 @@ mod test { .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .expect_err("Duplicate filename"); let zip = writer.finish().unwrap(); - let mut writer = ZipWriter::new_append(zip, false).unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); writer .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) .expect_err("Duplicate filename"); @@ -1815,10 +1849,10 @@ mod test { #[test] fn test_deep_copy() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); let options = FileOptions { - compression_method: Deflated, - compression_level: Some(9), + compression_method: CompressionMethod::default(), + compression_level: None, last_modified_time: DateTime::default(), permissions: Some(33188), large_file: false, @@ -1835,7 +1869,7 @@ mod test { .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) .unwrap(); let zip = writer.finish().unwrap(); - let mut writer = ZipWriter::new_append(zip, false).unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); writer .deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME) .unwrap(); @@ -1864,7 +1898,7 @@ mod test { #[test] fn duplicate_filenames() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file("foo/bar/test", FileOptions::default()) .unwrap(); @@ -1878,7 +1912,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file( "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1891,7 +1925,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_2() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file( "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1905,7 +1939,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_2a() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file( "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1919,7 +1953,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_3() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file("\0PK\u{6}\u{6}", FileOptions::default()) .unwrap(); @@ -1936,7 +1970,7 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_4() { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file("PK\u{6}\u{6}", FileOptions::default()) .unwrap(); @@ -1959,19 +1993,19 @@ mod test { #[test] fn test_filename_looks_like_zip64_locator_5() -> ZipResult<()> { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .add_directory("", FileOptions::default().with_alignment(21)) .unwrap(); - let mut writer = ZipWriter::new_append(writer.finish().unwrap(), false).unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); writer.shallow_copy_file("/", "").unwrap(); writer.shallow_copy_file("", "\0").unwrap(); writer.shallow_copy_file("\0", "PK\u{6}\u{6}").unwrap(); - let mut writer = ZipWriter::new_append(writer.finish().unwrap(), false).unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); writer .start_file("\0\0\0\0\0\0", FileOptions::default()) .unwrap(); - let mut writer = ZipWriter::new_append(writer.finish().unwrap(), false).unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); writer .start_file( "#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", @@ -1986,7 +2020,7 @@ mod test { #[test] fn remove_shallow_copy_keeps_original() -> ZipResult<()> { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer .start_file("original", FileOptions::default()) .unwrap(); @@ -2005,14 +2039,14 @@ mod test { #[test] fn remove_encrypted_file() -> ZipResult<()> { - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); let first_file_options = FileOptions::default() .with_alignment(65535) .with_deprecated_encryption(b"Password"); writer.start_file("", first_file_options).unwrap(); writer.abort_file().unwrap(); let zip = writer.finish().unwrap(); - let mut writer = ZipWriter::new(zip, false); + let mut writer = ZipWriter::new(zip); writer.start_file("", FileOptions::default()).unwrap(); Ok(()) } @@ -2022,12 +2056,12 @@ mod test { let mut options = FileOptions::default(); options = options.with_deprecated_encryption(b"Password"); options.alignment = 65535; - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer.add_symlink("", "s\t\0\0ggggg\0\0", options).unwrap(); writer.abort_file().unwrap(); let zip = writer.finish().unwrap(); println!("{:0>2x?}", zip.get_ref()); - let mut writer = ZipWriter::new_append(zip, false).unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); writer.start_file("", FileOptions::default()).unwrap(); Ok(()) } @@ -2037,9 +2071,9 @@ mod test { fn zopfli_empty_write() -> ZipResult<()> { let mut options = FileOptions::default(); options = options - .compression_method(Deflated) + .compression_method(CompressionMethod::default()) .compression_level(Some(264)); - let mut writer = ZipWriter::new(io::Cursor::new(Vec::new()), false); + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); writer.start_file("", options).unwrap(); writer.write_all(&[]).unwrap(); writer.write_all(&[]).unwrap(); diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 30f48506..70e59f61 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -35,7 +35,7 @@ fn copy() { { let mut src_archive = zip_next::ZipArchive::new(src_file).unwrap(); - let mut zip = ZipWriter::new(&mut tgt_file, false); + let mut zip = ZipWriter::new(&mut tgt_file); { let file = src_archive @@ -73,7 +73,7 @@ fn append() { write_test_archive(file, method, *shallow_copy); { - let mut zip = ZipWriter::new_append(&mut file, false).unwrap(); + let mut zip = ZipWriter::new_append(&mut file).unwrap(); zip.start_file( COPY_ENTRY_NAME, FileOptions::default() @@ -95,7 +95,7 @@ fn append() { // Write a test zip archive to buffer. fn write_test_archive(file: &mut Cursor>, method: CompressionMethod, shallow_copy: bool) { - let mut zip = ZipWriter::new(file, false); + let mut zip = ZipWriter::new(file); zip.add_directory("test/", Default::default()).unwrap(); diff --git a/tests/zip_crypto.rs b/tests/zip_crypto.rs index 4b25de8d..45612aac 100644 --- a/tests/zip_crypto.rs +++ b/tests/zip_crypto.rs @@ -25,7 +25,7 @@ fn encrypting_file() { use std::io::{Read, Write}; use zip_next::unstable::write::FileOptionsExt; let mut buf = vec![0; 2048]; - let mut archive = zip_next::write::ZipWriter::new(Cursor::new(&mut buf), false); + let mut archive = zip_next::write::ZipWriter::new(Cursor::new(&mut buf)); archive .start_file( "name", From 12eae587e1715951d6e9508e45d95798a63a0b6b Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 18:21:48 -0700 Subject: [PATCH 263/281] Remove unused import --- src/write.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/write.rs b/src/write.rs index b882a326..8af8427e 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1632,14 +1632,6 @@ mod test { use crate::compression::CompressionMethod; use crate::result::ZipResult; use crate::types::DateTime; - #[cfg(any( - feature = "deflate", - feature = "deflate-zlib", - feature = "deflate-zlib-ng", - feature = "deflate-miniz", - feature = "deflate-zopfli" - ))] - use crate::CompressionMethod::Deflated; use crate::ZipArchive; use std::io; use std::io::{Read, Write}; From 21d50041f0b7eb7cf1700e73c10ab727e59cb2a1 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Tue, 30 May 2023 20:17:39 -0700 Subject: [PATCH 264/281] Reduce fuzz_write_with_no_features to 10 million runs --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f837e6b5..0b902c48 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -148,7 +148,7 @@ jobs: cargo fuzz build --no-default-features fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=1000000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=100000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 From 3daa3a7ef4c7aeea3b9b53626fe17e8ccf17a9af Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 31 May 2023 09:01:40 -0700 Subject: [PATCH 265/281] Reduce fuzz_write_with_no_features to 1 million runs --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0b902c48..94678d51 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -148,7 +148,7 @@ jobs: cargo fuzz build --no-default-features fuzz_write - name: run fuzz run: | - cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=100000 -max_len=5000000000 + cargo fuzz run fuzz_write -- -timeout=5s -jobs=100 -workers=2 -runs=10000 -max_len=5000000000 - name: Upload any failure inputs if: always() uses: actions/upload-artifact@v3 From 60553541edcc8b0f8c758802442d69d00868a749 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 12:41:54 -0700 Subject: [PATCH 266/281] Change error return type to `DateTimeRangeError` --- CHANGELOG.md | 6 +++++- src/types.rs | 42 +++++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f48ba12..8a799138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -209,4 +209,8 @@ ### Fixed - Fixes build errors that occur when all default features are disabled. - - Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. \ No newline at end of file + - Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.10.1] + + - Date and time conversion methods now return `DateTimeRangeError` rather than `()` on error. \ No newline at end of file diff --git a/src/types.rs b/src/types.rs index 8af87d0b..b3ec71d7 100644 --- a/src/types.rs +++ b/src/types.rs @@ -53,6 +53,7 @@ mod atomic { #[cfg(feature = "time")] use time::{error::ComponentRange, Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; +use crate::result::DateTimeRangeError; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum System { @@ -115,33 +116,31 @@ impl arbitrary::Arbitrary<'_> for DateTime { } #[cfg(feature = "chrono")] -#[allow(clippy::result_unit_err)] impl TryFrom for DateTime { - type Error = (); + type Error = DateTimeRangeError; fn try_from(value: NaiveDateTime) -> Result { DateTime::from_date_and_time( - value.year().try_into().map_err(|_| ())?, - value.month().try_into().map_err(|_| ())?, - value.day().try_into().map_err(|_| ())?, - value.hour().try_into().map_err(|_| ())?, - value.minute().try_into().map_err(|_| ())?, - value.second().try_into().map_err(|_| ())?, + value.year().try_into()?, + value.month().try_into()?, + value.day().try_into()?, + value.hour().try_into()?, + value.minute().try_into()?, + value.second().try_into()?, ) } } #[cfg(feature = "chrono")] -#[allow(clippy::result_unit_err)] impl TryInto for DateTime { - type Error = (); + type Error = DateTimeRangeError; fn try_into(self) -> Result { let date = NaiveDate::from_ymd_opt(self.year.into(), self.month.into(), self.day.into()) - .ok_or(())?; + .ok_or(DateTimeRangeError)?; let time = NaiveTime::from_hms_opt(self.hour.into(), self.minute.into(), self.second.into()) - .ok_or(())?; + .ok_or(DateTimeRangeError)?; Ok(NaiveDateTime::new(date, time)) } } @@ -189,7 +188,6 @@ impl DateTime { /// * hour: [0, 23] /// * minute: [0, 59] /// * second: [0, 60] - #[allow(clippy::result_unit_err)] pub fn from_date_and_time( year: u16, month: u8, @@ -197,7 +195,7 @@ impl DateTime { hour: u8, minute: u8, second: u8, - ) -> Result { + ) -> Result { if (1980..=2107).contains(&year) && (1..=12).contains(&month) && (1..=31).contains(&day) @@ -214,7 +212,7 @@ impl DateTime { second, }) } else { - Err(()) + Err(DateTimeRangeError) } } @@ -235,10 +233,9 @@ impl DateTime { /// Converts a OffsetDateTime object to a DateTime /// /// Returns `Err` when this object is out of bounds - #[allow(clippy::result_unit_err)] #[deprecated(note = "use `DateTime::try_from()`")] - pub fn from_time(dt: OffsetDateTime) -> Result { - dt.try_into().map_err(|_err| ()) + pub fn from_time(dt: OffsetDateTime) -> Result { + dt.try_into().map_err(|_err| DateTimeRangeError) } /// Gets the time portion of this datetime in the msdos representation @@ -312,22 +309,21 @@ impl DateTime { } #[cfg(feature = "time")] -#[allow(clippy::result_unit_err)] impl TryFrom for DateTime { - type Error = (); + type Error = DateTimeRangeError; fn try_from(dt: OffsetDateTime) -> Result { if dt.year() >= 1980 && dt.year() <= 2107 { Ok(DateTime { - year: (dt.year()) as u16, - month: (dt.month()) as u8, + year: (dt.year()).try_into()?, + month: (dt.month()).try_into()?, day: dt.day(), hour: dt.hour(), minute: dt.minute(), second: dt.second(), }) } else { - Err(()) + Err(DateTimeRangeError) } } } From 7b23b3212aaa87b3752d3818febf3b10cfacb798 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 12:56:11 -0700 Subject: [PATCH 267/281] Update doc comment about `compression_method` --- src/write.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8af8427e..67200c9f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -231,8 +231,10 @@ impl arbitrary::Arbitrary<'_> for FileOptions { impl FileOptions { /// Set the compression method for the new file /// - /// The default is `CompressionMethod::Deflated`. If the deflate compression feature is - /// disabled, `CompressionMethod::Stored` becomes the default. + /// The default is `CompressionMethod::Deflated` if it is enabled. If not, + /// `CompressionMethod::Bzip2` is the default if it is enabled. If neither `bzip2` nor `deflate` + /// is enabled, `CompressionMethod::Zlib` is the default. If all else fails, + /// `CompressionMethod::Stored` becomes the default and files are written uncompressed. #[must_use] pub fn compression_method(mut self, method: CompressionMethod) -> FileOptions { self.compression_method = method; From 07679d3b7f540187dd1a100f741baa82ae1974cf Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 12:59:17 -0700 Subject: [PATCH 268/281] Reformat and fix error --- src/result.rs | 7 +++++++ src/types.rs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/result.rs b/src/result.rs index 0c4776eb..dec3be1e 100644 --- a/src/result.rs +++ b/src/result.rs @@ -1,5 +1,6 @@ //! Error types that can be emitted from this library +use std::convert::Infallible; use std::error::Error; use std::fmt; use std::io; @@ -101,6 +102,12 @@ impl From for DateTimeRangeError { } } +impl From for DateTimeRangeError { + fn from(_value: Infallible) -> Self { + DateTimeRangeError + } +} + impl fmt::Display for DateTimeRangeError { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!( diff --git a/src/types.rs b/src/types.rs index b3ec71d7..1eb684a9 100644 --- a/src/types.rs +++ b/src/types.rs @@ -51,9 +51,9 @@ mod atomic { } } +use crate::result::DateTimeRangeError; #[cfg(feature = "time")] use time::{error::ComponentRange, Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; -use crate::result::DateTimeRangeError; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum System { From a6a6484bf82eb5fb87b2f1382a2569344c3ad499 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 13:11:42 -0700 Subject: [PATCH 269/281] Refactor: From is unreachable --- src/result.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/result.rs b/src/result.rs index dec3be1e..d8fb4190 100644 --- a/src/result.rs +++ b/src/result.rs @@ -104,7 +104,7 @@ impl From for DateTimeRangeError { impl From for DateTimeRangeError { fn from(_value: Infallible) -> Self { - DateTimeRangeError + unreachable!() } } From 34945fe884417873e6868f8015869f9460a7b722 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 14:26:15 -0700 Subject: [PATCH 270/281] Refactor to eliminate `From` --- src/result.rs | 7 ------- src/types.rs | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/result.rs b/src/result.rs index d8fb4190..0c4776eb 100644 --- a/src/result.rs +++ b/src/result.rs @@ -1,6 +1,5 @@ //! Error types that can be emitted from this library -use std::convert::Infallible; use std::error::Error; use std::fmt; use std::io; @@ -102,12 +101,6 @@ impl From for DateTimeRangeError { } } -impl From for DateTimeRangeError { - fn from(_value: Infallible) -> Self { - unreachable!() - } -} - impl fmt::Display for DateTimeRangeError { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!( diff --git a/src/types.rs b/src/types.rs index 1eb684a9..502f7262 100644 --- a/src/types.rs +++ b/src/types.rs @@ -122,7 +122,7 @@ impl TryFrom for DateTime { fn try_from(value: NaiveDateTime) -> Result { DateTime::from_date_and_time( value.year().try_into()?, - value.month().try_into()?, + u8::from(value.month()).into(), value.day().try_into()?, value.hour().try_into()?, value.minute().try_into()?, @@ -316,7 +316,7 @@ impl TryFrom for DateTime { if dt.year() >= 1980 && dt.year() <= 2107 { Ok(DateTime { year: (dt.year()).try_into()?, - month: (dt.month()).try_into()?, + month: dt.month().into(), day: dt.day(), hour: dt.hour(), minute: dt.minute(), From bcd8fb27d28575525f9a6d7e78dde9b0c9c33ea5 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 14:27:55 -0700 Subject: [PATCH 271/281] Bug fix --- src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index 502f7262..d9ced849 100644 --- a/src/types.rs +++ b/src/types.rs @@ -122,7 +122,7 @@ impl TryFrom for DateTime { fn try_from(value: NaiveDateTime) -> Result { DateTime::from_date_and_time( value.year().try_into()?, - u8::from(value.month()).into(), + value.month().try_into()?, value.day().try_into()?, value.hour().try_into()?, value.minute().try_into()?, From 52db1d03a6c8951bd9ee2b12d5bbcb5ba2f4459f Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Thu, 1 Jun 2023 21:48:25 -0700 Subject: [PATCH 272/281] Bump version to 0.10.1 --- Cargo.toml | 2 +- README.md | 6 +++--- src/lib.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 86256105..d0d04c68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.10.0" +version = "0.10.1" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 12902363..5158ef16 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ zip_next [![Build Status](https://github.com/Pr0methean/zip-next/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip-next/actions?query=branch%3Amaster+workflow%3ACI) [![Crates.io version](https://img.shields.io/crates/v/zip_next.svg)](https://crates.io/crates/zip_next) -[Documentation](https://docs.rs/zip_next/0.9.2/zip_next/) +[Documentation](https://docs.rs/zip_next/0.10.1/zip_next/) Info ---- @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.10.0" +zip_next = "0.10.1" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.10.0", default-features = false } +zip_next = { version = "0.10.1", default-features = false } ``` The features available are: diff --git a/src/lib.rs b/src/lib.rs index c18edfe4..89d7e113 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,6 @@ mod zipcrypto; /// /// ```toml /// [dependencies] -/// zip_next = "=0.10.0" +/// zip_next = "=0.10.1" /// ``` pub mod unstable; From b0687378fd981e23c9da423241013c583b8a05a4 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 7 Jun 2023 16:03:33 -0700 Subject: [PATCH 273/281] Update dependencies --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d0d04c68..b72d432b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ flate2 = { version = "1.0.26", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } pbkdf2 = {version = "0.12.1", optional = true } sha1 = {version = "0.10.5", optional = true } -time = { version = "0.3.21", optional = true, default-features = false, features = ["std"] } +time = { version = "0.3.22", optional = true, default-features = false, features = ["std"] } zstd = { version = "0.12.3", optional = true, default-features = false } zopfli = { version = "0.7.4", optional = true } @@ -35,9 +35,9 @@ arbitrary = { version = "1.3.0", features = ["derive"] } [dev-dependencies] bencher = "0.1.5" -getrandom = { version = "0.2.9", features = ["js"] } +getrandom = { version = "0.2.10", features = ["js"] } walkdir = "2.3.3" -time = { version = "0.3.21", features = ["formatting", "macros"] } +time = { version = "0.3.22", features = ["formatting", "macros"] } [features] aes-crypto = [ "aes", "constant_time_eq", "hmac", "pbkdf2", "sha1" ] From 3cb826fe8b1ae9f5f8e24db586a922b7fffc1d01 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 7 Jun 2023 22:15:06 -0700 Subject: [PATCH 274/281] Make functions `const` where possible --- src/aes.rs | 2 +- src/compression.rs | 4 ++-- src/read.rs | 4 ++-- src/read/stream.rs | 4 ++-- src/types.rs | 32 ++++++++++++++++---------------- src/write.rs | 22 +++++++++++----------- src/zipcrypto.rs | 2 +- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/aes.rs b/src/aes.rs index c42a05ca..55ea8bf3 100644 --- a/src/aes.rs +++ b/src/aes.rs @@ -45,7 +45,7 @@ pub struct AesReader { } impl AesReader { - pub fn new(reader: R, aes_mode: AesMode, compressed_size: u64) -> AesReader { + pub const fn new(reader: R, aes_mode: AesMode, compressed_size: u64) -> AesReader { let data_length = compressed_size - (PWD_VERIFY_LENGTH + AUTH_CODE_LENGTH + aes_mode.salt_length()) as u64; diff --git a/src/compression.rs b/src/compression.rs index 9abd34af..fddc54c2 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -100,7 +100,7 @@ impl CompressionMethod { since = "0.5.7", note = "use a constant to construct a compression method" )] - pub fn from_u16(val: u16) -> CompressionMethod { + pub const fn from_u16(val: u16) -> CompressionMethod { #[allow(deprecated)] match val { 0 => CompressionMethod::Stored, @@ -128,7 +128,7 @@ impl CompressionMethod { since = "0.5.7", note = "to match on other compression methods, use a constant" )] - pub fn to_u16(self) -> u16 { + pub const fn to_u16(self) -> u16 { #[allow(deprecated)] match self { CompressionMethod::Stored => 0, diff --git a/src/read.rs b/src/read.rs index 5c8d818d..2c7809ce 100644 --- a/src/read.rs +++ b/src/read.rs @@ -107,7 +107,7 @@ impl<'a> CryptoReader<'a> { } /// Returns `true` if the data is encrypted using AE2. - pub fn is_ae2_encrypted(&self) -> bool { + pub const fn is_ae2_encrypted(&self) -> bool { #[cfg(feature = "aes-crypto")] return matches!( self, @@ -692,7 +692,7 @@ impl ZipArchive { } } -fn unsupported_zip_error(detail: &'static str) -> ZipResult { +const fn unsupported_zip_error(detail: &'static str) -> ZipResult { Err(ZipError::UnsupportedArchive(detail)) } diff --git a/src/read/stream.rs b/src/read/stream.rs index 1075c423..97e7019a 100644 --- a/src/read/stream.rs +++ b/src/read/stream.rs @@ -15,7 +15,7 @@ pub struct ZipStreamReader(R); impl ZipStreamReader { /// Create a new ZipStreamReader - pub fn new(reader: R) -> Self { + pub const fn new(reader: R) -> Self { Self(reader) } } @@ -204,7 +204,7 @@ impl ZipStreamFileMetadata { } /// Get unix mode for the file - pub fn unix_mode(&self) -> Option { + pub const fn unix_mode(&self) -> Option { self.0.unix_mode() } } diff --git a/src/types.rs b/src/types.rs index d9ced849..78896903 100644 --- a/src/types.rs +++ b/src/types.rs @@ -63,7 +63,7 @@ pub enum System { } impl System { - pub fn from_u8(system: u8) -> System { + pub const fn from_u8(system: u8) -> System { use self::System::*; match system { @@ -161,7 +161,7 @@ impl Default for DateTime { impl DateTime { /// Converts an msdos (u16, u16) pair to a DateTime object - pub fn from_msdos(datepart: u16, timepart: u16) -> DateTime { + pub const fn from_msdos(datepart: u16, timepart: u16) -> DateTime { let seconds = (timepart & 0b0000000000011111) << 1; let minutes = (timepart & 0b0000011111100000) >> 5; let hours = (timepart & 0b1111100000000000) >> 11; @@ -239,12 +239,12 @@ impl DateTime { } /// Gets the time portion of this datetime in the msdos representation - pub fn timepart(&self) -> u16 { + pub const fn timepart(&self) -> u16 { ((self.second as u16) >> 1) | ((self.minute as u16) << 5) | ((self.hour as u16) << 11) } /// Gets the date portion of this datetime in the msdos representation - pub fn datepart(&self) -> u16 { + pub const fn datepart(&self) -> u16 { (self.day as u16) | ((self.month as u16) << 5) | ((self.year - 1980) << 9) } @@ -258,7 +258,7 @@ impl DateTime { } /// Get the year. There is no epoch, i.e. 2018 will be returned as 2018. - pub fn year(&self) -> u16 { + pub const fn year(&self) -> u16 { self.year } @@ -267,7 +267,7 @@ impl DateTime { /// # Warning /// /// When read from a zip file, this may not be a reasonable value - pub fn month(&self) -> u8 { + pub const fn month(&self) -> u8 { self.month } @@ -276,7 +276,7 @@ impl DateTime { /// # Warning /// /// When read from a zip file, this may not be a reasonable value - pub fn day(&self) -> u8 { + pub const fn day(&self) -> u8 { self.day } @@ -285,7 +285,7 @@ impl DateTime { /// # Warning /// /// When read from a zip file, this may not be a reasonable value - pub fn hour(&self) -> u8 { + pub const fn hour(&self) -> u8 { self.hour } @@ -294,7 +294,7 @@ impl DateTime { /// # Warning /// /// When read from a zip file, this may not be a reasonable value - pub fn minute(&self) -> u8 { + pub const fn minute(&self) -> u8 { self.minute } @@ -303,7 +303,7 @@ impl DateTime { /// # Warning /// /// When read from a zip file, this may not be a reasonable value - pub fn second(&self) -> u8 { + pub const fn second(&self) -> u8 { self.second } } @@ -338,7 +338,7 @@ pub const DEFAULT_VERSION: u8 = 46; pub struct AtomicU64(atomic::AtomicU64); impl AtomicU64 { - pub fn new(v: u64) -> Self { + pub const fn new(v: u64) -> Self { Self(atomic::AtomicU64::new(v)) } @@ -456,7 +456,7 @@ impl ZipFileData { } /// Get unix mode for the file - pub(crate) fn unix_mode(&self) -> Option { + pub(crate) const fn unix_mode(&self) -> Option { if self.external_attributes == 0 { return None; } @@ -480,13 +480,13 @@ impl ZipFileData { } } - pub fn zip64_extension(&self) -> bool { + pub const fn zip64_extension(&self) -> bool { self.uncompressed_size > 0xFFFFFFFF || self.compressed_size > 0xFFFFFFFF || self.header_start > 0xFFFFFFFF } - pub fn version_needed(&self) -> u16 { + pub const fn version_needed(&self) -> u16 { // higher versions matched first match (self.zip64_extension(), self.compression_method) { #[cfg(feature = "bzip2")] @@ -517,11 +517,11 @@ pub enum AesMode { #[cfg(feature = "aes-crypto")] impl AesMode { - pub fn salt_length(&self) -> usize { + pub const fn salt_length(&self) -> usize { self.key_length() / 2 } - pub fn key_length(&self) -> usize { + pub const fn key_length(&self) -> usize { match self { Self::Aes128 => 16, Self::Aes192 => 24, diff --git a/src/write.rs b/src/write.rs index 67200c9f..f3545cf6 100644 --- a/src/write.rs +++ b/src/write.rs @@ -236,7 +236,7 @@ impl FileOptions { /// is enabled, `CompressionMethod::Zlib` is the default. If all else fails, /// `CompressionMethod::Stored` becomes the default and files are written uncompressed. #[must_use] - pub fn compression_method(mut self, method: CompressionMethod) -> FileOptions { + pub const fn compression_method(mut self, method: CompressionMethod) -> FileOptions { self.compression_method = method; self } @@ -252,7 +252,7 @@ impl FileOptions { /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 /// * others: only `None` is allowed #[must_use] - pub fn compression_level(mut self, level: Option) -> FileOptions { + pub const fn compression_level(mut self, level: Option) -> FileOptions { self.compression_level = level; self } @@ -262,7 +262,7 @@ impl FileOptions { /// The default is the current timestamp if the 'time' feature is enabled, and 1980-01-01 /// otherwise #[must_use] - pub fn last_modified_time(mut self, mod_time: DateTime) -> FileOptions { + pub const fn last_modified_time(mut self, mod_time: DateTime) -> FileOptions { self.last_modified_time = mod_time; self } @@ -277,7 +277,7 @@ impl FileOptions { /// higher file mode bits. So it cannot be used to denote an entry as a directory, /// symlink, or other special file type. #[must_use] - pub fn unix_permissions(mut self, mode: u32) -> FileOptions { + pub const fn unix_permissions(mut self, mode: u32) -> FileOptions { self.permissions = Some(mode & 0o777); self } @@ -288,7 +288,7 @@ impl FileOptions { /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not /// exceed the limit, 20 B are wasted. The default is `false`. #[must_use] - pub fn large_file(mut self, large: bool) -> FileOptions { + pub const fn large_file(mut self, large: bool) -> FileOptions { self.large_file = large; self } @@ -346,7 +346,7 @@ impl FileOptions { /// Sets the alignment to the given number of bytes. #[must_use] - pub fn with_alignment(mut self, alignment: u16) -> FileOptions { + pub const fn with_alignment(mut self, alignment: u16) -> FileOptions { self.alignment = alignment; self } @@ -357,13 +357,13 @@ impl FileOptions { /// are larger than about 32 KiB. #[must_use] #[cfg(feature = "deflate-zopfli")] - pub fn with_zopfli_buffer(mut self, size: Option) -> FileOptions { + pub const fn with_zopfli_buffer(mut self, size: Option) -> FileOptions { self.zopfli_buffer_size = size; self } /// Returns the compression level currently set. - pub fn get_compression_level(&self) -> Option { + pub const fn get_compression_level(&self) -> Option { self.compression_level } } @@ -581,7 +581,7 @@ impl ZipWriter { } /// Returns true if a file is currently open for writing. - pub fn is_writing_file(&self) -> bool { + pub const fn is_writing_file(&self) -> bool { self.writing_to_file && !self.inner.is_closed() } @@ -610,7 +610,7 @@ impl ZipWriter { /// /// This returns the raw bytes of the comment. The comment /// is typically expected to be encoded in UTF-8 - pub fn get_raw_comment(&self) -> &Vec { + pub const fn get_raw_comment(&self) -> &Vec { &self.comment } @@ -1364,7 +1364,7 @@ impl GenericZipWriter { } } - fn is_closed(&self) -> bool { + const fn is_closed(&self) -> bool { matches!(*self, GenericZipWriter::Closed) } diff --git a/src/zipcrypto.rs b/src/zipcrypto.rs index 7826ac2d..43ef1048 100644 --- a/src/zipcrypto.rs +++ b/src/zipcrypto.rs @@ -36,7 +36,7 @@ impl Debug for ZipCryptoKeys { } impl ZipCryptoKeys { - fn new() -> ZipCryptoKeys { + const fn new() -> ZipCryptoKeys { ZipCryptoKeys { key_0: Wrapping(0x12345678), key_1: Wrapping(0x23456789), From 56718b3344de986e960c74f28883629f6c74bdaa Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 7 Jun 2023 22:26:49 -0700 Subject: [PATCH 275/281] Bump version to 0.10.2 and update CHANGELOG --- CHANGELOG.md | 10 +++++++++- Cargo.toml | 2 +- README.md | 4 ++-- src/lib.rs | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a799138..04c74761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -213,4 +213,12 @@ ## [0.10.1] - - Date and time conversion methods now return `DateTimeRangeError` rather than `()` on error. \ No newline at end of file +### Changed + + - Date and time conversion methods now return `DateTimeRangeError` rather than `()` on error. + +## [0.10.2] + +### Changed + + - Where possible, methods are now `const`. This improves performance, especially when reading. diff --git a/Cargo.toml b/Cargo.toml index b72d432b..910e3120 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.10.1" +version = "0.10.2" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 5158ef16..15910975 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.10.1" +zip_next = "0.10.2" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.10.1", default-features = false } +zip_next = { version = "0.10.2", default-features = false } ``` The features available are: diff --git a/src/lib.rs b/src/lib.rs index 89d7e113..662ca779 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,6 @@ mod zipcrypto; /// /// ```toml /// [dependencies] -/// zip_next = "=0.10.1" +/// zip_next = "=0.10.2" /// ``` pub mod unstable; From 2e423f12db2a935f383ae847fdd9c60fc7512208 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 10 Jun 2023 21:03:26 -0700 Subject: [PATCH 276/281] Update constant_time_eq --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 910e3120..b4b7e1bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ aes = { version = "0.8.2", optional = true } byteorder = "1.4.3" bzip2 = { version = "0.4.4", optional = true } chrono = { version = "0.4.26", optional = true } -constant_time_eq = { version = "0.2.5", optional = true } +constant_time_eq = { version = "0.2.6", optional = true } crc32fast = "1.3.2" flate2 = { version = "1.0.26", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } From 367f231f157ef111fe7c90a52ad22a61210a53d7 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sun, 11 Jun 2023 11:17:54 -0700 Subject: [PATCH 277/281] Bug fix: don't run `cargo doc` on dependencies --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 94678d51..38d2fcd4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,7 +77,7 @@ jobs: run: cargo fmt --all -- --check - name: Docs - run: cargo doc + run: cargo doc --no-deps fuzz_read: runs-on: ubuntu-latest From 93f7e87f1647f765f08b6e2b536429d4662f8fe0 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 14 Jun 2023 09:53:52 -0700 Subject: [PATCH 278/281] Update crossbeam-utils --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b4b7e1bf..3cb68c34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ zstd = { version = "0.12.3", optional = true, default-features = false } zopfli = { version = "0.7.4", optional = true } [target.'cfg(any(all(target_arch = "arm", target_pointer_width = "32"), target_arch = "mips", target_arch = "powerpc"))'.dependencies] -crossbeam-utils = "0.8.15" +crossbeam-utils = "0.8.16" [target.'cfg(fuzzing)'.dependencies] arbitrary = { version = "1.3.0", features = ["derive"] } From 9ba7c80761856ae766ec54914add4d9a718932e5 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 17 Jun 2023 17:52:58 -0700 Subject: [PATCH 279/281] Update aes and constant-time-eq --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3cb68c34..f045beeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ Library to support the reading and writing of zip files. edition = "2021" [dependencies] -aes = { version = "0.8.2", optional = true } +aes = { version = "0.8.3", optional = true } byteorder = "1.4.3" bzip2 = { version = "0.4.4", optional = true } chrono = { version = "0.4.26", optional = true } -constant_time_eq = { version = "0.2.6", optional = true } +constant_time_eq = { version = "0.3.0", optional = true } crc32fast = "1.3.2" flate2 = { version = "1.0.26", default-features = false, optional = true } hmac = { version = "0.12.1", optional = true, features = ["reset"] } From 8b6baf9fdda65f489630b13cc8355ee49f4b3255 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Sat, 17 Jun 2023 17:58:56 -0700 Subject: [PATCH 280/281] Bump version to 0.10.3 --- CHANGELOG.md | 6 ++++++ Cargo.toml | 2 +- README.md | 4 ++-- src/lib.rs | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04c74761..e8a652dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -222,3 +222,9 @@ ### Changed - Where possible, methods are now `const`. This improves performance, especially when reading. + +## [0.10.3] + +### Changed + + - Updated dependencies. diff --git a/Cargo.toml b/Cargo.toml index f045beeb..2e848f2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zip_next" -version = "0.10.2" +version = "0.10.3" authors = ["Mathijs van de Nes ", "Marli Frost ", "Ryan Levick ", "Chris Hennick "] license = "MIT" diff --git a/README.md b/README.md index 15910975..324baf51 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@ With all default features: ```toml [dependencies] -zip_next = "0.10.2" +zip_next = "0.10.3" ``` Without the default features: ```toml [dependencies] -zip_next = { version = "0.10.2", default-features = false } +zip_next = { version = "0.10.3", default-features = false } ``` The features available are: diff --git a/src/lib.rs b/src/lib.rs index 662ca779..27771a21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,6 @@ mod zipcrypto; /// /// ```toml /// [dependencies] -/// zip_next = "=0.10.2" +/// zip_next = "=0.10.3" /// ``` pub mod unstable; From a6e6ad246e4b0e9cb5700b6bf01178d3a96fb970 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:07:29 +0000 Subject: [PATCH 281/281] Bump dependabot/fetch-metadata from 1.5.1 to 1.6.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.5.1 to 1.6.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.5.1...v1.6.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/dependabot_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot_automation.yml b/.github/workflows/dependabot_automation.yml index 67e9e10e..91ff8c9e 100644 --- a/.github/workflows/dependabot_automation.yml +++ b/.github/workflows/dependabot_automation.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.5.1 + uses: dependabot/fetch-metadata@v1.6.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Approve