From d5f27dfad03adba87881f044da31ba4380ea31d1 Mon Sep 17 00:00:00 2001 From: hidez8891 <427357+hidez8891@users.noreply.github.com> Date: Sat, 13 May 2023 23:51:42 +0900 Subject: [PATCH 01/31] Fixed writing wrong UTF-8 flag If the UTF-8 flag (generic bit 11) is set, file names and comments must be saved in UTF-8 format. (APPENDIX D) However, the UTF-8 flag is set even for formats that are non-UTF-8 (GB18030, SHIFT_JIS, etc.). Fix this problem. --- src/write.rs | 64 +++++++++++++++++++++++++++++----------- tests/data/non_utf8.zip | Bin 0 -> 224 bytes 2 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 tests/data/non_utf8.zip diff --git a/src/write.rs b/src/write.rs index 4cdc031b..210e5026 100644 --- a/src/write.rs +++ b/src/write.rs @@ -387,8 +387,8 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: name.into(), - file_name_raw: Vec::new(), // Never used for saving + file_name: String::new(), // Never used for saving + file_name_raw: name.into().bytes().collect(), extra_field: Vec::new(), file_comment: String::new(), header_start, @@ -1092,11 +1092,10 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR // version needed to extract writer.write_u16::(file.version_needed())?; // general purpose bit flag - let flag = if !file.file_name.is_ascii() { - 1u16 << 11 - } else { - 0 - } | if file.encrypted { 1u16 << 0 } else { 0 }; + let is_utf8 = std::str::from_utf8(&file.file_name_raw).is_ok(); + let is_ascii = file.file_name_raw.is_ascii(); + let flag = if is_utf8 && !is_ascii { 1u16 << 11 } else { 0 } + | if file.encrypted { 1u16 << 0 } else { 0 }; writer.write_u16::(flag)?; // Compression method #[allow(deprecated)] @@ -1115,12 +1114,12 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR writer.write_u32::(file.uncompressed_size as u32)?; } // file name length - writer.write_u16::(file.file_name.as_bytes().len() as u16)?; + writer.write_u16::(file.file_name_raw.len() as u16)?; // extra field length let extra_field_length = if file.large_file { 20 } else { 0 } + file.extra_field.len() as u16; writer.write_u16::(extra_field_length)?; // file name - writer.write_all(file.file_name.as_bytes())?; + writer.write_all(&file.file_name_raw)?; // zip64 extra field if file.large_file { write_local_zip64_extra_field(writer, file)?; @@ -1167,11 +1166,10 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // version needed to extract writer.write_u16::(file.version_needed())?; // general puprose bit flag - let flag = if !file.file_name.is_ascii() { - 1u16 << 11 - } else { - 0 - } | if file.encrypted { 1u16 << 0 } else { 0 }; + let is_utf8 = std::str::from_utf8(&file.file_name_raw).is_ok(); + let is_ascii = file.file_name_raw.is_ascii(); + let flag = if is_utf8 && !is_ascii { 1u16 << 11 } else { 0 } + | if file.encrypted { 1u16 << 0 } else { 0 }; writer.write_u16::(flag)?; // compression method #[allow(deprecated)] @@ -1186,7 +1184,7 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // uncompressed size writer.write_u32::(file.uncompressed_size.min(spec::ZIP64_BYTES_THR) as u32)?; // file name length - writer.write_u16::(file.file_name.as_bytes().len() as u16)?; + writer.write_u16::(file.file_name_raw.len() as u16)?; // extra field length writer.write_u16::(zip64_extra_field_length + file.extra_field.len() as u16)?; // file comment length @@ -1200,7 +1198,7 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) // relative offset of local header writer.write_u32::(file.header_start.min(spec::ZIP64_BYTES_THR) as u32)?; // file name - writer.write_all(file.file_name.as_bytes())?; + writer.write_all(&file.file_name_raw)?; // zip64 extra field writer.write_all(&zip64_extra_field[..zip64_extra_field_length as usize])?; // extra field @@ -1281,7 +1279,7 @@ fn update_local_zip64_extra_field( writer: &mut T, file: &ZipFileData, ) -> ZipResult<()> { - let zip64_extra_field = file.header_start + 30 + file.file_name.as_bytes().len() as u64; + let zip64_extra_field = file.header_start + 30 + file.file_name_raw.len() as u64; writer.seek(io::SeekFrom::Start(zip64_extra_field + 4))?; writer.write_u64::(file.uncompressed_size)?; writer.write_u64::(file.compressed_size)?; @@ -1480,6 +1478,38 @@ mod test { assert_eq!(result.get_ref(), &v); } + #[test] + fn write_non_utf8() { + let mut writer = ZipWriter::new(io::Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + }; + + // GB18030 + // "中文" = [214, 208, 206, 196] + let filename = unsafe { String::from_utf8_unchecked(vec![214, 208, 206, 196]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding GB18030").unwrap(); + + // SHIFT_JIS + // "日文" = [147, 250, 149, 182] + let filename = unsafe { String::from_utf8_unchecked(vec![147, 250, 149, 182]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding SHIFT_JIS").unwrap(); + + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 224); + + let mut v = Vec::new(); + v.extend_from_slice(include_bytes!("../tests/data/non_utf8.zip")); + assert_eq!(result.get_ref(), &v); + } + #[test] fn path_to_string() { let mut path = std::path::PathBuf::new(); diff --git a/tests/data/non_utf8.zip b/tests/data/non_utf8.zip new file mode 100644 index 0000000000000000000000000000000000000000..be628652eab9eb35a59b7c03087f6bdf984c1f35 GIT binary patch literal 224 zcmWIWW@Zs#fB;1X+3mI70zeK3vjFk63+Ilc<|XH+WagzSxH}nI7#JG_py=?< Date: Thu, 9 May 2024 12:20:40 -0700 Subject: [PATCH 02/31] Update to use write_u16_le Signed-off-by: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 63c0a8e2..fe78dfb8 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1847,7 +1847,7 @@ fn write_central_directory_header(writer: &mut T, file: &ZipFileData) let is_ascii = file.file_name_raw.is_ascii(); let flag = if is_utf8 && !is_ascii { 1u16 << 11 } else { 0 } | if file.encrypted { 1u16 << 0 } else { 0 }; - writer.write_u16::(flag)?; + writer.write_u16_le(flag)?; // compression method #[allow(deprecated)] writer.write_u16_le(file.compression_method.to_u16())?; From d9a2cb1f71e54c8ea08b65a6bee25ee7f7d73940 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 12:22:28 -0700 Subject: [PATCH 03/31] Update to work without byteorder --- src/write.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/write.rs b/src/write.rs index fe78dfb8..3ccc8903 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1727,36 +1727,36 @@ fn clamp_opt>( fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { // local file header signature - writer.write_u32::(spec::LOCAL_FILE_HEADER_SIGNATURE)?; + writer.write_u32_le(spec::LOCAL_FILE_HEADER_SIGNATURE)?; // version needed to extract - writer.write_u16::(file.version_needed())?; + writer.write_u16_le(file.version_needed())?; // general purpose bit flag let is_utf8 = std::str::from_utf8(&file.file_name_raw).is_ok(); let is_ascii = file.file_name_raw.is_ascii(); let flag = if is_utf8 && !is_ascii { 1u16 << 11 } else { 0 } | if file.encrypted { 1u16 << 0 } else { 0 }; - writer.write_u16::(flag)?; + writer.write_u16_le(flag)?; // Compression method #[allow(deprecated)] - writer.write_u16::(file.compression_method.to_u16())?; + writer.write_u16_le(file.compression_method.to_u16())?; // last mod file time and last mod file date - writer.write_u16::(file.last_modified_time.timepart())?; - writer.write_u16::(file.last_modified_time.datepart())?; + writer.write_u16_le(file.last_modified_time.timepart())?; + writer.write_u16_le(file.last_modified_time.datepart())?; // crc-32 - writer.write_u32::(file.crc32)?; + writer.write_u32_le(file.crc32)?; // compressed size and uncompressed size if file.large_file { - writer.write_u32::(spec::ZIP64_BYTES_THR as u32)?; - writer.write_u32::(spec::ZIP64_BYTES_THR as u32)?; + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; } else { - writer.write_u32::(file.compressed_size as u32)?; - writer.write_u32::(file.uncompressed_size as u32)?; + writer.write_u32_le(file.compressed_size as u32)?; + writer.write_u32_le(file.uncompressed_size as u32)?; } // file name length - writer.write_u16::(file.file_name_raw.len() as u16)?; + writer.write_u16_le(file.file_name_raw.len() as u16)?; // extra field length let extra_field_length = if file.large_file { 20 } else { 0 } + file.extra_field.len() as u16; - writer.write_u16::(extra_field_length)?; + writer.write_u16_le(extra_field_length)?; // file name writer.write_all(&file.file_name_raw)?; // zip64 extra field From b9bf6f4ce29d70fe4ce8a2ea86e6ce12e21c1924 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 12:24:18 -0700 Subject: [PATCH 04/31] Fix deleted closing curly Signed-off-by: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> --- src/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.rs b/src/write.rs index 3ccc8903..00510b7f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1763,6 +1763,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR if file.large_file { write_local_zip64_extra_field(writer, file)?; } +} fn update_aes_extra_data( writer: &mut W, From 169b802136447010100609a0bd56960dcdc11b74 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 12:59:33 -0700 Subject: [PATCH 05/31] Fix build errors --- src/write.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 00510b7f..74d751a8 100644 --- a/src/write.rs +++ b/src/write.rs @@ -796,7 +796,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: Box::new([]), // Never used for saving + file_name: Box::new(""), // Never used for saving file_name_raw: name.into().bytes().collect(), extra_field, central_extra_field: options.extended_options.central_extra_data().cloned(), @@ -1755,7 +1755,8 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR // file name length writer.write_u16_le(file.file_name_raw.len() as u16)?; // extra field length - let extra_field_length = if file.large_file { 20 } else { 0 } + file.extra_field.len() as u16; + let extra_field_length = if file.large_file { 20 } else { 0 } + + file.extra_field.map(|field| field.len()).unwrap_or(0) as u16; writer.write_u16_le(extra_field_length)?; // file name writer.write_all(&file.file_name_raw)?; @@ -1763,6 +1764,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR if file.large_file { write_local_zip64_extra_field(writer, file)?; } + Ok(()) } fn update_aes_extra_data( From 0f086dcc755229f6562ffb5068c97f64a2264d3a Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 13:17:34 -0700 Subject: [PATCH 06/31] Fix remaining build errors --- src/write.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/write.rs b/src/write.rs index 74d751a8..56a7ff66 100644 --- a/src/write.rs +++ b/src/write.rs @@ -796,7 +796,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: Box::new(""), // Never used for saving + file_name: "".into(), // Never used for saving file_name_raw: name.into().bytes().collect(), extra_field, central_extra_field: options.extended_options.central_extra_data().cloned(), @@ -1755,8 +1755,10 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR // file name length writer.write_u16_le(file.file_name_raw.len() as u16)?; // extra field length - let extra_field_length = if file.large_file { 20 } else { 0 } - + file.extra_field.map(|field| field.len()).unwrap_or(0) as u16; + let mut extra_field_length = if file.large_file { 20 } else { 0 }; + if let Some(field) = file.extra_field { + extra_field_length += field.len(); + } writer.write_u16_le(extra_field_length)?; // file name writer.write_all(&file.file_name_raw)?; From 20e4c182801b23a5c698f3e5db20c816b8781308 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 13:36:37 -0700 Subject: [PATCH 07/31] Fix type mismatch for extra_field_length --- src/write.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 56a7ff66..e6737f0c 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1759,7 +1759,11 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR if let Some(field) = file.extra_field { extra_field_length += field.len(); } - writer.write_u16_le(extra_field_length)?; + match extra_field_length.try_into::() { + Ok(length_u16) => writer.write_u16_le(length_u16)?, + Err(_) => return ZipError::InvalidArchive("Extra field is too long"), + } + writer.write_u16_le(extra_field_length.try_into().map_err(|_| ZipError))?; // file name writer.write_all(&file.file_name_raw)?; // zip64 extra field From b0a2cbbe52a202d5de3811740bab6b3c2276470e Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 14:17:05 -0700 Subject: [PATCH 08/31] Fix build errors again --- src/write.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index e6737f0c..87691b34 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1761,9 +1761,8 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR } match extra_field_length.try_into::() { Ok(length_u16) => writer.write_u16_le(length_u16)?, - Err(_) => return ZipError::InvalidArchive("Extra field is too long"), + Err(_) => return Err(ZipError::InvalidArchive("Extra field is too long")), } - writer.write_u16_le(extra_field_length.try_into().map_err(|_| ZipError))?; // file name writer.write_all(&file.file_name_raw)?; // zip64 extra field From e9e5e01e3d0f22ee4ed9143ab148f914ba1f2fb1 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 17:12:52 -0700 Subject: [PATCH 09/31] chore: Remove turbofish to fix a build error --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 87691b34..094621d4 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1759,7 +1759,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR if let Some(field) = file.extra_field { extra_field_length += field.len(); } - match extra_field_length.try_into::() { + match extra_field_length.try_into() { Ok(length_u16) => writer.write_u16_le(length_u16)?, Err(_) => return Err(ZipError::InvalidArchive("Extra field is too long")), } From c23bcc55fbdb004e2b00860d00d0b932f957cbed Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 17:15:07 -0700 Subject: [PATCH 10/31] chore: Add a borrow to fix a build error Why didn't the compiler raise this last time?! --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 094621d4..8f85b4e4 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1756,7 +1756,7 @@ fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipR writer.write_u16_le(file.file_name_raw.len() as u16)?; // extra field length let mut extra_field_length = if file.large_file { 20 } else { 0 }; - if let Some(field) = file.extra_field { + if let Some(field) = &file.extra_field { extra_field_length += field.len(); } match extra_field_length.try_into() { From 0482a1329a735c05056ef4e1077bee2fd1e8ba0a Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 17:48:24 -0700 Subject: [PATCH 11/31] Fix bad mergee: write_local_file_header is now part of start_entry --- src/write.rs | 140 +++++---------------------------------------------- 1 file changed, 13 insertions(+), 127 deletions(-) diff --git a/src/write.rs b/src/write.rs index 8f85b4e4..ea95ac2f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -815,15 +815,15 @@ impl ZipWriter { let index = self.insert_file_data(file)?; let file = &mut self.files[index]; let writer = self.inner.get_plain(); + // local file header signature writer.write_u32_le(spec::LOCAL_FILE_HEADER_SIGNATURE)?; // version needed to extract writer.write_u16_le(file.version_needed())?; // general purpose bit flag - let flag = if !file.file_name.is_ascii() { - 1u16 << 11 - } else { - 0 - } | if file.encrypted { 1u16 << 0 } else { 0 }; + let is_utf8 = std::str::from_utf8(&file.file_name_raw).is_ok(); + let is_ascii = file.file_name_raw.is_ascii(); + let flag = if is_utf8 && !is_ascii { 1u16 << 11 } else { 0 } + | if file.encrypted { 1u16 << 0 } else { 0 }; writer.write_u16_le(flag)?; // Compression method #[allow(deprecated)] @@ -842,89 +842,22 @@ impl ZipWriter { writer.write_u32_le(file.uncompressed_size as u32)?; } // file name length - writer.write_u16_le(file.file_name.as_bytes().len() as u16)?; + writer.write_u16_le(file.file_name_raw.len() as u16)?; // extra field length - let mut extra_field_length = file.extra_field_len(); - if file.large_file { - extra_field_length += 20; + let mut extra_field_length = if file.large_file { 20 } else { 0 }; + if let Some(field) = &file.extra_field { + extra_field_length += field.len(); } - if extra_field_length + file.central_extra_field_len() > u16::MAX as usize { - let _ = self.abort_file(); - return Err(InvalidArchive("Extra data field is too large")); + match extra_field_length.try_into() { + Ok(length_u16) => writer.write_u16_le(length_u16)?, + Err(_) => return Err(ZipError::InvalidArchive("Extra field is too long")), } - let extra_field_length = extra_field_length as u16; - writer.write_u16_le(extra_field_length)?; // file name - writer.write_all(file.file_name.as_bytes())?; + writer.write_all(&file.file_name_raw)?; // zip64 extra field if file.large_file { write_local_zip64_extra_field(writer, file)?; } - if let Some(extra_field) = &file.extra_field { - file.extra_data_start = Some(writer.stream_position()?); - writer.write_all(extra_field)?; - } - let mut header_end = writer.stream_position()?; - if options.alignment > 1 { - let align = options.alignment as u64; - let unaligned_header_bytes = header_end % align; - if unaligned_header_bytes != 0 { - let pad_length = (align - unaligned_header_bytes) as usize; - let Some(new_extra_field_length) = - (pad_length as u16).checked_add(extra_field_length) - else { - let _ = self.abort_file(); - return Err(InvalidArchive( - "Extra data field would be larger than allowed after aligning", - )); - }; - if pad_length >= 4 { - // Add an extra field to the extra_data - let pad_body = vec![0; pad_length - 4]; - writer.write_all(b"za").map_err(ZipError::from)?; // 0x617a - writer - .write_u16_le(pad_body.len() as u16) - .map_err(ZipError::from)?; - writer.write_all(&pad_body).map_err(ZipError::from)?; - } else { - // extra_data padding is too small for an extra field header, so pad with - // zeroes - let pad = vec![0; pad_length]; - writer.write_all(&pad).map_err(ZipError::from)?; - } - header_end = writer.stream_position()?; - - // Update extra field length in local file header. - writer.seek(SeekFrom::Start(file.header_start + 28))?; - writer.write_u16_le(new_extra_field_length)?; - writer.seek(SeekFrom::Start(header_end))?; - debug_assert_eq!(header_end % align, 0); - } - } - match options.encrypt_with { - #[cfg(feature = "aes-crypto")] - Some(EncryptWith::Aes { mode, password }) => { - let aeswriter = AesWriter::new( - mem::replace(&mut self.inner, GenericZipWriter::Closed).unwrap(), - mode, - password.as_bytes(), - )?; - self.inner = GenericZipWriter::Storer(MaybeEncrypted::Aes(aeswriter)); - } - Some(EncryptWith::ZipCrypto(keys, ..)) => { - let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { - writer: mem::replace(&mut self.inner, Closed).unwrap(), - buffer: vec![], - keys, - }; - let crypto_header = [0u8; 12]; - - zipwriter.write_all(&crypto_header)?; - header_end = zipwriter.writer.stream_position()?; - self.inner = Storer(MaybeEncrypted::ZipCrypto(zipwriter)); - } - None => {} - } self.stats.start = header_end; debug_assert!(file.data_start.get().is_none()); file.data_start.get_or_init(|| header_end); @@ -1725,53 +1658,6 @@ fn clamp_opt>( } } -fn write_local_file_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { - // local file header signature - writer.write_u32_le(spec::LOCAL_FILE_HEADER_SIGNATURE)?; - // version needed to extract - writer.write_u16_le(file.version_needed())?; - // general purpose bit flag - let is_utf8 = std::str::from_utf8(&file.file_name_raw).is_ok(); - let is_ascii = file.file_name_raw.is_ascii(); - let flag = if is_utf8 && !is_ascii { 1u16 << 11 } else { 0 } - | if file.encrypted { 1u16 << 0 } else { 0 }; - writer.write_u16_le(flag)?; - // Compression method - #[allow(deprecated)] - writer.write_u16_le(file.compression_method.to_u16())?; - // last mod file time and last mod file date - writer.write_u16_le(file.last_modified_time.timepart())?; - writer.write_u16_le(file.last_modified_time.datepart())?; - // crc-32 - writer.write_u32_le(file.crc32)?; - // compressed size and uncompressed size - if file.large_file { - writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; - writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; - } else { - writer.write_u32_le(file.compressed_size as u32)?; - writer.write_u32_le(file.uncompressed_size as u32)?; - } - // file name length - writer.write_u16_le(file.file_name_raw.len() as u16)?; - // extra field length - let mut extra_field_length = if file.large_file { 20 } else { 0 }; - if let Some(field) = &file.extra_field { - extra_field_length += field.len(); - } - match extra_field_length.try_into() { - Ok(length_u16) => writer.write_u16_le(length_u16)?, - Err(_) => return Err(ZipError::InvalidArchive("Extra field is too long")), - } - // file name - writer.write_all(&file.file_name_raw)?; - // zip64 extra field - if file.large_file { - write_local_zip64_extra_field(writer, file)?; - } - Ok(()) -} - fn update_aes_extra_data( writer: &mut W, file: &mut ZipFileData, From 00537ae34ea7b8c74d6b78bbf955cba704c308bc Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 17:58:45 -0700 Subject: [PATCH 12/31] Fix bad merge: revert extra_field_length change --- src/write.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/write.rs b/src/write.rs index ea95ac2f..7bbfa5a6 100644 --- a/src/write.rs +++ b/src/write.rs @@ -844,14 +844,16 @@ impl ZipWriter { // file name length writer.write_u16_le(file.file_name_raw.len() as u16)?; // extra field length - let mut extra_field_length = if file.large_file { 20 } else { 0 }; - if let Some(field) = &file.extra_field { - extra_field_length += field.len(); + let mut extra_field_length = file.extra_field_len(); + if file.large_file { + extra_field_length += 20; } - match extra_field_length.try_into() { - Ok(length_u16) => writer.write_u16_le(length_u16)?, - Err(_) => return Err(ZipError::InvalidArchive("Extra field is too long")), + if extra_field_length + file.central_extra_field_len() > u16::MAX as usize { + let _ = self.abort_file(); + return Err(InvalidArchive("Extra data field is too large")); } + let extra_field_length = extra_field_length as u16; + writer.write_u16_le(extra_field_length)?; // file name writer.write_all(&file.file_name_raw)?; // zip64 extra field From e334f15f12fc070b3528433e500a77966c9eafdd Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:00:40 -0700 Subject: [PATCH 13/31] Fix bad merge: revert deletion of zip64 extra field --- src/write.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/write.rs b/src/write.rs index 7bbfa5a6..55d96578 100644 --- a/src/write.rs +++ b/src/write.rs @@ -860,6 +860,71 @@ impl ZipWriter { if file.large_file { write_local_zip64_extra_field(writer, file)?; } + if let Some(extra_field) = &file.extra_field { + file.extra_data_start = Some(writer.stream_position()?); + writer.write_all(extra_field)?; + } + let mut header_end = writer.stream_position()?; + if options.alignment > 1 { + let align = options.alignment as u64; + let unaligned_header_bytes = header_end % align; + if unaligned_header_bytes != 0 { + let pad_length = (align - unaligned_header_bytes) as usize; + let Some(new_extra_field_length) = + (pad_length as u16).checked_add(extra_field_length) + else { + let _ = self.abort_file(); + return Err(InvalidArchive( + "Extra data field would be larger than allowed after aligning", + )); + }; + if pad_length >= 4 { + // Add an extra field to the extra_data + let pad_body = vec![0; pad_length - 4]; + writer.write_all(b"za").map_err(ZipError::from)?; // 0x617a + writer + .write_u16_le(pad_body.len() as u16) + .map_err(ZipError::from)?; + writer.write_all(&pad_body).map_err(ZipError::from)?; + } else { + // extra_data padding is too small for an extra field header, so pad with + // zeroes + let pad = vec![0; pad_length]; + writer.write_all(&pad).map_err(ZipError::from)?; + } + header_end = writer.stream_position()?; + + // Update extra field length in local file header. + writer.seek(SeekFrom::Start(file.header_start + 28))?; + writer.write_u16_le(new_extra_field_length)?; + writer.seek(SeekFrom::Start(header_end))?; + debug_assert_eq!(header_end % align, 0); + } + } + match options.encrypt_with { + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, password }) => { + let aeswriter = AesWriter::new( + mem::replace(&mut self.inner, GenericZipWriter::Closed).unwrap(), + mode, + password.as_bytes(), + )?; + self.inner = GenericZipWriter::Storer(MaybeEncrypted::Aes(aeswriter)); + } + Some(EncryptWith::ZipCrypto(keys, ..)) => { + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + buffer: vec![], + keys, + }; + let crypto_header = [0u8; 12]; + + zipwriter.write_all(&crypto_header)?; + header_end = zipwriter.writer.stream_position()?; + self.inner = Storer(MaybeEncrypted::ZipCrypto(zipwriter)); + } + None => {} + } self.stats.start = header_end; debug_assert!(file.data_start.get().is_none()); file.data_start.get_or_init(|| header_end); From 0b896a9071e42c86db20516c820087ae0ed3dd59 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:06:23 -0700 Subject: [PATCH 14/31] chore: Fix build errors in test write_non_utf8() --- src/write.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 55d96578..6fc44c60 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2145,6 +2145,8 @@ mod test { permissions: Some(33188), large_file: false, encrypt_with: None, + extended_options: (), + alignment: 1, }; // GB18030 @@ -2179,7 +2181,7 @@ mod test { path.push("."); path.push("system32"); let path_str = super::path_to_string(&path); - assert_eq!(path_str, "windows/system32"); + assert_eq!(&path_str, "windows/system32"); } #[test] From 124ed1bac3984b9afffdb1b125945542a34f035e Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:08:39 -0700 Subject: [PATCH 15/31] chore: Box's & doesn't auto-deref --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 6fc44c60..318b9bca 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2181,7 +2181,7 @@ mod test { path.push("."); path.push("system32"); let path_str = super::path_to_string(&path); - assert_eq!(&path_str, "windows/system32"); + assert_eq!(&*path_str, "windows/system32"); } #[test] From 7fb1ec752d5d6c633f1a12f09b22ef8014c6e67b Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:10:50 -0700 Subject: [PATCH 16/31] chore: Fix build error: write_non_utf8 was missing zopfli_buffer_size --- src/write.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/write.rs b/src/write.rs index 318b9bca..19371557 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2147,6 +2147,8 @@ mod test { encrypt_with: None, extended_options: (), alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None }; // GB18030 From 48de16ca0d99b32314a9d47f636940a569a9189e Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:24:21 -0700 Subject: [PATCH 17/31] Chore: fix bug: file_name is needed for insert_file_data --- src/write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 19371557..6a57cc19 100644 --- a/src/write.rs +++ b/src/write.rs @@ -796,7 +796,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: "".into(), // Never used for saving + file_name: name.into(), // Never used for saving file_name_raw: name.into().bytes().collect(), extra_field, central_extra_field: options.extended_options.central_extra_data().cloned(), @@ -2183,7 +2183,7 @@ mod test { path.push("."); path.push("system32"); let path_str = super::path_to_string(&path); - assert_eq!(&*path_str, "windows/system32"); + assert_eq!(&*path_str, "system32"); } #[test] From 95c8b07bc2993a76821c1ff8ccc4cb06015ab185 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:25:23 -0700 Subject: [PATCH 18/31] chore: Update misleading comment --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 6a57cc19..ebc2486f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -796,7 +796,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: name.into(), // Never used for saving + file_name: name.into(), // Never used for saving, but used as map key in insert_file_data() file_name_raw: name.into().bytes().collect(), extra_field, central_extra_field: options.extended_options.central_extra_data().cloned(), From 955d1eddcdc5af90198386b9d89037ca22e9050f Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:26:39 -0700 Subject: [PATCH 19/31] chore: Add missing to_owned Needed so that `name` can be used to populate both `file_name` (for `insert_file_data`) and `file_name_raw` (for saving) --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index ebc2486f..429ca865 100644 --- a/src/write.rs +++ b/src/write.rs @@ -796,7 +796,7 @@ impl ZipWriter { crc32: raw_values.crc32, compressed_size: raw_values.compressed_size, uncompressed_size: raw_values.uncompressed_size, - file_name: name.into(), // Never used for saving, but used as map key in insert_file_data() + file_name: name.to_owned().into(), // Never used for saving, but used as map key in insert_file_data() file_name_raw: name.into().bytes().collect(), extra_field, central_extra_field: options.extended_options.central_extra_data().cloned(), From dcc6850c51e2001e0b9eb90fe34781c20fcf4243 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:27:54 -0700 Subject: [PATCH 20/31] style: Fix cargo fmt error re trailing comma --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 429ca865..05f77ed3 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2148,7 +2148,7 @@ mod test { extended_options: (), alignment: 1, #[cfg(feature = "deflate-zopfli")] - zopfli_buffer_size: None + zopfli_buffer_size: None, }; // GB18030 From 8e74a9da079ce7aef1013eeeb045181ea0dd0d7c Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:31:07 -0700 Subject: [PATCH 21/31] chore: Fix type bound so that to_owned() is available --- src/write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 05f77ed3..b6e3e9e4 100644 --- a/src/write.rs +++ b/src/write.rs @@ -13,6 +13,7 @@ use crate::types::{ffi, AesVendorVersion, DateTime, System, ZipFileData, DEFAULT use core::num::NonZeroU64; use crc32fast::Hasher; use indexmap::IndexMap; +use std::borrow::ToOwned; use std::default::Default; use std::io; use std::io::prelude::*; @@ -730,7 +731,7 @@ impl ZipWriter { raw_values: Option, ) -> ZipResult<()> where - S: Into>, + S: Into> + ToOwned, { self.finish_file()?; From df8479ac4f11955b49806e8bb50b485fbd757a7b Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:33:41 -0700 Subject: [PATCH 22/31] chore: Bug fix: ToOwned requires an explicit type bound --- src/write.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index b6e3e9e4..ccacbf0f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -724,14 +724,15 @@ impl ZipWriter { } /// Start a new file for with the requested options. - fn start_entry( + fn start_entry( &mut self, name: S, options: FileOptions, raw_values: Option, ) -> ZipResult<()> where - S: Into> + ToOwned, + S: Into> + ToOwned, + SOwned: Into> { self.finish_file()?; From 113d18c6a0dd260456ffb41489ef4261a2ca7780 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:34:03 -0700 Subject: [PATCH 23/31] chore: fix typo --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index ccacbf0f..bb99eb11 100644 --- a/src/write.rs +++ b/src/write.rs @@ -732,7 +732,7 @@ impl ZipWriter { ) -> ZipResult<()> where S: Into> + ToOwned, - SOwned: Into> + SToOwned: Into> { self.finish_file()?; From e1eae16e12db666650b2ccbbf367a93ea337e7e2 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:35:28 -0700 Subject: [PATCH 24/31] style: Fix cargo fmt re trailing comma in type-constraint list --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index bb99eb11..6bde75ce 100644 --- a/src/write.rs +++ b/src/write.rs @@ -732,7 +732,7 @@ impl ZipWriter { ) -> ZipResult<()> where S: Into> + ToOwned, - SToOwned: Into> + SToOwned: Into>, { self.finish_file()?; From 7ba16ae622d6f13d471b74d1620832c1640e3e4d Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:41:48 -0700 Subject: [PATCH 25/31] chore: Update generic type bounds --- src/write.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/write.rs b/src/write.rs index 6bde75ce..fe0074cd 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1055,13 +1055,14 @@ impl ZipWriter { /// same name as a file already in the archive. /// /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] - pub fn start_file( + pub fn start_file( &mut self, name: S, mut options: FileOptions, ) -> ZipResult<()> where - S: Into>, + S: Into> + ToOwned, + SToOwned: Into>, { Self::normalize_options(&mut options); let make_new_self = self.inner.prepare_next_writer( @@ -1190,9 +1191,10 @@ impl ZipWriter { /// Ok(()) /// } /// ``` - pub fn raw_copy_file_rename(&mut self, mut file: ZipFile, name: S) -> ZipResult<()> + pub fn raw_copy_file_rename(&mut self, mut file: ZipFile, name: S) -> ZipResult<()> where - S: Into>, + S: Into> + ToOwned, + SToOwned: Into>, { let mut options = SimpleFileOptions::default() .large_file(file.compressed_size().max(file.size()) > spec::ZIP64_BYTES_THR) @@ -1325,14 +1327,15 @@ impl ZipWriter { /// implementations may materialize a symlink as a regular file, possibly with the /// content incorrectly set to the symlink target. For maximum portability, consider /// storing a regular file instead. - pub fn add_symlink( + pub fn add_symlink( &mut self, name: N, target: T, mut options: FileOptions, ) -> ZipResult<()> where - N: Into>, + N: Into> + ToOwned, + NToOwned: Into>, T: Into>, { if options.permissions.is_none() { From 3454f84e8530061e5f0d96b2b0f2627dc83a92b2 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:50:49 -0700 Subject: [PATCH 26/31] chore: Patch out-of-date binary file --- src/write.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/write.rs b/src/write.rs index fe0074cd..2ad8a44a 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2173,6 +2173,11 @@ mod test { let mut v = Vec::new(); v.extend_from_slice(include_bytes!("../tests/data/non_utf8.zip")); + + // FIXME: Update the actual file once https://github.com/zip-rs/zip2/pull/106 is merged + v[4] = 20; + v[54] = 20; + assert_eq!(result.get_ref(), &v); } From 650dd9a71f62ec9a39459cb5716501ac6d083557 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 18:57:15 -0700 Subject: [PATCH 27/31] chore: More patches to binary file --- src/write.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index 2ad8a44a..620a2b8b 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2175,8 +2175,10 @@ mod test { v.extend_from_slice(include_bytes!("../tests/data/non_utf8.zip")); // FIXME: Update the actual file once https://github.com/zip-rs/zip2/pull/106 is merged - v[4] = 20; - v[54] = 20; + v[4] = 10; + v[54] = 10; + v[108] = 10; + v[158] = 10; assert_eq!(result.get_ref(), &v); } From 34b99956c9f8ab142c4ef824a3d5fefc1848155b Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 19:04:08 -0700 Subject: [PATCH 28/31] chore: shallow_copy_file needs to update file_name_raw --- src/write.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index 620a2b8b..bbbef3aa 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1456,6 +1456,7 @@ impl ZipWriter { let src_index = self.index_by_name(src_name)?; let mut dest_data = self.files[src_index].to_owned(); dest_data.file_name = dest_name.into(); + dest_data.file_name_raw = dest_name.into(); self.insert_file_data(dest_data)?; Ok(()) } @@ -2179,7 +2180,7 @@ mod test { v[54] = 10; v[108] = 10; v[158] = 10; - + assert_eq!(result.get_ref(), &v); } From 35b3a8f2573a519b30ae07b5004a3dc1289c7a3e Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 19:06:44 -0700 Subject: [PATCH 29/31] chore: fix type mismatch in shallow_copy_file caused by last fix --- src/write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/write.rs b/src/write.rs index bbbef3aa..ddf46a50 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1455,8 +1455,8 @@ impl ZipWriter { self.finish_file()?; let src_index = self.index_by_name(src_name)?; let mut dest_data = self.files[src_index].to_owned(); - dest_data.file_name = dest_name.into(); - dest_data.file_name_raw = dest_name.into(); + dest_data.file_name = dest_name.to_string().into(); + dest_data.file_name_raw = dest_name.into_boxed_bytes(); self.insert_file_data(dest_data)?; Ok(()) } From f7ea2764a274618abde2804349b00c9009bb27d4 Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 19:08:55 -0700 Subject: [PATCH 30/31] chore: &str doesn't have into_boxed_bytes, so call to_string first --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index ddf46a50..b60386fb 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1456,7 +1456,7 @@ impl ZipWriter { let src_index = self.index_by_name(src_name)?; let mut dest_data = self.files[src_index].to_owned(); dest_data.file_name = dest_name.to_string().into(); - dest_data.file_name_raw = dest_name.into_boxed_bytes(); + dest_data.file_name_raw = dest_name.to_string().into_boxed_bytes(); self.insert_file_data(dest_data)?; Ok(()) } From 186c89deeafe5198fa6c7495e172dac4e548bc5b Mon Sep 17 00:00:00 2001 From: Chris Hennick <4961925+Pr0methean@users.noreply.github.com> Date: Thu, 9 May 2024 19:11:16 -0700 Subject: [PATCH 31/31] chore: String has into_bytes but not into_boxed_bytes --- src/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.rs b/src/write.rs index b60386fb..529f3e82 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1456,7 +1456,7 @@ impl ZipWriter { let src_index = self.index_by_name(src_name)?; let mut dest_data = self.files[src_index].to_owned(); dest_data.file_name = dest_name.to_string().into(); - dest_data.file_name_raw = dest_name.to_string().into_boxed_bytes(); + dest_data.file_name_raw = dest_name.to_string().into_bytes().into(); self.insert_file_data(dest_data)?; Ok(()) }