diff --git a/Cargo.lock b/Cargo.lock index afb1510..b4455c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1614,7 +1614,7 @@ dependencies = [ "async-compression", "bstr", "lune-utils", - "lz4_flex", + "lz4", "mlua", "serde", "serde_json", @@ -1677,15 +1677,6 @@ dependencies = [ "libc", ] -[[package]] -name = "lz4_flex" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" -dependencies = [ - "twox-hash", -] - [[package]] name = "lzma-rs" version = "0.3.0" @@ -2744,12 +2735,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "stdweb" version = "0.4.20" @@ -3219,16 +3204,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] - [[package]] name = "typed-arena" version = "2.0.2" diff --git a/crates/lune-std-serde/Cargo.toml b/crates/lune-std-serde/Cargo.toml index d35d401..4f8faef 100644 --- a/crates/lune-std-serde/Cargo.toml +++ b/crates/lune-std-serde/Cargo.toml @@ -21,7 +21,7 @@ async-compression = { version = "0.4", features = [ "zlib", ] } bstr = "1.9" -lz4_flex = "0.11" +lz4 = "1.24" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["preserve_order"] } serde_yaml = "0.9" diff --git a/crates/lune-std-serde/src/compress_decompress.rs b/crates/lune-std-serde/src/compress_decompress.rs index 837625b..3e5aaa0 100644 --- a/crates/lune-std-serde/src/compress_decompress.rs +++ b/crates/lune-std-serde/src/compress_decompress.rs @@ -1,6 +1,8 @@ +use std::io::{copy as copy_std, Cursor, Read as _, Write as _}; + use mlua::prelude::*; -use lz4_flex::{compress_prepend_size, decompress_size_prepended}; +use lz4::{Decoder, EncoderBuilder}; use tokio::{ io::{copy, BufReader}, task::spawn_blocking, @@ -120,8 +122,9 @@ pub async fn compress<'lua>( ) -> LuaResult> { if let CompressDecompressFormat::LZ4 = format { let source = source.as_ref().to_vec(); - return spawn_blocking(move || compress_prepend_size(&source)) + return spawn_blocking(move || compress_lz4(source)) .await + .into_lua_err()? .into_lua_err(); } @@ -160,7 +163,7 @@ pub async fn decompress<'lua>( ) -> LuaResult> { if let CompressDecompressFormat::LZ4 = format { let source = source.as_ref().to_vec(); - return spawn_blocking(move || decompress_size_prepended(&source)) + return spawn_blocking(move || decompress_lz4(source)) .await .into_lua_err()? .into_lua_err(); @@ -187,3 +190,47 @@ pub async fn decompress<'lua>( Ok(bytes) } + +// TODO: Remove the compatibility layer. Prepending size is no longer +// necessary, using lz4 create instead of lz4-flex, but we must remove +// it in a major version to not unexpectedly break compatibility + +fn compress_lz4(input: Vec) -> LuaResult> { + let mut input = Cursor::new(input); + let mut output = Cursor::new(Vec::new()); + + // Prepend size for compatibility with old lz4-flex implementation + let len = input.get_ref().len() as u32; + output.write_all(len.to_le_bytes().as_ref())?; + + let mut encoder = EncoderBuilder::new() + .level(16) + .checksum(lz4::ContentChecksum::ChecksumEnabled) + .block_mode(lz4::BlockMode::Independent) + .build(output)?; + + copy_std(&mut input, &mut encoder)?; + let (output, result) = encoder.finish(); + result?; + + Ok(output.into_inner()) +} + +fn decompress_lz4(input: Vec) -> LuaResult> { + let mut input = Cursor::new(input); + + // Skip size for compatibility with old lz4-flex implementation + // Note that right now we use it for preallocating the output buffer + // and a small efficiency gain, maybe we can expose this as some kind + // of "size hint" parameter instead in the serde library in the future + let mut size = [0; 4]; + input.read_exact(&mut size)?; + + let capacity = u32::from_le_bytes(size) as usize; + let mut output = Cursor::new(Vec::with_capacity(capacity)); + + let mut decoder = Decoder::new(input)?; + copy_std(&mut decoder, &mut output)?; + + Ok(output.into_inner()) +} diff --git a/scripts/generate_compression_test_files.luau b/scripts/generate_compression_test_files.luau index 5e8336a..b7e7fc5 100644 --- a/scripts/generate_compression_test_files.luau +++ b/scripts/generate_compression_test_files.luau @@ -82,6 +82,7 @@ local function processLz4PrependSize(output: string): string -- Lune supports only lz4 with the decompressed size -- prepended to it, but the lz4 command line tool -- doesn't add this automatically, so we have to + -- TODO: Remove this in the future when no longer needed local buf = buffer.create(4 + #output) buffer.writeu32(buf, 0, #INPUT_FILE_CONTENTS) buffer.writestring(buf, 4, output) @@ -157,7 +158,7 @@ local OUTPUT_FILES = { { command = BIN_LZ4, format = "lz4" :: serde.CompressDecompressFormat, - args = { "-3", "--content-size", TEMP_FILE, TEMP_FILE .. ".lz4" }, + args = { "--best", TEMP_FILE, TEMP_FILE .. ".lz4" }, output = TEMP_FILE .. ".lz4", process = processLz4PrependSize, final = INPUT_FILE .. ".lz4", diff --git a/tests/serde/test-files/loremipsum.txt.lz4 b/tests/serde/test-files/loremipsum.txt.lz4 index 9d8d46f..801bf90 100644 Binary files a/tests/serde/test-files/loremipsum.txt.lz4 and b/tests/serde/test-files/loremipsum.txt.lz4 differ