Migrate to lz4 crate to be able to decode hc mode

lz4-flex unfortunately does not support this and we should prefer compatibility over maximum speed

Cargo.lock

@@ -1614,7 +1614,7 @@ dependencies = [
  "async-compression",
  "bstr",
  "lune-utils",
- "lz4_flex",
+ "lz4",
  "mlua",
  "serde",
  "serde_json",
@@ -1677,15 +1677,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "lz4_flex"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
-dependencies = [
- "twox-hash",
-]
-
 [[package]]
 name = "lzma-rs"
 version = "0.3.0"
@@ -2744,12 +2735,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
 [[package]]
 name = "stdweb"
 version = "0.4.20"
@@ -3219,16 +3204,6 @@ dependencies = [
  "utf-8",
 ]
 
-[[package]]
-name = "twox-hash"
-version = "1.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
-dependencies = [
- "cfg-if",
- "static_assertions",
-]
-
 [[package]]
 name = "typed-arena"
 version = "2.0.2"

crates/lune-std-serde/Cargo.toml

@@ -21,7 +21,7 @@ async-compression = { version = "0.4", features = [
     "zlib",
 ] }
 bstr = "1.9"
-lz4_flex = "0.11"
+lz4 = "1.24"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["preserve_order"] }
 serde_yaml = "0.9"

crates/lune-std-serde/src/compress_decompress.rs

@@ -1,6 +1,8 @@
+use std::io::{copy as copy_std, Cursor, Read as _, Write as _};
+
 use mlua::prelude::*;
 
-use lz4_flex::{compress_prepend_size, decompress_size_prepended};
+use lz4::{Decoder, EncoderBuilder};
 use tokio::{
     io::{copy, BufReader},
     task::spawn_blocking,
@@ -120,8 +122,9 @@ pub async fn compress<'lua>(
 ) -> LuaResult<Vec<u8>> {
     if let CompressDecompressFormat::LZ4 = format {
         let source = source.as_ref().to_vec();
-        return spawn_blocking(move || compress_prepend_size(&source))
+        return spawn_blocking(move || compress_lz4(source))
             .await
+            .into_lua_err()?
             .into_lua_err();
     }
 
@@ -160,7 +163,7 @@ pub async fn decompress<'lua>(
 ) -> LuaResult<Vec<u8>> {
     if let CompressDecompressFormat::LZ4 = format {
         let source = source.as_ref().to_vec();
-        return spawn_blocking(move || decompress_size_prepended(&source))
+        return spawn_blocking(move || decompress_lz4(source))
             .await
             .into_lua_err()?
             .into_lua_err();
@@ -187,3 +190,47 @@ pub async fn decompress<'lua>(
 
     Ok(bytes)
 }
+
+// TODO: Remove the compatibility layer. Prepending size is no longer
+// necessary, using lz4 create instead of lz4-flex, but we must remove
+// it in a major version to not unexpectedly break compatibility
+
+fn compress_lz4(input: Vec<u8>) -> LuaResult<Vec<u8>> {
+    let mut input = Cursor::new(input);
+    let mut output = Cursor::new(Vec::new());
+
+    // Prepend size for compatibility with old lz4-flex implementation
+    let len = input.get_ref().len() as u32;
+    output.write_all(len.to_le_bytes().as_ref())?;
+
+    let mut encoder = EncoderBuilder::new()
+        .level(16)
+        .checksum(lz4::ContentChecksum::ChecksumEnabled)
+        .block_mode(lz4::BlockMode::Independent)
+        .build(output)?;
+
+    copy_std(&mut input, &mut encoder)?;
+    let (output, result) = encoder.finish();
+    result?;
+
+    Ok(output.into_inner())
+}
+
+fn decompress_lz4(input: Vec<u8>) -> LuaResult<Vec<u8>> {
+    let mut input = Cursor::new(input);
+
+    // Skip size for compatibility with old lz4-flex implementation
+    // Note that right now we use it for preallocating the output buffer
+    // and a small efficiency gain, maybe we can expose this as some kind
+    // of "size hint" parameter instead in the serde library in the future
+    let mut size = [0; 4];
+    input.read_exact(&mut size)?;
+
+    let capacity = u32::from_le_bytes(size) as usize;
+    let mut output = Cursor::new(Vec::with_capacity(capacity));
+
+    let mut decoder = Decoder::new(input)?;
+    copy_std(&mut decoder, &mut output)?;
+
+    Ok(output.into_inner())
+}

scripts/generate_compression_test_files.luau

@@ -82,6 +82,7 @@ local function processLz4PrependSize(output: string): string
 	-- Lune supports only lz4 with the decompressed size
 	-- prepended to it, but the lz4 command line tool
 	-- doesn't add this automatically, so we have to
+	-- TODO: Remove this in the future when no longer needed
 	local buf = buffer.create(4 + #output)
 	buffer.writeu32(buf, 0, #INPUT_FILE_CONTENTS)
 	buffer.writestring(buf, 4, output)
@@ -157,7 +158,7 @@ local OUTPUT_FILES = {
 	{
 		command = BIN_LZ4,
 		format = "lz4" :: serde.CompressDecompressFormat,
-		args = { "-3", "--content-size", TEMP_FILE, TEMP_FILE .. ".lz4" },
+		args = { "--best", TEMP_FILE, TEMP_FILE .. ".lz4" },
 		output = TEMP_FILE .. ".lz4",
 		process = processLz4PrependSize,
 		final = INPUT_FILE .. ".lz4",