From dc0d693d1a20041866f2bcda1c4f3df17a488628 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 20 May 2023 09:22:50 +0200 Subject: [PATCH] Add lz4 compression format --- CHANGELOG.md | 2 +- Cargo.lock | 26 ++++++++ docs/typedefs/Serde.luau | 36 ++++++++++- packages/lib/Cargo.toml | 1 + .../lib/src/lua/serde/compress_decompress.rs | 62 ++++++++++++++----- 5 files changed, 109 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7545599..1a5628e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, or `zlib`. +- Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, `lz4`, or `zlib`. Example usage: diff --git a/Cargo.lock b/Cargo.lock index f81d19c..9853c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,6 +1151,7 @@ dependencies = [ "hyper", "hyper-tungstenite", "lune-roblox", + "lz4_flex", "mlua", "once_cell", "os_str_bytes", @@ -1225,6 +1226,15 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8c72594ac26bfd34f2d99dfced2edfaddfe8a476e3ff2ca0eb293d925c4f83" +dependencies = [ + "twox-hash", +] + [[package]] name = "memchr" version = "2.5.0" @@ -1966,6 +1976,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stdweb" version = "0.4.20" @@ -2328,6 +2344,16 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typenum" version = "1.16.0" diff --git a/docs/typedefs/Serde.luau b/docs/typedefs/Serde.luau index f6fd70a..2771e5b 100644 --- a/docs/typedefs/Serde.luau +++ b/docs/typedefs/Serde.luau @@ -1,6 +1,6 @@ export type EncodeDecodeFormat = "json" | "yaml" | "toml" -export type CompressDecompressFormat = "brotli" | "gzip" | "zlib" +export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib" --[=[ @class Serde @@ -34,6 +34,14 @@ return { Encodes the given value using the given format. + Currently supported formats: + + | Name | Learn More | + |:-------|:---------------------| + | `json` | https://www.json.org | + | `yaml` | https://yaml.org | + | `toml` | https://toml.io | + @param format The format to use @param value The value to encode @param pretty If the encoded string should be human-readable, including things such as newlines and spaces. Only supported for json and toml formats, and defaults to false @@ -48,6 +56,14 @@ return { Decodes the given string using the given format into a lua value. + Currently supported formats: + + | Name | Learn More | + |:-------|:---------------------| + | `json` | https://www.json.org | + | `yaml` | https://yaml.org | + | `toml` | https://toml.io | + @param format The format to use @param encoded The string to decode @return The decoded lua value @@ -61,6 +77,15 @@ return { Compresses the given string using the given format. + Currently supported formats: + + | Name | Learn More | + |:---------|:----------------------------------| + | `brotli` | https://github.com/google/brotli | + | `gzip` | https://www.gnu.org/software/gzip | + | `lz4` | https://github.com/lz4/lz4 | + | `zlib` | https://www.zlib.net | + @param format The format to use @param encoded The string to compress @return The compressed string @@ -74,6 +99,15 @@ return { Decompresses the given string using the given format. + Currently supported formats: + + | Name | Learn More | + |:---------|:----------------------------------| + | `brotli` | https://github.com/google/brotli | + | `gzip` | https://www.gnu.org/software/gzip | + | `lz4` | https://github.com/lz4/lz4 | + | `zlib` | https://www.zlib.net | + @param format The format to use @param encoded The string to decompress @return The decompressed string diff --git a/packages/lib/Cargo.toml b/packages/lib/Cargo.toml index 63b186a..76272fb 100644 --- a/packages/lib/Cargo.toml +++ b/packages/lib/Cargo.toml @@ -38,6 +38,7 @@ async-trait = "0.1" blocking = "1.3" dialoguer = "0.10" dunce = "1.0" +lz4_flex = "0.10" pin-project = "1.0" os_str_bytes = "6.4" urlencoding = "2.1.2" diff --git a/packages/lib/src/lua/serde/compress_decompress.rs b/packages/lib/src/lua/serde/compress_decompress.rs index 4759f0e..f749abd 100644 --- a/packages/lib/src/lua/serde/compress_decompress.rs +++ b/packages/lib/src/lua/serde/compress_decompress.rs @@ -1,6 +1,8 @@ use async_compression::tokio::write::{ BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder, }; +use blocking::unblock; +use lz4_flex::{compress_prepend_size, decompress_size_prepended}; use mlua::prelude::*; use tokio::io::AsyncWriteExt; @@ -8,27 +10,44 @@ use tokio::io::AsyncWriteExt; pub enum CompressDecompressFormat { Brotli, GZip, + LZ4, ZLib, } #[allow(dead_code)] impl CompressDecompressFormat { pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option { - let bytes = bytes.as_ref(); - if bytes[0..4] == [0x0B, 0x24, 0x72, 0x68] { - Some(Self::Brotli) - } else if bytes[0..3] == [0x1F, 0x8B, 0x08] { - Some(Self::GZip) - } - // https://stackoverflow.com/a/54915442 - else if (bytes[0..2] == [0x78, 0x01]) - || (bytes[0..2] == [0x78, 0x5E]) - || (bytes[0..2] == [0x78, 0x9C]) - || (bytes[0..2] == [0x78, 0xDA]) - { - Some(Self::ZLib) - } else { - None + match bytes.as_ref() { + // https://github.com/PSeitz/lz4_flex/blob/main/src/frame/header.rs#L28 + b if b.len() >= 4 + && matches!( + u32::from_le_bytes(b[0..4].try_into().unwrap()), + 0x184D2204 | 0x184C2102 + ) => + { + Some(Self::LZ4) + } + // https://github.com/dropbox/rust-brotli/blob/master/src/enc/brotli_bit_stream.rs#L2805 + b if b.len() >= 4 + && matches!( + b[0..3], + [0xE1, 0x97, 0x81] | [0xE1, 0x97, 0x82] | [0xE1, 0x97, 0x80] + ) => + { + Some(Self::Brotli) + } + // https://github.com/rust-lang/flate2-rs/blob/main/src/gz/mod.rs#L135 + b if b.len() >= 3 && matches!(b[0..3], [0x1F, 0x8B, 0x08]) => Some(Self::GZip), + // https://stackoverflow.com/a/43170354 + b if b.len() >= 2 + && matches!( + b[0..2], + [0x78, 0x01] | [0x78, 0x5E] | [0x78, 0x9C] | [0x78, 0xDA] + ) => + { + Some(Self::ZLib) + } + _ => None, } } @@ -49,12 +68,13 @@ impl<'lua> FromLua<'lua> for CompressDecompressFormat { match s.to_string_lossy().to_ascii_lowercase().trim() { "brotli" => Ok(Self::Brotli), "gzip" => Ok(Self::GZip), + "lz4" => Ok(Self::LZ4), "zlib" => Ok(Self::ZLib), kind => Err(LuaError::FromLuaConversionError { from: value.type_name(), to: "CompressDecompressFormat", message: Some(format!( - "Invalid format '{kind}', valid formats are: brotli, gzip, zlib" + "Invalid format '{kind}', valid formats are: brotli, gzip, lz4, zlib" )), }), } @@ -89,6 +109,10 @@ pub async fn compress<'lua>( .write_all(source.as_ref()) .await? } + CompressDecompressFormat::LZ4 => { + let source = source.as_ref().to_vec(); + bytes = unblock(move || compress_prepend_size(&source)).await; + } } Ok(bytes) } @@ -114,6 +138,12 @@ pub async fn decompress<'lua>( .write_all(source.as_ref()) .await? } + CompressDecompressFormat::LZ4 => { + let source = source.as_ref().to_vec(); + bytes = unblock(move || decompress_size_prepended(&source)) + .await + .map_err(LuaError::external)?; + } } Ok(bytes) }