Add lz4 compression format

CHANGELOG.md

@@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
--   Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, or `zlib`.
+-   Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, `lz4`, or `zlib`.
 
     Example usage:
 

Cargo.lock

@@ -1151,6 +1151,7 @@ dependencies = [
  "hyper",
  "hyper-tungstenite",
  "lune-roblox",
+ "lz4_flex",
  "mlua",
  "once_cell",
  "os_str_bytes",
@@ -1225,6 +1226,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "lz4_flex"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b8c72594ac26bfd34f2d99dfced2edfaddfe8a476e3ff2ca0eb293d925c4f83"
+dependencies = [
+ "twox-hash",
+]
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -1966,6 +1976,12 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "stdweb"
 version = "0.4.20"
@@ -2328,6 +2344,16 @@ dependencies = [
  "utf-8",
 ]
 
+[[package]]
+name = "twox-hash"
+version = "1.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
+dependencies = [
+ "cfg-if",
+ "static_assertions",
+]
+
 [[package]]
 name = "typenum"
 version = "1.16.0"

docs/typedefs/Serde.luau

@@ -1,6 +1,6 @@
 export type EncodeDecodeFormat = "json" | "yaml" | "toml"
 
-export type CompressDecompressFormat = "brotli" | "gzip" | "zlib"
+export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib"
 
 --[=[
 	@class Serde
@@ -34,6 +34,14 @@ return {
 
 		Encodes the given value using the given format.
 
+		Currently supported formats:
+
+		| Name   | Learn More           |
+		|:-------|:---------------------|
+		| `json` | https://www.json.org |
+		| `yaml` | https://yaml.org     |
+		| `toml` | https://toml.io      |
+
 		@param format The format to use
 		@param value The value to encode
 		@param pretty If the encoded string should be human-readable, including things such as newlines and spaces. Only supported for json and toml formats, and defaults to false
@@ -48,6 +56,14 @@ return {
 
 		Decodes the given string using the given format into a lua value.
 
+		Currently supported formats:
+
+		| Name   | Learn More           |
+		|:-------|:---------------------|
+		| `json` | https://www.json.org |
+		| `yaml` | https://yaml.org     |
+		| `toml` | https://toml.io      |
+
 		@param format The format to use
 		@param encoded The string to decode
 		@return The decoded lua value
@@ -61,6 +77,15 @@ return {
 
 		Compresses the given string using the given format.
 
+		Currently supported formats:
+
+		| Name     | Learn More                        |
+		|:---------|:----------------------------------|
+		| `brotli` | https://github.com/google/brotli  |
+		| `gzip`   | https://www.gnu.org/software/gzip |
+		| `lz4`    | https://github.com/lz4/lz4        |
+		| `zlib`   | https://www.zlib.net              |
+
 		@param format The format to use
 		@param encoded The string to compress
 		@return The compressed string
@@ -74,6 +99,15 @@ return {
 
 		Decompresses the given string using the given format.
 
+		Currently supported formats:
+
+		| Name     | Learn More                        |
+		|:---------|:----------------------------------|
+		| `brotli` | https://github.com/google/brotli  |
+		| `gzip`   | https://www.gnu.org/software/gzip |
+		| `lz4`    | https://github.com/lz4/lz4        |
+		| `zlib`   | https://www.zlib.net              |
+
 		@param format The format to use
 		@param encoded The string to decompress
 		@return The decompressed string

packages/lib/Cargo.toml

@@ -38,6 +38,7 @@ async-trait = "0.1"
 blocking = "1.3"
 dialoguer = "0.10"
 dunce = "1.0"
+lz4_flex = "0.10"
 pin-project = "1.0"
 os_str_bytes = "6.4"
 urlencoding = "2.1.2"

packages/lib/src/lua/serde/compress_decompress.rs

@@ -1,6 +1,8 @@
 use async_compression::tokio::write::{
     BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder,
 };
+use blocking::unblock;
+use lz4_flex::{compress_prepend_size, decompress_size_prepended};
 use mlua::prelude::*;
 use tokio::io::AsyncWriteExt;
 
@@ -8,27 +10,44 @@ use tokio::io::AsyncWriteExt;
 pub enum CompressDecompressFormat {
     Brotli,
     GZip,
+    LZ4,
     ZLib,
 }
 
 #[allow(dead_code)]
 impl CompressDecompressFormat {
     pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option<Self> {
-        let bytes = bytes.as_ref();
+        match bytes.as_ref() {
-        if bytes[0..4] == [0x0B, 0x24, 0x72, 0x68] {
+            // https://github.com/PSeitz/lz4_flex/blob/main/src/frame/header.rs#L28
-            Some(Self::Brotli)
+            b if b.len() >= 4
-        } else if bytes[0..3] == [0x1F, 0x8B, 0x08] {
+                && matches!(
-            Some(Self::GZip)
+                    u32::from_le_bytes(b[0..4].try_into().unwrap()),
-        }
+                    0x184D2204 | 0x184C2102
-        // https://stackoverflow.com/a/54915442
+                ) =>
-        else if (bytes[0..2] == [0x78, 0x01])
+            {
-            || (bytes[0..2] == [0x78, 0x5E])
+                Some(Self::LZ4)
-            || (bytes[0..2] == [0x78, 0x9C])
+            }
-            || (bytes[0..2] == [0x78, 0xDA])
+            // https://github.com/dropbox/rust-brotli/blob/master/src/enc/brotli_bit_stream.rs#L2805
-        {
+            b if b.len() >= 4
-            Some(Self::ZLib)
+                && matches!(
-        } else {
+                    b[0..3],
-            None
+                    [0xE1, 0x97, 0x81] | [0xE1, 0x97, 0x82] | [0xE1, 0x97, 0x80]
+                ) =>
+            {
+                Some(Self::Brotli)
+            }
+            // https://github.com/rust-lang/flate2-rs/blob/main/src/gz/mod.rs#L135
+            b if b.len() >= 3 && matches!(b[0..3], [0x1F, 0x8B, 0x08]) => Some(Self::GZip),
+            // https://stackoverflow.com/a/43170354
+            b if b.len() >= 2
+                && matches!(
+                    b[0..2],
+                    [0x78, 0x01] | [0x78, 0x5E] | [0x78, 0x9C] | [0x78, 0xDA]
+                ) =>
+            {
+                Some(Self::ZLib)
+            }
+            _ => None,
         }
     }
 
@@ -49,12 +68,13 @@ impl<'lua> FromLua<'lua> for CompressDecompressFormat {
             match s.to_string_lossy().to_ascii_lowercase().trim() {
                 "brotli" => Ok(Self::Brotli),
                 "gzip" => Ok(Self::GZip),
+                "lz4" => Ok(Self::LZ4),
                 "zlib" => Ok(Self::ZLib),
                 kind => Err(LuaError::FromLuaConversionError {
                     from: value.type_name(),
                     to: "CompressDecompressFormat",
                     message: Some(format!(
-                        "Invalid format '{kind}', valid formats are:  brotli, gzip, zlib"
+                        "Invalid format '{kind}', valid formats are:  brotli, gzip, lz4, zlib"
                     )),
                 }),
             }
@@ -89,6 +109,10 @@ pub async fn compress<'lua>(
                 .write_all(source.as_ref())
                 .await?
         }
+        CompressDecompressFormat::LZ4 => {
+            let source = source.as_ref().to_vec();
+            bytes = unblock(move || compress_prepend_size(&source)).await;
+        }
     }
     Ok(bytes)
 }
@@ -114,6 +138,12 @@ pub async fn decompress<'lua>(
                 .write_all(source.as_ref())
                 .await?
         }
+        CompressDecompressFormat::LZ4 => {
+            let source = source.as_ref().to_vec();
+            bytes = unblock(move || decompress_size_prepended(&source))
+                .await
+                .map_err(LuaError::external)?;
+        }
     }
     Ok(bytes)
 }