Implement compression in serde builtin

CHANGELOG.md

@@ -10,6 +10,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+-   Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, or `zlib`.
+
+    Example usage:
+
+    ```lua
+    local INPUT = string.rep("Input string to compress", 16)
+
+    local serde = require("@lune/serde")
+
+    local compressed = serde.compress("gzip", INPUT)
+    local decompressed = serde.decompress("gzip", compressed)
+
+    assert(compressed == "H4sIAAAAAAAAA/PMKygtUSguKcrMS1coyVdIzs8tKEotLvYcFaeLOADSF8BBgAEAAA==")
+    assert(decompressed == INPUT)
+    ```
+
 ### Changed
 
 -   Both `stdio.write` and `stdio.ewrite` now support writing arbitrary bytes, instead of only valid UTF-8.

Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
 [[package]]
 name = "aho-corasick"
 version = "1.0.1"
@@ -11,6 +17,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "alloc-no-stdlib"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -104,6 +125,20 @@ dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "async-compression"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11"
+dependencies = [
+ "brotli",
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-lock"
 version = "2.7.0"
@@ -232,6 +267,27 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "brotli"
+version = "3.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
 [[package]]
 name = "bstr"
 version = "0.2.17"
@@ -411,6 +467,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.15"
@@ -596,6 +661,16 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1065,6 +1140,7 @@ name = "lune"
 version = "0.6.7"
 dependencies = [
  "anyhow",
+ "async-compression",
  "async-trait",
  "blocking",
  "console",
@@ -1161,6 +1237,15 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.6"

docs/typedefs/Serde.luau

@@ -1,9 +1,14 @@
 export type EncodeDecodeFormat = "json" | "yaml" | "toml"
 
+export type CompressDecompressFormat = "brotli" | "gzip" | "zlib"
+
 --[=[
 	@class Serde
 
-	Built-in serialization/deserialization & encoding/decoding functions
+	Built-in library for:
+	- serialization & deserialization
+	- encoding & decoding
+	- compression
 
 	### Example usage
 
@@ -50,4 +55,30 @@ return {
 	decode = function(format: EncodeDecodeFormat, encoded: string): any
 		return nil :: any
 	end,
+	--[=[
+		@within Serde
+		@must_use
+
+		Compresses the given string using the given format.
+
+		@param format The format to use
+		@param encoded The string to compress
+		@return The compressed string
+	]=]
+	compress = function(format: CompressDecompressFormat, s: string): string
+		return nil :: any
+	end,
+	--[=[
+		@within Serde
+		@must_use
+
+		Decompresses the given string using the given format.
+
+		@param format The format to use
+		@param encoded The string to decompress
+		@return The decompressed string
+	]=]
+	decompress = function(format: CompressDecompressFormat, s: string): string
+		return nil :: any
+	end,
 }

packages/lib/Cargo.toml

@@ -42,6 +42,13 @@ pin-project = "1.0"
 os_str_bytes = "6.4"
 urlencoding = "2.1.2"
 
+async-compression = { version = "0.4", features = [
+	"tokio",
+	"brotli",
+	"deflate",
+	"gzip",
+	"zlib",
+] }
 hyper = { version = "0.14", features = ["full"] }
 hyper-tungstenite = { version = "0.9" }
 tokio-tungstenite = { version = "0.18" }

packages/lib/src/builtins/serde.rs

@@ -1,7 +1,9 @@
 use mlua::prelude::*;
 
 use crate::lua::{
-    serde::{EncodeDecodeConfig, EncodeDecodeFormat},
+    serde::{
+        compress, decompress, CompressDecompressFormat, EncodeDecodeConfig, EncodeDecodeFormat,
+    },
     table::TableBuilder,
 };
 
@@ -9,6 +11,8 @@ pub fn create(lua: &'static Lua) -> LuaResult<LuaTable> {
     TableBuilder::new(lua)?
         .with_function("encode", serde_encode)?
         .with_function("decode", serde_decode)?
+        .with_async_function("compress", serde_compress)?
+        .with_async_function("decompress", serde_decompress)?
         .build_readonly()
 }
 
@@ -27,3 +31,19 @@ fn serde_decode<'a>(
     let config = EncodeDecodeConfig::from(format);
     config.deserialize_from_string(lua, str)
 }
+
+async fn serde_compress<'a>(
+    lua: &'static Lua,
+    (format, str): (CompressDecompressFormat, LuaString<'a>),
+) -> LuaResult<LuaString<'a>> {
+    let bytes = compress(format, str).await?;
+    lua.create_string(&bytes)
+}
+
+async fn serde_decompress<'a>(
+    lua: &'static Lua,
+    (format, str): (CompressDecompressFormat, LuaString<'a>),
+) -> LuaResult<LuaString<'a>> {
+    let bytes = decompress(format, str).await?;
+    lua.create_string(&bytes)
+}

packages/lib/src/lua/serde/compress_decompress.rs

@@ -0,0 +1,119 @@
+use async_compression::tokio::write::{
+    BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder,
+};
+use mlua::prelude::*;
+use tokio::io::AsyncWriteExt;
+
+#[derive(Debug, Clone, Copy)]
+pub enum CompressDecompressFormat {
+    Brotli,
+    GZip,
+    ZLib,
+}
+
+#[allow(dead_code)]
+impl CompressDecompressFormat {
+    pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option<Self> {
+        let bytes = bytes.as_ref();
+        if bytes[0..4] == [0x0B, 0x24, 0x72, 0x68] {
+            Some(Self::Brotli)
+        } else if bytes[0..3] == [0x1F, 0x8B, 0x08] {
+            Some(Self::GZip)
+        }
+        // https://stackoverflow.com/a/54915442
+        else if (bytes[0..2] == [0x78, 0x01])
+            || (bytes[0..2] == [0x78, 0x5E])
+            || (bytes[0..2] == [0x78, 0x9C])
+            || (bytes[0..2] == [0x78, 0xDA])
+        {
+            Some(Self::ZLib)
+        } else {
+            None
+        }
+    }
+
+    pub fn detect_from_header_str(header: impl AsRef<str>) -> Option<Self> {
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#directives
+        match header.as_ref().to_ascii_lowercase().trim() {
+            "br" => Some(Self::Brotli),
+            "deflate" => Some(Self::ZLib),
+            "gzip" => Some(Self::GZip),
+            _ => None,
+        }
+    }
+}
+
+impl<'lua> FromLua<'lua> for CompressDecompressFormat {
+    fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult<Self> {
+        if let LuaValue::String(s) = &value {
+            match s.to_string_lossy().to_ascii_lowercase().trim() {
+                "brotli" => Ok(Self::Brotli),
+                "gzip" => Ok(Self::GZip),
+                "zlib" => Ok(Self::ZLib),
+                kind => Err(LuaError::FromLuaConversionError {
+                    from: value.type_name(),
+                    to: "CompressDecompressFormat",
+                    message: Some(format!(
+                        "Invalid format '{kind}', valid formats are:  brotli, gzip, zlib"
+                    )),
+                }),
+            }
+        } else {
+            Err(LuaError::FromLuaConversionError {
+                from: value.type_name(),
+                to: "CompressDecompressFormat",
+                message: None,
+            })
+        }
+    }
+}
+
+pub async fn compress<'lua>(
+    format: CompressDecompressFormat,
+    source: impl AsRef<[u8]>,
+) -> LuaResult<Vec<u8>> {
+    let mut bytes = Vec::new();
+    match format {
+        CompressDecompressFormat::Brotli => {
+            BrotliEncoder::new(&mut bytes)
+                .write_all(source.as_ref())
+                .await?
+        }
+        CompressDecompressFormat::GZip => {
+            GzipEncoder::new(&mut bytes)
+                .write_all(source.as_ref())
+                .await?
+        }
+        CompressDecompressFormat::ZLib => {
+            ZlibEncoder::new(&mut bytes)
+                .write_all(source.as_ref())
+                .await?
+        }
+    }
+    Ok(bytes)
+}
+
+pub async fn decompress<'lua>(
+    format: CompressDecompressFormat,
+    source: impl AsRef<[u8]>,
+) -> LuaResult<Vec<u8>> {
+    let mut bytes = Vec::new();
+    match format {
+        CompressDecompressFormat::Brotli => {
+            BrotliDecoder::new(&mut bytes)
+                .write_all(source.as_ref())
+                .await?
+        }
+        CompressDecompressFormat::GZip => {
+            GzipDecoder::new(&mut bytes)
+                .write_all(source.as_ref())
+                .await?
+        }
+        CompressDecompressFormat::ZLib => {
+            ZlibDecoder::new(&mut bytes)
+                .write_all(source.as_ref())
+                .await?
+        }
+    }
+    Ok(bytes)
+}

packages/lib/src/lua/serde/encode_decode.rs

@@ -4,8 +4,6 @@ use serde_json::Value as JsonValue;
 use serde_yaml::Value as YamlValue;
 use toml::Value as TomlValue;
 
-// Serde config
-
 #[derive(Debug, Clone, Copy)]
 pub enum EncodeDecodeFormat {
     Json,

packages/lib/src/lua/serde/mod.rs

@@ -1,3 +1,5 @@
+mod compress_decompress;
 mod encode_decode;
 
+pub use compress_decompress::{compress, decompress, CompressDecompressFormat};
 pub use encode_decode::{EncodeDecodeConfig, EncodeDecodeFormat};