From 860c696212b10a64c6a848706af019ef1be2f01d Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Fri, 19 May 2023 17:01:59 +0200 Subject: [PATCH] Implement compression in serde builtin --- CHANGELOG.md | 18 +++ Cargo.lock | 85 +++++++++++++ docs/typedefs/Serde.luau | 33 ++++- packages/lib/Cargo.toml | 7 ++ packages/lib/src/builtins/serde.rs | 22 +++- .../lib/src/lua/serde/compress_decompress.rs | 119 ++++++++++++++++++ packages/lib/src/lua/serde/encode_decode.rs | 2 - packages/lib/src/lua/serde/mod.rs | 2 + 8 files changed, 284 insertions(+), 4 deletions(-) create mode 100644 packages/lib/src/lua/serde/compress_decompress.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index eb4c7f7..a055fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, or `zlib`. + + Example usage: + + ```lua + local INPUT = string.rep("Input string to compress", 16) + + local serde = require("@lune/serde") + + local compressed = serde.compress("gzip", INPUT) + local decompressed = serde.decompress("gzip", compressed) + + assert(compressed == "H4sIAAAAAAAAA/PMKygtUSguKcrMS1coyVdIzs8tKEotLvYcFaeLOADSF8BBgAEAAA==") + assert(decompressed == INPUT) + ``` + ### Changed - Both `stdio.write` and `stdio.ewrite` now support writing arbitrary bytes, instead of only valid UTF-8. diff --git a/Cargo.lock b/Cargo.lock index cf8c3ae..f81d19c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.0.1" @@ -11,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -104,6 +125,20 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-compression" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-lock" version = "2.7.0" @@ -232,6 +267,27 @@ dependencies = [ "log", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "0.2.17" @@ -411,6 +467,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -596,6 +661,16 @@ dependencies = [ "instant", ] +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1065,6 +1140,7 @@ name = "lune" version = "0.6.7" dependencies = [ "anyhow", + "async-compression", "async-trait", "blocking", "console", @@ -1161,6 +1237,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.6" diff --git a/docs/typedefs/Serde.luau b/docs/typedefs/Serde.luau index 8ca0a2b..f6fd70a 100644 --- a/docs/typedefs/Serde.luau +++ b/docs/typedefs/Serde.luau @@ -1,9 +1,14 @@ export type EncodeDecodeFormat = "json" | "yaml" | "toml" +export type CompressDecompressFormat = "brotli" | "gzip" | "zlib" + --[=[ @class Serde - Built-in serialization/deserialization & encoding/decoding functions + Built-in library for: + - serialization & deserialization + - encoding & decoding + - compression ### Example usage @@ -50,4 +55,30 @@ return { decode = function(format: EncodeDecodeFormat, encoded: string): any return nil :: any end, + --[=[ + @within Serde + @must_use + + Compresses the given string using the given format. + + @param format The format to use + @param encoded The string to compress + @return The compressed string + ]=] + compress = function(format: CompressDecompressFormat, s: string): string + return nil :: any + end, + --[=[ + @within Serde + @must_use + + Decompresses the given string using the given format. + + @param format The format to use + @param encoded The string to decompress + @return The decompressed string + ]=] + decompress = function(format: CompressDecompressFormat, s: string): string + return nil :: any + end, } diff --git a/packages/lib/Cargo.toml b/packages/lib/Cargo.toml index b82a02f..63b186a 100644 --- a/packages/lib/Cargo.toml +++ b/packages/lib/Cargo.toml @@ -42,6 +42,13 @@ pin-project = "1.0" os_str_bytes = "6.4" urlencoding = "2.1.2" +async-compression = { version = "0.4", features = [ + "tokio", + "brotli", + "deflate", + "gzip", + "zlib", +] } hyper = { version = "0.14", features = ["full"] } hyper-tungstenite = { version = "0.9" } tokio-tungstenite = { version = "0.18" } diff --git a/packages/lib/src/builtins/serde.rs b/packages/lib/src/builtins/serde.rs index 9beaf5e..7b5cf3d 100644 --- a/packages/lib/src/builtins/serde.rs +++ b/packages/lib/src/builtins/serde.rs @@ -1,7 +1,9 @@ use mlua::prelude::*; use crate::lua::{ - serde::{EncodeDecodeConfig, EncodeDecodeFormat}, + serde::{ + compress, decompress, CompressDecompressFormat, EncodeDecodeConfig, EncodeDecodeFormat, + }, table::TableBuilder, }; @@ -9,6 +11,8 @@ pub fn create(lua: &'static Lua) -> LuaResult { TableBuilder::new(lua)? .with_function("encode", serde_encode)? .with_function("decode", serde_decode)? + .with_async_function("compress", serde_compress)? + .with_async_function("decompress", serde_decompress)? .build_readonly() } @@ -27,3 +31,19 @@ fn serde_decode<'a>( let config = EncodeDecodeConfig::from(format); config.deserialize_from_string(lua, str) } + +async fn serde_compress<'a>( + lua: &'static Lua, + (format, str): (CompressDecompressFormat, LuaString<'a>), +) -> LuaResult> { + let bytes = compress(format, str).await?; + lua.create_string(&bytes) +} + +async fn serde_decompress<'a>( + lua: &'static Lua, + (format, str): (CompressDecompressFormat, LuaString<'a>), +) -> LuaResult> { + let bytes = decompress(format, str).await?; + lua.create_string(&bytes) +} diff --git a/packages/lib/src/lua/serde/compress_decompress.rs b/packages/lib/src/lua/serde/compress_decompress.rs new file mode 100644 index 0000000..4759f0e --- /dev/null +++ b/packages/lib/src/lua/serde/compress_decompress.rs @@ -0,0 +1,119 @@ +use async_compression::tokio::write::{ + BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder, +}; +use mlua::prelude::*; +use tokio::io::AsyncWriteExt; + +#[derive(Debug, Clone, Copy)] +pub enum CompressDecompressFormat { + Brotli, + GZip, + ZLib, +} + +#[allow(dead_code)] +impl CompressDecompressFormat { + pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option { + let bytes = bytes.as_ref(); + if bytes[0..4] == [0x0B, 0x24, 0x72, 0x68] { + Some(Self::Brotli) + } else if bytes[0..3] == [0x1F, 0x8B, 0x08] { + Some(Self::GZip) + } + // https://stackoverflow.com/a/54915442 + else if (bytes[0..2] == [0x78, 0x01]) + || (bytes[0..2] == [0x78, 0x5E]) + || (bytes[0..2] == [0x78, 0x9C]) + || (bytes[0..2] == [0x78, 0xDA]) + { + Some(Self::ZLib) + } else { + None + } + } + + pub fn detect_from_header_str(header: impl AsRef) -> Option { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#directives + match header.as_ref().to_ascii_lowercase().trim() { + "br" => Some(Self::Brotli), + "deflate" => Some(Self::ZLib), + "gzip" => Some(Self::GZip), + _ => None, + } + } +} + +impl<'lua> FromLua<'lua> for CompressDecompressFormat { + fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult { + if let LuaValue::String(s) = &value { + match s.to_string_lossy().to_ascii_lowercase().trim() { + "brotli" => Ok(Self::Brotli), + "gzip" => Ok(Self::GZip), + "zlib" => Ok(Self::ZLib), + kind => Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "CompressDecompressFormat", + message: Some(format!( + "Invalid format '{kind}', valid formats are: brotli, gzip, zlib" + )), + }), + } + } else { + Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "CompressDecompressFormat", + message: None, + }) + } + } +} + +pub async fn compress<'lua>( + format: CompressDecompressFormat, + source: impl AsRef<[u8]>, +) -> LuaResult> { + let mut bytes = Vec::new(); + match format { + CompressDecompressFormat::Brotli => { + BrotliEncoder::new(&mut bytes) + .write_all(source.as_ref()) + .await? + } + CompressDecompressFormat::GZip => { + GzipEncoder::new(&mut bytes) + .write_all(source.as_ref()) + .await? + } + CompressDecompressFormat::ZLib => { + ZlibEncoder::new(&mut bytes) + .write_all(source.as_ref()) + .await? + } + } + Ok(bytes) +} + +pub async fn decompress<'lua>( + format: CompressDecompressFormat, + source: impl AsRef<[u8]>, +) -> LuaResult> { + let mut bytes = Vec::new(); + match format { + CompressDecompressFormat::Brotli => { + BrotliDecoder::new(&mut bytes) + .write_all(source.as_ref()) + .await? + } + CompressDecompressFormat::GZip => { + GzipDecoder::new(&mut bytes) + .write_all(source.as_ref()) + .await? + } + CompressDecompressFormat::ZLib => { + ZlibDecoder::new(&mut bytes) + .write_all(source.as_ref()) + .await? + } + } + Ok(bytes) +} diff --git a/packages/lib/src/lua/serde/encode_decode.rs b/packages/lib/src/lua/serde/encode_decode.rs index 5caeccf..a4d62e2 100644 --- a/packages/lib/src/lua/serde/encode_decode.rs +++ b/packages/lib/src/lua/serde/encode_decode.rs @@ -4,8 +4,6 @@ use serde_json::Value as JsonValue; use serde_yaml::Value as YamlValue; use toml::Value as TomlValue; -// Serde config - #[derive(Debug, Clone, Copy)] pub enum EncodeDecodeFormat { Json, diff --git a/packages/lib/src/lua/serde/mod.rs b/packages/lib/src/lua/serde/mod.rs index 6cbbbcf..ab554ef 100644 --- a/packages/lib/src/lua/serde/mod.rs +++ b/packages/lib/src/lua/serde/mod.rs @@ -1,3 +1,5 @@ +mod compress_decompress; mod encode_decode; +pub use compress_decompress::{compress, decompress, CompressDecompressFormat}; pub use encode_decode::{EncodeDecodeConfig, EncodeDecodeFormat};