diff --git a/Cargo.lock b/Cargo.lock index d757106..91bbf9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1623,8 +1623,16 @@ dependencies = [ name = "lune-std-serde" version = "0.1.0" dependencies = [ + "async-compression", + "bstr", "lune-utils", + "lz4_flex", "mlua", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "toml", ] [[package]] diff --git a/crates/lune-std-serde/Cargo.toml b/crates/lune-std-serde/Cargo.toml index 96afde8..35f2ba3 100644 --- a/crates/lune-std-serde/Cargo.toml +++ b/crates/lune-std-serde/Cargo.toml @@ -13,4 +13,20 @@ workspace = true [dependencies] mlua = { version = "0.9.7", features = ["luau"] } +async-compression = { version = "0.4", features = [ + "tokio", + "brotli", + "deflate", + "gzip", + "zlib", +] } +bstr = "1.9" +lz4_flex = "0.11" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["preserve_order"] } +serde_yaml = "0.9" +toml = { version = "0.8", features = ["preserve_order"] } + +tokio = { version = "1", default-features = false } + lune-utils = { version = "0.1.0", path = "../lune-utils" } diff --git a/crates/lune-std-serde/src/compress_decompress.rs b/crates/lune-std-serde/src/compress_decompress.rs new file mode 100644 index 0000000..837625b --- /dev/null +++ b/crates/lune-std-serde/src/compress_decompress.rs @@ -0,0 +1,189 @@ +use mlua::prelude::*; + +use lz4_flex::{compress_prepend_size, decompress_size_prepended}; +use tokio::{ + io::{copy, BufReader}, + task::spawn_blocking, +}; + +use async_compression::{ + tokio::bufread::{ + BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder, + }, + Level::Best as CompressionQuality, +}; + +/** + A compression and decompression format supported by Lune. +*/ +#[derive(Debug, Clone, Copy)] +pub enum CompressDecompressFormat { + Brotli, + GZip, + LZ4, + ZLib, +} + +#[allow(dead_code)] +impl CompressDecompressFormat { + /** + Detects a supported compression format from the given bytes. + */ + #[allow(clippy::missing_panics_doc)] + pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option { + match bytes.as_ref() { + // https://github.com/PSeitz/lz4_flex/blob/main/src/frame/header.rs#L28 + b if b.len() >= 4 + && matches!( + u32::from_le_bytes(b[0..4].try_into().unwrap()), + 0x184D2204 | 0x184C2102 + ) => + { + Some(Self::LZ4) + } + // https://github.com/dropbox/rust-brotli/blob/master/src/enc/brotli_bit_stream.rs#L2805 + b if b.len() >= 4 + && matches!( + b[0..3], + [0xE1, 0x97, 0x81] | [0xE1, 0x97, 0x82] | [0xE1, 0x97, 0x80] + ) => + { + Some(Self::Brotli) + } + // https://github.com/rust-lang/flate2-rs/blob/main/src/gz/mod.rs#L135 + b if b.len() >= 3 && matches!(b[0..3], [0x1F, 0x8B, 0x08]) => Some(Self::GZip), + // https://stackoverflow.com/a/43170354 + b if b.len() >= 2 + && matches!( + b[0..2], + [0x78, 0x01] | [0x78, 0x5E] | [0x78, 0x9C] | [0x78, 0xDA] + ) => + { + Some(Self::ZLib) + } + _ => None, + } + } + + /** + Detects a supported compression format from the given header string. + + The given header script should be a valid `Content-Encoding` header value. + */ + pub fn detect_from_header_str(header: impl AsRef) -> Option { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#directives + match header.as_ref().to_ascii_lowercase().trim() { + "br" | "brotli" => Some(Self::Brotli), + "deflate" => Some(Self::ZLib), + "gz" | "gzip" => Some(Self::GZip), + _ => None, + } + } +} + +impl<'lua> FromLua<'lua> for CompressDecompressFormat { + fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult { + if let LuaValue::String(s) = &value { + match s.to_string_lossy().to_ascii_lowercase().trim() { + "brotli" => Ok(Self::Brotli), + "gzip" => Ok(Self::GZip), + "lz4" => Ok(Self::LZ4), + "zlib" => Ok(Self::ZLib), + kind => Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "CompressDecompressFormat", + message: Some(format!( + "Invalid format '{kind}', valid formats are: brotli, gzip, lz4, zlib" + )), + }), + } + } else { + Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "CompressDecompressFormat", + message: None, + }) + } + } +} + +/** + Compresses the given bytes using the specified format. + + # Errors + + Errors when the compression fails. +*/ +pub async fn compress<'lua>( + source: impl AsRef<[u8]>, + format: CompressDecompressFormat, +) -> LuaResult> { + if let CompressDecompressFormat::LZ4 = format { + let source = source.as_ref().to_vec(); + return spawn_blocking(move || compress_prepend_size(&source)) + .await + .into_lua_err(); + } + + let mut bytes = Vec::new(); + let reader = BufReader::new(source.as_ref()); + + match format { + CompressDecompressFormat::Brotli => { + let mut encoder = BrotliEncoder::with_quality(reader, CompressionQuality); + copy(&mut encoder, &mut bytes).await?; + } + CompressDecompressFormat::GZip => { + let mut encoder = GzipEncoder::with_quality(reader, CompressionQuality); + copy(&mut encoder, &mut bytes).await?; + } + CompressDecompressFormat::ZLib => { + let mut encoder = ZlibEncoder::with_quality(reader, CompressionQuality); + copy(&mut encoder, &mut bytes).await?; + } + CompressDecompressFormat::LZ4 => unreachable!(), + } + + Ok(bytes) +} + +/** + Decompresses the given bytes using the specified format. + + # Errors + + Errors when the decompression fails. +*/ +pub async fn decompress<'lua>( + source: impl AsRef<[u8]>, + format: CompressDecompressFormat, +) -> LuaResult> { + if let CompressDecompressFormat::LZ4 = format { + let source = source.as_ref().to_vec(); + return spawn_blocking(move || decompress_size_prepended(&source)) + .await + .into_lua_err()? + .into_lua_err(); + } + + let mut bytes = Vec::new(); + let reader = BufReader::new(source.as_ref()); + + match format { + CompressDecompressFormat::Brotli => { + let mut decoder = BrotliDecoder::new(reader); + copy(&mut decoder, &mut bytes).await?; + } + CompressDecompressFormat::GZip => { + let mut decoder = GzipDecoder::new(reader); + copy(&mut decoder, &mut bytes).await?; + } + CompressDecompressFormat::ZLib => { + let mut decoder = ZlibDecoder::new(reader); + copy(&mut decoder, &mut bytes).await?; + } + CompressDecompressFormat::LZ4 => unreachable!(), + } + + Ok(bytes) +} diff --git a/crates/lune-std-serde/src/encode_decode.rs b/crates/lune-std-serde/src/encode_decode.rs new file mode 100644 index 0000000..80e1a5f --- /dev/null +++ b/crates/lune-std-serde/src/encode_decode.rs @@ -0,0 +1,158 @@ +use mlua::prelude::*; + +use serde_json::Value as JsonValue; +use serde_yaml::Value as YamlValue; +use toml::Value as TomlValue; + +// NOTE: These are options for going from other format -> lua ("serializing" lua values) +const LUA_SERIALIZE_OPTIONS: LuaSerializeOptions = LuaSerializeOptions::new() + .set_array_metatable(false) + .serialize_none_to_null(false) + .serialize_unit_to_null(false); + +// NOTE: These are options for going from lua -> other format ("deserializing" lua values) +const LUA_DESERIALIZE_OPTIONS: LuaDeserializeOptions = LuaDeserializeOptions::new() + .sort_keys(true) + .deny_recursive_tables(false) + .deny_unsupported_types(true); + +/** + An encoding and decoding format supported by Lune. + + Encode / decode in this case is synonymous with serialize / deserialize. +*/ +#[derive(Debug, Clone, Copy)] +pub enum EncodeDecodeFormat { + Json, + Yaml, + Toml, +} + +impl<'lua> FromLua<'lua> for EncodeDecodeFormat { + fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult { + if let LuaValue::String(s) = &value { + match s.to_string_lossy().to_ascii_lowercase().trim() { + "json" => Ok(Self::Json), + "yaml" => Ok(Self::Yaml), + "toml" => Ok(Self::Toml), + kind => Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "EncodeDecodeFormat", + message: Some(format!( + "Invalid format '{kind}', valid formats are: json, yaml, toml" + )), + }), + } + } else { + Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "EncodeDecodeFormat", + message: None, + }) + } + } +} + +/** + Configuration for encoding and decoding values. + + Encoding / decoding in this case is synonymous with serialize / deserialize. +*/ +#[derive(Debug, Clone, Copy)] +pub struct EncodeDecodeConfig { + pub format: EncodeDecodeFormat, + pub pretty: bool, +} + +impl From for EncodeDecodeConfig { + fn from(format: EncodeDecodeFormat) -> Self { + Self { + format, + pretty: false, + } + } +} + +impl From<(EncodeDecodeFormat, bool)> for EncodeDecodeConfig { + fn from(value: (EncodeDecodeFormat, bool)) -> Self { + Self { + format: value.0, + pretty: value.1, + } + } +} + +/** + Encodes / serializes the given value into a string, using the specified configuration. + + # Errors + + Errors when the encoding fails. +*/ +pub fn encode<'lua>( + value: LuaValue<'lua>, + lua: &'lua Lua, + config: EncodeDecodeConfig, +) -> LuaResult> { + let bytes = match config.format { + EncodeDecodeFormat::Json => { + let serialized: JsonValue = lua.from_value_with(value, LUA_DESERIALIZE_OPTIONS)?; + if config.pretty { + serde_json::to_vec_pretty(&serialized).into_lua_err()? + } else { + serde_json::to_vec(&serialized).into_lua_err()? + } + } + EncodeDecodeFormat::Yaml => { + let serialized: YamlValue = lua.from_value_with(value, LUA_DESERIALIZE_OPTIONS)?; + let mut writer = Vec::with_capacity(128); + serde_yaml::to_writer(&mut writer, &serialized).into_lua_err()?; + writer + } + EncodeDecodeFormat::Toml => { + let serialized: TomlValue = lua.from_value_with(value, LUA_DESERIALIZE_OPTIONS)?; + let s = if config.pretty { + toml::to_string_pretty(&serialized).into_lua_err()? + } else { + toml::to_string(&serialized).into_lua_err()? + }; + s.as_bytes().to_vec() + } + }; + lua.create_string(bytes) +} + +/** + Decodes / deserializes the given string into a value, using the specified configuration. + + # Errors + + Errors when the decoding fails. +*/ +pub fn decode( + bytes: impl AsRef<[u8]>, + lua: &Lua, + config: EncodeDecodeConfig, +) -> LuaResult { + let bytes = bytes.as_ref(); + match config.format { + EncodeDecodeFormat::Json => { + let value: JsonValue = serde_json::from_slice(bytes).into_lua_err()?; + lua.to_value_with(&value, LUA_SERIALIZE_OPTIONS) + } + EncodeDecodeFormat::Yaml => { + let value: YamlValue = serde_yaml::from_slice(bytes).into_lua_err()?; + lua.to_value_with(&value, LUA_SERIALIZE_OPTIONS) + } + EncodeDecodeFormat::Toml => { + if let Ok(s) = String::from_utf8(bytes.to_vec()) { + let value: TomlValue = toml::from_str(&s).into_lua_err()?; + lua.to_value_with(&value, LUA_SERIALIZE_OPTIONS) + } else { + Err(LuaError::RuntimeError( + "TOML must be valid utf-8".to_string(), + )) + } + } + } +} diff --git a/crates/lune-std-serde/src/lib.rs b/crates/lune-std-serde/src/lib.rs index 38f6980..4514a75 100644 --- a/crates/lune-std-serde/src/lib.rs +++ b/crates/lune-std-serde/src/lib.rs @@ -1,9 +1,16 @@ #![allow(clippy::cargo_common_metadata)] +use bstr::BString; use mlua::prelude::*; use lune_utils::TableBuilder; +mod compress_decompress; +mod encode_decode; + +pub use self::compress_decompress::{compress, decompress, CompressDecompressFormat}; +pub use self::encode_decode::{decode, encode, EncodeDecodeConfig, EncodeDecodeFormat}; + /** Creates the `serde` standard library module. @@ -12,5 +19,39 @@ use lune_utils::TableBuilder; Errors when out of memory. */ pub fn module(lua: &Lua) -> LuaResult { - TableBuilder::new(lua)?.build_readonly() + TableBuilder::new(lua)? + .with_function("encode", serde_encode)? + .with_function("decode", serde_decode)? + .with_async_function("compress", serde_compress)? + .with_async_function("decompress", serde_decompress)? + .build_readonly() +} + +fn serde_encode<'lua>( + lua: &'lua Lua, + (format, value, pretty): (EncodeDecodeFormat, LuaValue<'lua>, Option), +) -> LuaResult> { + let config = EncodeDecodeConfig::from((format, pretty.unwrap_or_default())); + encode(value, lua, config) +} + +fn serde_decode(lua: &Lua, (format, bs): (EncodeDecodeFormat, BString)) -> LuaResult { + let config = EncodeDecodeConfig::from(format); + decode(bs, lua, config) +} + +async fn serde_compress( + lua: &Lua, + (format, bs): (CompressDecompressFormat, BString), +) -> LuaResult { + let bytes = compress(bs, format).await?; + lua.create_string(bytes) +} + +async fn serde_decompress( + lua: &Lua, + (format, bs): (CompressDecompressFormat, BString), +) -> LuaResult { + let bytes = decompress(bs, format).await?; + lua.create_string(bytes) }