Migrate serde builtin to lune-std-serde crate

Cargo.lock

@@ -1623,8 +1623,16 @@ dependencies = [
 name = "lune-std-serde"
 version = "0.1.0"
 dependencies = [
+ "async-compression",
+ "bstr",
  "lune-utils",
+ "lz4_flex",
  "mlua",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "tokio",
+ "toml",
 ]
 
 [[package]]

crates/lune-std-serde/Cargo.toml

@@ -13,4 +13,20 @@ workspace = true
 [dependencies]
 mlua = { version = "0.9.7", features = ["luau"] }
 
+async-compression = { version = "0.4", features = [
+    "tokio",
+    "brotli",
+    "deflate",
+    "gzip",
+    "zlib",
+] }
+bstr = "1.9"
+lz4_flex = "0.11"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = ["preserve_order"] }
+serde_yaml = "0.9"
+toml = { version = "0.8", features = ["preserve_order"] }
+
+tokio = { version = "1", default-features = false }
+
 lune-utils = { version = "0.1.0", path = "../lune-utils" }

crates/lune-std-serde/src/compress_decompress.rs

@@ -0,0 +1,189 @@
+use mlua::prelude::*;
+
+use lz4_flex::{compress_prepend_size, decompress_size_prepended};
+use tokio::{
+    io::{copy, BufReader},
+    task::spawn_blocking,
+};
+
+use async_compression::{
+    tokio::bufread::{
+        BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder,
+    },
+    Level::Best as CompressionQuality,
+};
+
+/**
+    A compression and decompression format supported by Lune.
+*/
+#[derive(Debug, Clone, Copy)]
+pub enum CompressDecompressFormat {
+    Brotli,
+    GZip,
+    LZ4,
+    ZLib,
+}
+
+#[allow(dead_code)]
+impl CompressDecompressFormat {
+    /**
+        Detects a supported compression format from the given bytes.
+    */
+    #[allow(clippy::missing_panics_doc)]
+    pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option<Self> {
+        match bytes.as_ref() {
+            // https://github.com/PSeitz/lz4_flex/blob/main/src/frame/header.rs#L28
+            b if b.len() >= 4
+                && matches!(
+                    u32::from_le_bytes(b[0..4].try_into().unwrap()),
+                    0x184D2204 | 0x184C2102
+                ) =>
+            {
+                Some(Self::LZ4)
+            }
+            // https://github.com/dropbox/rust-brotli/blob/master/src/enc/brotli_bit_stream.rs#L2805
+            b if b.len() >= 4
+                && matches!(
+                    b[0..3],
+                    [0xE1, 0x97, 0x81] | [0xE1, 0x97, 0x82] | [0xE1, 0x97, 0x80]
+                ) =>
+            {
+                Some(Self::Brotli)
+            }
+            // https://github.com/rust-lang/flate2-rs/blob/main/src/gz/mod.rs#L135
+            b if b.len() >= 3 && matches!(b[0..3], [0x1F, 0x8B, 0x08]) => Some(Self::GZip),
+            // https://stackoverflow.com/a/43170354
+            b if b.len() >= 2
+                && matches!(
+                    b[0..2],
+                    [0x78, 0x01] | [0x78, 0x5E] | [0x78, 0x9C] | [0x78, 0xDA]
+                ) =>
+            {
+                Some(Self::ZLib)
+            }
+            _ => None,
+        }
+    }
+
+    /**
+        Detects a supported compression format from the given header string.
+
+        The given header script should be a valid `Content-Encoding` header value.
+    */
+    pub fn detect_from_header_str(header: impl AsRef<str>) -> Option<Self> {
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#directives
+        match header.as_ref().to_ascii_lowercase().trim() {
+            "br" | "brotli" => Some(Self::Brotli),
+            "deflate" => Some(Self::ZLib),
+            "gz" | "gzip" => Some(Self::GZip),
+            _ => None,
+        }
+    }
+}
+
+impl<'lua> FromLua<'lua> for CompressDecompressFormat {
+    fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult<Self> {
+        if let LuaValue::String(s) = &value {
+            match s.to_string_lossy().to_ascii_lowercase().trim() {
+                "brotli" => Ok(Self::Brotli),
+                "gzip" => Ok(Self::GZip),
+                "lz4" => Ok(Self::LZ4),
+                "zlib" => Ok(Self::ZLib),
+                kind => Err(LuaError::FromLuaConversionError {
+                    from: value.type_name(),
+                    to: "CompressDecompressFormat",
+                    message: Some(format!(
+                        "Invalid format '{kind}', valid formats are:  brotli, gzip, lz4, zlib"
+                    )),
+                }),
+            }
+        } else {
+            Err(LuaError::FromLuaConversionError {
+                from: value.type_name(),
+                to: "CompressDecompressFormat",
+                message: None,
+            })
+        }
+    }
+}
+
+/**
+    Compresses the given bytes using the specified format.
+
+    # Errors
+
+    Errors when the compression fails.
+*/
+pub async fn compress<'lua>(
+    source: impl AsRef<[u8]>,
+    format: CompressDecompressFormat,
+) -> LuaResult<Vec<u8>> {
+    if let CompressDecompressFormat::LZ4 = format {
+        let source = source.as_ref().to_vec();
+        return spawn_blocking(move || compress_prepend_size(&source))
+            .await
+            .into_lua_err();
+    }
+
+    let mut bytes = Vec::new();
+    let reader = BufReader::new(source.as_ref());
+
+    match format {
+        CompressDecompressFormat::Brotli => {
+            let mut encoder = BrotliEncoder::with_quality(reader, CompressionQuality);
+            copy(&mut encoder, &mut bytes).await?;
+        }
+        CompressDecompressFormat::GZip => {
+            let mut encoder = GzipEncoder::with_quality(reader, CompressionQuality);
+            copy(&mut encoder, &mut bytes).await?;
+        }
+        CompressDecompressFormat::ZLib => {
+            let mut encoder = ZlibEncoder::with_quality(reader, CompressionQuality);
+            copy(&mut encoder, &mut bytes).await?;
+        }
+        CompressDecompressFormat::LZ4 => unreachable!(),
+    }
+
+    Ok(bytes)
+}
+
+/**
+    Decompresses the given bytes using the specified format.
+
+    # Errors
+
+    Errors when the decompression fails.
+*/
+pub async fn decompress<'lua>(
+    source: impl AsRef<[u8]>,
+    format: CompressDecompressFormat,
+) -> LuaResult<Vec<u8>> {
+    if let CompressDecompressFormat::LZ4 = format {
+        let source = source.as_ref().to_vec();
+        return spawn_blocking(move || decompress_size_prepended(&source))
+            .await
+            .into_lua_err()?
+            .into_lua_err();
+    }
+
+    let mut bytes = Vec::new();
+    let reader = BufReader::new(source.as_ref());
+
+    match format {
+        CompressDecompressFormat::Brotli => {
+            let mut decoder = BrotliDecoder::new(reader);
+            copy(&mut decoder, &mut bytes).await?;
+        }
+        CompressDecompressFormat::GZip => {
+            let mut decoder = GzipDecoder::new(reader);
+            copy(&mut decoder, &mut bytes).await?;
+        }
+        CompressDecompressFormat::ZLib => {
+            let mut decoder = ZlibDecoder::new(reader);
+            copy(&mut decoder, &mut bytes).await?;
+        }
+        CompressDecompressFormat::LZ4 => unreachable!(),
+    }
+
+    Ok(bytes)
+}

crates/lune-std-serde/src/encode_decode.rs

@@ -0,0 +1,158 @@
+use mlua::prelude::*;
+
+use serde_json::Value as JsonValue;
+use serde_yaml::Value as YamlValue;
+use toml::Value as TomlValue;
+
+// NOTE: These are options for going from other format -> lua ("serializing" lua values)
+const LUA_SERIALIZE_OPTIONS: LuaSerializeOptions = LuaSerializeOptions::new()
+    .set_array_metatable(false)
+    .serialize_none_to_null(false)
+    .serialize_unit_to_null(false);
+
+// NOTE: These are options for going from lua -> other format ("deserializing" lua values)
+const LUA_DESERIALIZE_OPTIONS: LuaDeserializeOptions = LuaDeserializeOptions::new()
+    .sort_keys(true)
+    .deny_recursive_tables(false)
+    .deny_unsupported_types(true);
+
+/**
+    An encoding and decoding format supported by Lune.
+
+    Encode / decode in this case is synonymous with serialize / deserialize.
+*/
+#[derive(Debug, Clone, Copy)]
+pub enum EncodeDecodeFormat {
+    Json,
+    Yaml,
+    Toml,
+}
+
+impl<'lua> FromLua<'lua> for EncodeDecodeFormat {
+    fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult<Self> {
+        if let LuaValue::String(s) = &value {
+            match s.to_string_lossy().to_ascii_lowercase().trim() {
+                "json" => Ok(Self::Json),
+                "yaml" => Ok(Self::Yaml),
+                "toml" => Ok(Self::Toml),
+                kind => Err(LuaError::FromLuaConversionError {
+                    from: value.type_name(),
+                    to: "EncodeDecodeFormat",
+                    message: Some(format!(
+                        "Invalid format '{kind}', valid formats are:  json, yaml, toml"
+                    )),
+                }),
+            }
+        } else {
+            Err(LuaError::FromLuaConversionError {
+                from: value.type_name(),
+                to: "EncodeDecodeFormat",
+                message: None,
+            })
+        }
+    }
+}
+
+/**
+    Configuration for encoding and decoding values.
+
+    Encoding / decoding in this case is synonymous with serialize / deserialize.
+*/
+#[derive(Debug, Clone, Copy)]
+pub struct EncodeDecodeConfig {
+    pub format: EncodeDecodeFormat,
+    pub pretty: bool,
+}
+
+impl From<EncodeDecodeFormat> for EncodeDecodeConfig {
+    fn from(format: EncodeDecodeFormat) -> Self {
+        Self {
+            format,
+            pretty: false,
+        }
+    }
+}
+
+impl From<(EncodeDecodeFormat, bool)> for EncodeDecodeConfig {
+    fn from(value: (EncodeDecodeFormat, bool)) -> Self {
+        Self {
+            format: value.0,
+            pretty: value.1,
+        }
+    }
+}
+
+/**
+    Encodes / serializes the given value into a string, using the specified configuration.
+
+    # Errors
+
+    Errors when the encoding fails.
+*/
+pub fn encode<'lua>(
+    value: LuaValue<'lua>,
+    lua: &'lua Lua,
+    config: EncodeDecodeConfig,
+) -> LuaResult<LuaString<'lua>> {
+    let bytes = match config.format {
+        EncodeDecodeFormat::Json => {
+            let serialized: JsonValue = lua.from_value_with(value, LUA_DESERIALIZE_OPTIONS)?;
+            if config.pretty {
+                serde_json::to_vec_pretty(&serialized).into_lua_err()?
+            } else {
+                serde_json::to_vec(&serialized).into_lua_err()?
+            }
+        }
+        EncodeDecodeFormat::Yaml => {
+            let serialized: YamlValue = lua.from_value_with(value, LUA_DESERIALIZE_OPTIONS)?;
+            let mut writer = Vec::with_capacity(128);
+            serde_yaml::to_writer(&mut writer, &serialized).into_lua_err()?;
+            writer
+        }
+        EncodeDecodeFormat::Toml => {
+            let serialized: TomlValue = lua.from_value_with(value, LUA_DESERIALIZE_OPTIONS)?;
+            let s = if config.pretty {
+                toml::to_string_pretty(&serialized).into_lua_err()?
+            } else {
+                toml::to_string(&serialized).into_lua_err()?
+            };
+            s.as_bytes().to_vec()
+        }
+    };
+    lua.create_string(bytes)
+}
+
+/**
+    Decodes / deserializes the given string into a value, using the specified configuration.
+
+    # Errors
+
+    Errors when the decoding fails.
+*/
+pub fn decode(
+    bytes: impl AsRef<[u8]>,
+    lua: &Lua,
+    config: EncodeDecodeConfig,
+) -> LuaResult<LuaValue> {
+    let bytes = bytes.as_ref();
+    match config.format {
+        EncodeDecodeFormat::Json => {
+            let value: JsonValue = serde_json::from_slice(bytes).into_lua_err()?;
+            lua.to_value_with(&value, LUA_SERIALIZE_OPTIONS)
+        }
+        EncodeDecodeFormat::Yaml => {
+            let value: YamlValue = serde_yaml::from_slice(bytes).into_lua_err()?;
+            lua.to_value_with(&value, LUA_SERIALIZE_OPTIONS)
+        }
+        EncodeDecodeFormat::Toml => {
+            if let Ok(s) = String::from_utf8(bytes.to_vec()) {
+                let value: TomlValue = toml::from_str(&s).into_lua_err()?;
+                lua.to_value_with(&value, LUA_SERIALIZE_OPTIONS)
+            } else {
+                Err(LuaError::RuntimeError(
+                    "TOML must be valid utf-8".to_string(),
+                ))
+            }
+        }
+    }
+}

crates/lune-std-serde/src/lib.rs

@@ -1,9 +1,16 @@
 #![allow(clippy::cargo_common_metadata)]
 
+use bstr::BString;
 use mlua::prelude::*;
 
 use lune_utils::TableBuilder;
 
+mod compress_decompress;
+mod encode_decode;
+
+pub use self::compress_decompress::{compress, decompress, CompressDecompressFormat};
+pub use self::encode_decode::{decode, encode, EncodeDecodeConfig, EncodeDecodeFormat};
+
 /**
     Creates the `serde` standard library module.
 
@@ -12,5 +19,39 @@ use lune_utils::TableBuilder;
     Errors when out of memory.
 */
 pub fn module(lua: &Lua) -> LuaResult<LuaTable> {
-    TableBuilder::new(lua)?.build_readonly()
+    TableBuilder::new(lua)?
+        .with_function("encode", serde_encode)?
+        .with_function("decode", serde_decode)?
+        .with_async_function("compress", serde_compress)?
+        .with_async_function("decompress", serde_decompress)?
+        .build_readonly()
+}
+
+fn serde_encode<'lua>(
+    lua: &'lua Lua,
+    (format, value, pretty): (EncodeDecodeFormat, LuaValue<'lua>, Option<bool>),
+) -> LuaResult<LuaString<'lua>> {
+    let config = EncodeDecodeConfig::from((format, pretty.unwrap_or_default()));
+    encode(value, lua, config)
+}
+
+fn serde_decode(lua: &Lua, (format, bs): (EncodeDecodeFormat, BString)) -> LuaResult<LuaValue> {
+    let config = EncodeDecodeConfig::from(format);
+    decode(bs, lua, config)
+}
+
+async fn serde_compress(
+    lua: &Lua,
+    (format, bs): (CompressDecompressFormat, BString),
+) -> LuaResult<LuaString> {
+    let bytes = compress(bs, format).await?;
+    lua.create_string(bytes)
+}
+
+async fn serde_decompress(
+    lua: &Lua,
+    (format, bs): (CompressDecompressFormat, BString),
+) -> LuaResult<LuaString> {
+    let bytes = decompress(bs, format).await?;
+    lua.create_string(bytes)
 }