mirror of
https://github.com/CompeyDev/lune-packaging.git
synced 2025-01-09 12:19:09 +00:00
Add lz4 compression format
This commit is contained in:
Filip Tibell
parent
8619de8ba5
commit
dc0d693d1a
5 changed files with 109 additions and 18 deletions
|
|
@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
### Added
|
||||
|
||||
- Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, or `zlib`.
|
||||
- Added `serde.compress` and `serde.decompress` for compressing and decompressing strings using one of several compression formats: `brotli`, `gzip`, `lz4`, or `zlib`.
|
||||
|
||||
Example usage:
|
||||
|
||||
|
|
|
|||
26
Cargo.lock
generated
26
Cargo.lock
generated
|
|
@ -1151,6 +1151,7 @@ dependencies = [
|
|||
"hyper",
|
||||
"hyper-tungstenite",
|
||||
"lune-roblox",
|
||||
"lz4_flex",
|
||||
"mlua",
|
||||
"once_cell",
|
||||
"os_str_bytes",
|
||||
|
|
@ -1225,6 +1226,15 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b8c72594ac26bfd34f2d99dfced2edfaddfe8a476e3ff2ca0eb293d925c4f83"
|
||||
dependencies = [
|
||||
"twox-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
|
|
@ -1966,6 +1976,12 @@ dependencies = [
|
|||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "stdweb"
|
||||
version = "0.4.20"
|
||||
|
|
@ -2328,6 +2344,16 @@ dependencies = [
|
|||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "twox-hash"
|
||||
version = "1.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.16.0"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
export type EncodeDecodeFormat = "json" | "yaml" | "toml"
|
||||
|
||||
export type CompressDecompressFormat = "brotli" | "gzip" | "zlib"
|
||||
export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib"
|
||||
|
||||
--[=[
|
||||
@class Serde
|
||||
|
|
@ -34,6 +34,14 @@ return {
|
|||
|
||||
Encodes the given value using the given format.
|
||||
|
||||
Currently supported formats:
|
||||
|
||||
| Name | Learn More |
|
||||
|:-------|:---------------------|
|
||||
| `json` | https://www.json.org |
|
||||
| `yaml` | https://yaml.org |
|
||||
| `toml` | https://toml.io |
|
||||
|
||||
@param format The format to use
|
||||
@param value The value to encode
|
||||
@param pretty If the encoded string should be human-readable, including things such as newlines and spaces. Only supported for json and toml formats, and defaults to false
|
||||
|
|
@ -48,6 +56,14 @@ return {
|
|||
|
||||
Decodes the given string using the given format into a lua value.
|
||||
|
||||
Currently supported formats:
|
||||
|
||||
| Name | Learn More |
|
||||
|:-------|:---------------------|
|
||||
| `json` | https://www.json.org |
|
||||
| `yaml` | https://yaml.org |
|
||||
| `toml` | https://toml.io |
|
||||
|
||||
@param format The format to use
|
||||
@param encoded The string to decode
|
||||
@return The decoded lua value
|
||||
|
|
@ -61,6 +77,15 @@ return {
|
|||
|
||||
Compresses the given string using the given format.
|
||||
|
||||
Currently supported formats:
|
||||
|
||||
| Name | Learn More |
|
||||
|:---------|:----------------------------------|
|
||||
| `brotli` | https://github.com/google/brotli |
|
||||
| `gzip` | https://www.gnu.org/software/gzip |
|
||||
| `lz4` | https://github.com/lz4/lz4 |
|
||||
| `zlib` | https://www.zlib.net |
|
||||
|
||||
@param format The format to use
|
||||
@param encoded The string to compress
|
||||
@return The compressed string
|
||||
|
|
@ -74,6 +99,15 @@ return {
|
|||
|
||||
Decompresses the given string using the given format.
|
||||
|
||||
Currently supported formats:
|
||||
|
||||
| Name | Learn More |
|
||||
|:---------|:----------------------------------|
|
||||
| `brotli` | https://github.com/google/brotli |
|
||||
| `gzip` | https://www.gnu.org/software/gzip |
|
||||
| `lz4` | https://github.com/lz4/lz4 |
|
||||
| `zlib` | https://www.zlib.net |
|
||||
|
||||
@param format The format to use
|
||||
@param encoded The string to decompress
|
||||
@return The decompressed string
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ async-trait = "0.1"
|
|||
blocking = "1.3"
|
||||
dialoguer = "0.10"
|
||||
dunce = "1.0"
|
||||
lz4_flex = "0.10"
|
||||
pin-project = "1.0"
|
||||
os_str_bytes = "6.4"
|
||||
urlencoding = "2.1.2"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
use async_compression::tokio::write::{
|
||||
BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder,
|
||||
};
|
||||
use blocking::unblock;
|
||||
use lz4_flex::{compress_prepend_size, decompress_size_prepended};
|
||||
use mlua::prelude::*;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
|
|
@ -8,27 +10,44 @@ use tokio::io::AsyncWriteExt;
|
|||
pub enum CompressDecompressFormat {
|
||||
Brotli,
|
||||
GZip,
|
||||
LZ4,
|
||||
ZLib,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl CompressDecompressFormat {
|
||||
pub fn detect_from_bytes(bytes: impl AsRef<[u8]>) -> Option<Self> {
|
||||
let bytes = bytes.as_ref();
|
||||
if bytes[0..4] == [0x0B, 0x24, 0x72, 0x68] {
|
||||
Some(Self::Brotli)
|
||||
} else if bytes[0..3] == [0x1F, 0x8B, 0x08] {
|
||||
Some(Self::GZip)
|
||||
}
|
||||
// https://stackoverflow.com/a/54915442
|
||||
else if (bytes[0..2] == [0x78, 0x01])
|
||||
|| (bytes[0..2] == [0x78, 0x5E])
|
||||
|| (bytes[0..2] == [0x78, 0x9C])
|
||||
|| (bytes[0..2] == [0x78, 0xDA])
|
||||
{
|
||||
Some(Self::ZLib)
|
||||
} else {
|
||||
None
|
||||
match bytes.as_ref() {
|
||||
// https://github.com/PSeitz/lz4_flex/blob/main/src/frame/header.rs#L28
|
||||
b if b.len() >= 4
|
||||
&& matches!(
|
||||
u32::from_le_bytes(b[0..4].try_into().unwrap()),
|
||||
0x184D2204 | 0x184C2102
|
||||
) =>
|
||||
{
|
||||
Some(Self::LZ4)
|
||||
}
|
||||
// https://github.com/dropbox/rust-brotli/blob/master/src/enc/brotli_bit_stream.rs#L2805
|
||||
b if b.len() >= 4
|
||||
&& matches!(
|
||||
b[0..3],
|
||||
[0xE1, 0x97, 0x81] | [0xE1, 0x97, 0x82] | [0xE1, 0x97, 0x80]
|
||||
) =>
|
||||
{
|
||||
Some(Self::Brotli)
|
||||
}
|
||||
// https://github.com/rust-lang/flate2-rs/blob/main/src/gz/mod.rs#L135
|
||||
b if b.len() >= 3 && matches!(b[0..3], [0x1F, 0x8B, 0x08]) => Some(Self::GZip),
|
||||
// https://stackoverflow.com/a/43170354
|
||||
b if b.len() >= 2
|
||||
&& matches!(
|
||||
b[0..2],
|
||||
[0x78, 0x01] | [0x78, 0x5E] | [0x78, 0x9C] | [0x78, 0xDA]
|
||||
) =>
|
||||
{
|
||||
Some(Self::ZLib)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -49,12 +68,13 @@ impl<'lua> FromLua<'lua> for CompressDecompressFormat {
|
|||
match s.to_string_lossy().to_ascii_lowercase().trim() {
|
||||
"brotli" => Ok(Self::Brotli),
|
||||
"gzip" => Ok(Self::GZip),
|
||||
"lz4" => Ok(Self::LZ4),
|
||||
"zlib" => Ok(Self::ZLib),
|
||||
kind => Err(LuaError::FromLuaConversionError {
|
||||
from: value.type_name(),
|
||||
to: "CompressDecompressFormat",
|
||||
message: Some(format!(
|
||||
"Invalid format '{kind}', valid formats are: brotli, gzip, zlib"
|
||||
"Invalid format '{kind}', valid formats are: brotli, gzip, lz4, zlib"
|
||||
)),
|
||||
}),
|
||||
}
|
||||
|
|
@ -89,6 +109,10 @@ pub async fn compress<'lua>(
|
|||
.write_all(source.as_ref())
|
||||
.await?
|
||||
}
|
||||
CompressDecompressFormat::LZ4 => {
|
||||
let source = source.as_ref().to_vec();
|
||||
bytes = unblock(move || compress_prepend_size(&source)).await;
|
||||
}
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
|
@ -114,6 +138,12 @@ pub async fn decompress<'lua>(
|
|||
.write_all(source.as_ref())
|
||||
.await?
|
||||
}
|
||||
CompressDecompressFormat::LZ4 => {
|
||||
let source = source.as_ref().to_vec();
|
||||
bytes = unblock(move || decompress_size_prepended(&source))
|
||||
.await
|
||||
.map_err(LuaError::external)?;
|
||||
}
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue