From 0de9083abf32ffdbdf307ae37d9e3408e1e0e74d Mon Sep 17 00:00:00 2001
From: Erica Marigold <hi@devcomp.xyz>
Date: Tue, 28 May 2024 13:24:19 +0530
Subject: [PATCH] feat(serde): zstd compression & decompression
Tests are broken since the generated ZStd file by the CLI does not match the one generated by serde. Not really sure why.
---
Cargo.lock | 283 ++++++++++--------
crates/lune-std-serde/Cargo.toml | 1 +
.../lune-std-serde/src/compress_decompress.rs | 18 ++
scripts/generate_compression_test_files.luau | 48 +--
tests/serde/compression/files.luau | 5 +
tests/serde/compression/roundtrip.luau | 2 +-
tests/serde/test-files/loremipsum.txt.zst | Bin 0 -> 128 bytes
tests/serde/test-files/uncompressed.json | 21 +-
types/serde.luau | 2 +-
9 files changed, 225 insertions(+), 155 deletions(-)
create mode 100644 tests/serde/test-files/loremipsum.txt.zst
diff --git a/Cargo.lock b/Cargo.lock
index cf0e82d..80f1b77 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -118,9 +118,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.83"
+version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3"
+checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
[[package]]
name = "arbitrary"
@@ -151,13 +151,12 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "async-channel"
-version = "2.2.1"
+version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928"
+checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"
dependencies = [
"concurrent-queue",
- "event-listener 5.3.0",
- "event-listener-strategy 0.5.2",
+ "event-listener-strategy",
"futures-core",
"pin-project-lite",
]
@@ -174,13 +173,15 @@ dependencies = [
"memchr",
"pin-project-lite",
"tokio",
+ "zstd",
+ "zstd-safe",
]
[[package]]
name = "async-executor"
-version = "1.11.0"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a"
+checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0"
dependencies = [
"async-task",
"concurrent-queue",
@@ -189,17 +190,6 @@ dependencies = [
"slab",
]
-[[package]]
-name = "async-lock"
-version = "3.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b"
-dependencies = [
- "event-listener 4.0.3",
- "event-listener-strategy 0.4.0",
- "pin-project-lite",
-]
-
[[package]]
name = "async-task"
version = "4.7.1"
@@ -298,12 +288,11 @@ dependencies = [
[[package]]
name = "blocking"
-version = "1.6.0"
+version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "495f7104e962b7356f0aeb34247aca1fe7d2e783b346582db7f2904cb5717e88"
+checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea"
dependencies = [
"async-channel",
- "async-lock",
"async-task",
"futures-io",
"futures-lite",
@@ -323,9 +312,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
-version = "4.0.0"
+version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76"
+checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -389,9 +378,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.0.97"
+version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4"
+checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
dependencies = [
"jobserver",
"libc",
@@ -426,9 +415,9 @@ dependencies = [
[[package]]
name = "chrono_lc"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa1812634894df89eb9d5075eba0b97d42b4affe477bde7d0db3d2cf8454a800"
+checksum = "568e485d6ad62f607516ebb0820ae8cc46361b0870aeb46eb8232440a00b2eb6"
dependencies = [
"chrono",
"lazy_static",
@@ -480,7 +469,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -611,18 +600,18 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32fast"
-version = "1.4.0"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-utils"
-version = "0.8.19"
+version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crypto-common"
@@ -663,7 +652,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -749,7 +738,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -787,11 +776,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "erased-serde"
-version = "0.4.4"
+version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3"
+checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d"
dependencies = [
"serde",
+ "typeid",
]
[[package]]
@@ -832,16 +822,6 @@ dependencies = [
"pin-project-lite",
]
-[[package]]
-name = "event-listener-strategy"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3"
-dependencies = [
- "event-listener 4.0.3",
- "pin-project-lite",
-]
-
[[package]]
name = "event-listener-strategy"
version = "0.5.2"
@@ -937,7 +917,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -1038,15 +1018,15 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.4.4"
+version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069"
+checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
dependencies = [
+ "atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
- "futures-util",
"http 1.1.0",
"indexmap",
"slab",
@@ -1192,7 +1172,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
- "h2 0.4.4",
+ "h2 0.4.5",
"http 1.1.0",
"http-body 1.0.0",
"httparse",
@@ -1235,9 +1215,9 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa"
+checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e"
dependencies = [
"bytes",
"futures-channel",
@@ -1369,9 +1349,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
-version = "0.2.154"
+version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]]
name = "libloading"
@@ -1411,9 +1391,9 @@ checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e"
[[package]]
name = "linux-raw-sys"
-version = "0.4.13"
+version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
+checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
[[package]]
name = "lock_api"
@@ -1425,6 +1405,12 @@ dependencies = [
"scopeguard",
]
+[[package]]
+name = "lockfree-object-pool"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
+
[[package]]
name = "log"
version = "0.4.21"
@@ -1433,9 +1419,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
[[package]]
name = "luau0-src"
-version = "0.8.6+luau622"
+version = "0.9.1+luau625"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07758c1f5908f7f9dd9109efaf8c66907cc38acf312db03287e7ad2a64b5de1c"
+checksum = "39df4ee5bd067cb2363b9f9e5eed0f1cf326fc80508af8ac037eed5b15459c4a"
dependencies = [
"cc",
]
@@ -1696,9 +1682,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "miniz_oxide"
-version = "0.7.2"
+version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
+checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
dependencies = [
"adler",
]
@@ -1716,9 +1702,9 @@ dependencies = [
[[package]]
name = "mlua"
-version = "0.9.7"
+version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d9bed6bce296397a9d6a86f995dd10a547a4e6949825d45225906bdcbfe7367"
+checksum = "e340c022072f3208a4105458286f4985ba5355bfe243c3073afe45cbe9ecf491"
dependencies = [
"bstr",
"erased-serde",
@@ -1751,9 +1737,9 @@ dependencies = [
[[package]]
name = "mlua-sys"
-version = "0.5.2"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d16a9ba1dd2c6ac971b204262d434c24d65067038598f0638b64e5dca28d52b8"
+checksum = "5552e7e4e22ada0463dfdeee6caf6dc057a189fdc83136408a8f950a5e5c5540"
dependencies = [
"cc",
"cfg-if",
@@ -1826,6 +1812,27 @@ dependencies = [
"libc",
]
+[[package]]
+name = "num_enum"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"
+dependencies = [
+ "num_enum_derive",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.66",
+]
+
[[package]]
name = "object"
version = "0.32.2"
@@ -1879,9 +1886,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
[[package]]
name = "parking_lot"
-version = "0.12.2"
+version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
@@ -1951,7 +1958,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -1968,9 +1975,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "piper"
-version = "0.2.1"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4"
+checksum = "464db0c665917b13ebb5d453ccdec4add5658ee1adc7affc7677615356a8afaf"
dependencies = [
"atomic-waker",
"fastrand",
@@ -2009,6 +2016,15 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+[[package]]
+name = "proc-macro-crate"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
+dependencies = [
+ "toml_edit 0.21.1",
+]
+
[[package]]
name = "proc-macro-hack"
version = "0.5.20+deprecated"
@@ -2017,9 +2033,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.82"
+version = "1.0.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b"
+checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6"
dependencies = [
"unicode-ident",
]
@@ -2040,7 +2056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd"
dependencies = [
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -2428,7 +2444,7 @@ dependencies = [
"log",
"ring",
"rustls-pki-types",
- "rustls-webpki 0.102.3",
+ "rustls-webpki 0.102.4",
"subtle",
"zeroize",
]
@@ -2460,9 +2476,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.102.3"
+version = "0.102.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf"
+checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
dependencies = [
"ring",
"rustls-pki-types",
@@ -2551,9 +2567,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
-version = "1.0.201"
+version = "1.0.203"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c"
+checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
dependencies = [
"serde_derive",
]
@@ -2570,13 +2586,13 @@ dependencies = [
[[package]]
name = "serde_derive"
-version = "1.0.201"
+version = "1.0.203"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865"
+checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -2593,9 +2609,9 @@ dependencies = [
[[package]]
name = "serde_spanned"
-version = "0.6.5"
+version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1"
+checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"
dependencies = [
"serde",
]
@@ -2795,9 +2811,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.63"
+version = "2.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704"
+checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
dependencies = [
"proc-macro2",
"quote",
@@ -2845,22 +2861,22 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "1.0.60"
+version = "1.0.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18"
+checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.60"
+version = "1.0.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524"
+checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -2984,7 +3000,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -3039,37 +3055,48 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.8.12"
+version = "0.8.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3"
+checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
- "toml_edit",
+ "toml_edit 0.22.13",
]
[[package]]
name = "toml_datetime"
-version = "0.6.5"
+version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
+checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
-version = "0.22.12"
+version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef"
+checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "winnow 0.5.40",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
- "winnow",
+ "winnow 0.6.8",
]
[[package]]
@@ -3085,7 +3112,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
- "tracing",
]
[[package]]
@@ -3106,7 +3132,6 @@ version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
- "log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -3120,7 +3145,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
]
[[package]]
@@ -3190,10 +3215,10 @@ dependencies = [
]
[[package]]
-name = "typed-arena"
-version = "2.0.2"
+name = "typeid"
+version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
+checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf"
[[package]]
name = "typenum"
@@ -3339,7 +3364,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
"wasm-bindgen-shared",
]
@@ -3373,7 +3398,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.63",
+ "syn 2.0.66",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -3588,6 +3613,15 @@ version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+[[package]]
+name = "winnow"
+version = "0.5.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "winnow"
version = "0.6.8"
@@ -3624,29 +3658,15 @@ checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193"
[[package]]
name = "zeroize"
-version = "1.7.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
-dependencies = [
- "zeroize_derive",
-]
-
-[[package]]
-name = "zeroize_derive"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.63",
-]
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
[[package]]
name = "zip"
-version = "1.2.3"
+version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c700ea425e148de30c29c580c1f9508b93ca57ad31c9f4e96b83c194c37a7a8f"
+checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164"
dependencies = [
"aes",
"arbitrary",
@@ -3660,12 +3680,11 @@ dependencies = [
"hmac",
"indexmap",
"lzma-rs",
+ "num_enum",
"pbkdf2",
- "rand",
"sha1 0.10.6",
"thiserror",
"time 0.3.36",
- "zeroize",
"zopfli",
"zstd",
]
@@ -3681,14 +3700,16 @@ dependencies = [
[[package]]
name = "zopfli"
-version = "0.8.0"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c1f48f3508a3a3f2faee01629564400bc12260f6214a056d06a3aaaa6ef0736"
+checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
dependencies = [
+ "bumpalo",
"crc32fast",
+ "lockfree-object-pool",
"log",
+ "once_cell",
"simd-adler32",
- "typed-arena",
]
[[package]]
diff --git a/crates/lune-std-serde/Cargo.toml b/crates/lune-std-serde/Cargo.toml
index 91786ff..a8110dd 100644
--- a/crates/lune-std-serde/Cargo.toml
+++ b/crates/lune-std-serde/Cargo.toml
@@ -21,6 +21,7 @@ async-compression = { version = "0.4", features = [
"deflate",
"gzip",
"zlib",
+ "zstd",
] }
bstr = "1.9"
lz4 = "1.24"
diff --git a/crates/lune-std-serde/src/compress_decompress.rs b/crates/lune-std-serde/src/compress_decompress.rs
index 3e5aaa0..574e742 100644
--- a/crates/lune-std-serde/src/compress_decompress.rs
+++ b/crates/lune-std-serde/src/compress_decompress.rs
@@ -11,8 +11,10 @@ use tokio::{
use async_compression::{
tokio::bufread::{
BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder,
+ ZstdDecoder, ZstdEncoder,
},
Level::Best as CompressionQuality,
+ Level::Precise as ZstdCompressionQuality,
};
/**
@@ -24,6 +26,7 @@ pub enum CompressDecompressFormat {
GZip,
LZ4,
ZLib,
+ ZStd,
}
#[allow(dead_code)]
@@ -63,6 +66,12 @@ impl CompressDecompressFormat {
{
Some(Self::ZLib)
}
+
+ b if b.len() >= 4
+ && matches!(u32::from_le_bytes(b[..4].try_into().unwrap()), 0xFD2FB528) =>
+ {
+ Some(Self::ZStd)
+ }
_ => None,
}
}
@@ -91,6 +100,7 @@ impl<'lua> FromLua<'lua> for CompressDecompressFormat {
"gzip" => Ok(Self::GZip),
"lz4" => Ok(Self::LZ4),
"zlib" => Ok(Self::ZLib),
+ "zstd" => Ok(Self::ZStd),
kind => Err(LuaError::FromLuaConversionError {
from: value.type_name(),
to: "CompressDecompressFormat",
@@ -144,6 +154,10 @@ pub async fn compress<'lua>(
let mut encoder = ZlibEncoder::with_quality(reader, CompressionQuality);
copy(&mut encoder, &mut bytes).await?;
}
+ CompressDecompressFormat::ZStd => {
+ let mut encoder = ZstdEncoder::with_quality(reader, ZstdCompressionQuality(22));
+ copy(&mut encoder, &mut bytes).await?;
+ }
CompressDecompressFormat::LZ4 => unreachable!(),
}
@@ -185,6 +199,10 @@ pub async fn decompress<'lua>(
let mut decoder = ZlibDecoder::new(reader);
copy(&mut decoder, &mut bytes).await?;
}
+ CompressDecompressFormat::ZStd => {
+ let mut decoder = ZstdDecoder::new(reader);
+ copy(&mut decoder, &mut bytes).await?;
+ }
CompressDecompressFormat::LZ4 => unreachable!(),
}
diff --git a/scripts/generate_compression_test_files.luau b/scripts/generate_compression_test_files.luau
index 954929a..d6b590d 100644
--- a/scripts/generate_compression_test_files.luau
+++ b/scripts/generate_compression_test_files.luau
@@ -105,6 +105,7 @@ local BIN_BROTLI = if process.os == "macos" then "/opt/homebrew/bin/brotli" else
local BIN_GZIP = if process.os == "macos" then "/opt/homebrew/bin/gzip" else "gzip"
local BIN_LZ4 = if process.os == "macos" then "/opt/homebrew/bin/lz4" else "lz4"
local BIN_ZLIB = if process.os == "macos" then "/opt/homebrew/bin/pigz" else "pigz"
+local BIN_ZSTD = if process.os == "macos" then "/opt/homebrew/bin/zstd" else "zstd"
local function checkInstalled(program: string, args: { string }?)
print("Checking if", program, "is installed")
@@ -119,6 +120,7 @@ checkInstalled(BIN_BROTLI, { "--version" })
checkInstalled(BIN_GZIP, { "--version" })
checkInstalled(BIN_LZ4, { "--version" })
checkInstalled(BIN_ZLIB, { "--version" })
+checkInstalled(BIN_ZSTD, { "--version" })
-- Run them to generate files
@@ -180,6 +182,14 @@ local OUTPUT_FILES = {
process = processNoop,
final = INPUT_FILE .. ".z",
},
+ {
+ command = BIN_ZSTD,
+ format = "zstd" :: serde.CompressDecompressFormat,
+ args = { "--compress", "--ultra", "-22", "--format=zstd", TEMP_FILE },
+ output = TEMP_FILE .. ".zst",
+ process = processNoop,
+ final = INPUT_FILE .. ".zst",
+ },
}
for _, spec in OUTPUT_FILES do
@@ -219,23 +229,25 @@ for _, spec in OUTPUT_FILES do
)
end
- -- If the newly compressed contents do not match the existing contents,
- -- warn the user about this and ask if they want to overwrite the file
- local existingContents = fs.readFile(spec.final)
- if compressedContents ~= existingContents then
- stdio.ewrite("\nCompressed file does not match existing contents!")
- stdio.ewrite("\n\nExisting:\n")
- stdio.ewrite(stringAsHex(existingContents))
- stdio.ewrite("\n\nCompressed:\n")
- stdio.ewrite(hexDiff(existingContents, compressedContents))
- stdio.ewrite("\n\n")
- local confirm = stdio.prompt("confirm", "Do you want to continue?")
- if confirm == true then
- print("Overwriting file!")
- else
- stdio.ewrite("\n\nAborting...\n")
- process.exit(1)
- return
+ if fs.isFile(spec.final) then
+ -- If the newly compressed contents do not match the existing contents,
+ -- warn the user about this and ask if they want to overwrite the file
+ local existingContents = fs.readFile(spec.final)
+ if compressedContents ~= existingContents then
+ stdio.ewrite("\nCompressed file does not match existing contents!")
+ stdio.ewrite("\n\nExisting:\n")
+ stdio.ewrite(stringAsHex(existingContents))
+ stdio.ewrite("\n\nCompressed:\n")
+ stdio.ewrite(hexDiff(existingContents, compressedContents))
+ stdio.ewrite("\n\n")
+ local confirm = stdio.prompt("confirm", "Do you want to continue?")
+ if confirm == true then
+ print("Overwriting file!")
+ else
+ stdio.ewrite("\n\nAborting...\n")
+ process.exit(1)
+ return
+ end
end
end
@@ -243,7 +255,7 @@ for _, spec in OUTPUT_FILES do
local decompressSuccess, decompressedContents =
pcall(serde.decompress, spec.format, compressedContents)
if not decompressSuccess then
- stdio.ewrite("\nCompressed contents could not be decompressed using serde!")
+ stdio.ewrite(`\nCompressed contents for format {spec.format} could not be decompressed using serde!`)
stdio.ewrite("\n\nCompressed:\n")
stdio.ewrite(stringAsHex(compressedContents))
stdio.ewrite("\n\nError:\n")
diff --git a/tests/serde/compression/files.luau b/tests/serde/compression/files.luau
index 6f32ed0..7a2a8fb 100644
--- a/tests/serde/compression/files.luau
+++ b/tests/serde/compression/files.luau
@@ -30,6 +30,11 @@ local TESTS: { Test } = {
Source = "tests/serde/test-files/loremipsum.txt",
Target = "tests/serde/test-files/loremipsum.txt.z",
},
+ {
+ Format = "zstd",
+ Source = "tests/serde/test-files/loremipsum.txt",
+ Target = "tests/serde/test-files/loremipsum.txt.zst",
+ },
}
local failed = false
diff --git a/tests/serde/compression/roundtrip.luau b/tests/serde/compression/roundtrip.luau
index ee78167..c68d3e0 100644
--- a/tests/serde/compression/roundtrip.luau
+++ b/tests/serde/compression/roundtrip.luau
@@ -3,7 +3,7 @@ local process = require("@lune/process")
local serde = require("@lune/serde")
local stdio = require("@lune/stdio")
-local FORMATS: { serde.CompressDecompressFormat } = { "brotli", "gzip", "lz4", "zlib" }
+local FORMATS: { serde.CompressDecompressFormat } = { "brotli", "gzip", "lz4", "zlib", "zstd" }
local FILES: { string } = {
"tests/serde/test-files/loremipsum.txt",
"tests/serde/test-files/uncompressed.csv",
diff --git a/tests/serde/test-files/loremipsum.txt.zst b/tests/serde/test-files/loremipsum.txt.zst
new file mode 100644
index 0000000000000000000000000000000000000000..7dd129bae8aba2b949165683c09052e33d91b1eb
GIT binary patch
literal 128
zcmV-`0Du1|wJ-f-hyj%Y05XXf5TNbo&1#mK)q+ESjTTjjBLfI-*zT<K`{p{g(bFZp
z=>sp!vqmkAnzB~62e^)9JnX1PQ_9DFOLc`G&@$G}qa+tmS6($etWHrm?3nQ+w>=Lr
iS$R}G+_mVe+38ur9RULX;w4jO1Ji$gSug{K0=ai&w>xqG
literal 0
HcmV?d00001
diff --git a/tests/serde/test-files/uncompressed.json b/tests/serde/test-files/uncompressed.json
index ef02f7a..c98a6ee 100644
--- a/tests/serde/test-files/uncompressed.json
+++ b/tests/serde/test-files/uncompressed.json
@@ -1,17 +1,30 @@
{
"name": "John",
"age": 30,
- "hobbies": ["reading", "writing", "coding", "👽"],
+ "hobbies": [
+ "reading",
+ "writing",
+ "coding",
+ "👽"
+ ],
"friends": [
{
"name": "Ξθής",
"age": 28,
- "hobbies": ["painting", "hiking", "🦛"]
+ "hobbies": [
+ "painting",
+ "hiking",
+ "🦛"
+ ]
},
{
"name": "Bob",
"age": 35,
- "hobbies": ["fishing", "gardening", "🌿"]
+ "hobbies": [
+ "fishing",
+ "gardening",
+ "🌿"
+ ]
}
]
-}
+}
\ No newline at end of file
diff --git a/types/serde.luau b/types/serde.luau
index c4a21d8..ac4ec49 100644
--- a/types/serde.luau
+++ b/types/serde.luau
@@ -1,6 +1,6 @@
export type EncodeDecodeFormat = "json" | "yaml" | "toml"
-export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib"
+export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib" | "zstd"
--[=[
@class Serde