diff --git a/Cargo.lock b/Cargo.lock index 5f6114a..afb1510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" +checksum = "9c90a406b4495d129f00461241616194cb8a032c8d1c53c657f0961d5f8e0498" dependencies = [ "brotli", "flate2", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "brotli" -version = "4.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -324,9 +324,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "3.0.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" +checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", diff --git a/scripts/generate_compression_test_files.luau b/scripts/generate_compression_test_files.luau index 42af77d..2d0e6b0 100644 --- a/scripts/generate_compression_test_files.luau +++ b/scripts/generate_compression_test_files.luau @@ -57,6 +57,21 @@ local function hexDiff(a: string, b: string): string return table.concat(diff) end +-- Make some processing functions for manipulating output of certain commands + +local function processNoop(output: string): string + return output +end + +local function processGzipSetOsUnknown(output: string): string + -- This will set the os bits to be "unknown" so that the + -- output is deterministic and consistent with serde lib + -- https://www.rfc-editor.org/rfc/rfc1952#section-2.3.1 + local buf = buffer.fromstring(output) + buffer.writeu8(buf, 9, 0xFF) + return buffer.tostring(buf) +end + -- Make sure we have all of the different compression tools installed, -- note that on macos we do not use the system-installed compression -- tools, instead preferring to use homebrew-installed (gnu) ones @@ -118,6 +133,7 @@ local OUTPUT_FILES = { format = "brotli" :: serde.CompressDecompressFormat, args = { "--best", "-w", "22", TEMP_FILE }, output = TEMP_FILE .. ".br", + process = processNoop, final = INPUT_FILE .. ".br", }, { @@ -125,13 +141,15 @@ local OUTPUT_FILES = { format = "gzip" :: serde.CompressDecompressFormat, args = { "--best", "--no-name", "--synchronous", TEMP_FILE }, output = TEMP_FILE .. ".gz", + process = processGzipSetOsUnknown, final = INPUT_FILE .. ".gz", }, { command = BIN_LZ4, format = "lz4" :: serde.CompressDecompressFormat, - args = { "--best", TEMP_FILE }, + args = { "--best", TEMP_FILE, TEMP_FILE .. ".lz4" }, output = TEMP_FILE .. ".lz4", + process = processNoop, final = INPUT_FILE .. ".lz4", }, -- { @@ -139,6 +157,7 @@ local OUTPUT_FILES = { -- format = "zlib" :: serde.CompressDecompressFormat, -- args = { "-c", INPUT_FILE }, -- output = TEMP_FILE .. ".z", + -- process = processNoop, -- final = INPUT_FILE .. ".z", -- }, } @@ -158,6 +177,7 @@ for _, spec in OUTPUT_FILES do local compressedContents pcall(function() compressedContents = fs.readFile(spec.output) + compressedContents = spec.process(compressedContents) fs.removeFile(spec.output) end) if not compressedContents then diff --git a/tests/serde/test-files/loremipsum.txt.gz b/tests/serde/test-files/loremipsum.txt.gz index 5d1cbf3..206a5e8 100644 Binary files a/tests/serde/test-files/loremipsum.txt.gz and b/tests/serde/test-files/loremipsum.txt.gz differ