mirror of
https://github.com/lune-org/lune.git
synced 2024-12-13 05:20:37 +00:00
304 lines
9 KiB
Text
304 lines
9 KiB
Text
|
local fs = require("@lune/fs")
|
||
|
local process = require("@lune/process")
|
||
|
local serde = require("@lune/serde")
|
||
|
local stdio = require("@lune/stdio")
|
||
|
|
||
|
local TEST_FILES_DIR = process.cwd .. "tests/serde/test-files"
|
||
|
local INPUT_FILE = TEST_FILES_DIR .. "/loremipsum.txt"
|
||
|
local TEMP_FILE = TEST_FILES_DIR .. "/loremipsum.temp"
|
||
|
|
||
|
local INPUT_FILE_CONTENTS = fs.readFile(INPUT_FILE)
|
||
|
|
||
|
-- Make some utility functions for viewing unexpected differences in files easier
|
||
|
|
||
|
local function stringAsHex(str: string): string
|
||
|
local hex = {}
|
||
|
for i = 1, #str do
|
||
|
table.insert(hex, string.format("%02x", string.byte(str, i)))
|
||
|
end
|
||
|
return table.concat(hex)
|
||
|
end
|
||
|
|
||
|
local function hexDiff(a: string, b: string): string
|
||
|
local diff = {}
|
||
|
for i = 1, math.max(#a, #b) do
|
||
|
local aByte = if #a >= i then string.byte(a, i) else nil
|
||
|
local bByte = if #b >= i then string.byte(b, i) else nil
|
||
|
if aByte == nil then
|
||
|
table.insert(
|
||
|
diff,
|
||
|
string.format(
|
||
|
"%s%02x%s",
|
||
|
stdio.color("green"),
|
||
|
assert(bByte, "unreachable"),
|
||
|
stdio.color("reset")
|
||
|
)
|
||
|
)
|
||
|
elseif bByte == nil then
|
||
|
table.insert(
|
||
|
diff,
|
||
|
string.format(
|
||
|
"%s%02x%s",
|
||
|
stdio.color("red"),
|
||
|
assert(aByte, "unreachable"),
|
||
|
stdio.color("reset")
|
||
|
)
|
||
|
)
|
||
|
else
|
||
|
if aByte == bByte then
|
||
|
table.insert(diff, string.format("%02x", aByte))
|
||
|
else
|
||
|
table.insert(
|
||
|
diff,
|
||
|
string.format(
|
||
|
"%s%02x%s",
|
||
|
stdio.color("yellow"),
|
||
|
assert(bByte, "unreachable"),
|
||
|
stdio.color("reset")
|
||
|
)
|
||
|
)
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
return table.concat(diff)
|
||
|
end
|
||
|
|
||
|
local function stripCwdIfPresent(path: string): string
|
||
|
if string.sub(path, 1, #process.cwd) == process.cwd then
|
||
|
return string.sub(path, #process.cwd + 1)
|
||
|
else
|
||
|
return path
|
||
|
end
|
||
|
end
|
||
|
|
||
|
-- Make some processing functions for manipulating output of certain commands
|
||
|
|
||
|
local function processNoop(output: string): string
|
||
|
return output
|
||
|
end
|
||
|
|
||
|
local function processGzipSetOsUnknown(output: string): string
|
||
|
-- This will set the os bits to be "unknown" so that the
|
||
|
-- output is deterministic and consistent with serde lib
|
||
|
-- https://www.rfc-editor.org/rfc/rfc1952#section-2.3.1
|
||
|
local buf = buffer.fromstring(output)
|
||
|
buffer.writeu8(buf, 9, 0xFF)
|
||
|
return buffer.tostring(buf)
|
||
|
end
|
||
|
|
||
|
local function processLz4PrependSize(output: string): string
|
||
|
-- Lune supports only lz4 with the decompressed size
|
||
|
-- prepended to it, but the lz4 command line tool
|
||
|
-- doesn't add this automatically, so we have to
|
||
|
-- TODO: Remove this in the future when no longer needed
|
||
|
local buf = buffer.create(4 + #output)
|
||
|
buffer.writeu32(buf, 0, #INPUT_FILE_CONTENTS)
|
||
|
buffer.writestring(buf, 4, output)
|
||
|
return buffer.tostring(buf)
|
||
|
end
|
||
|
|
||
|
-- Make sure we have all of the different compression tools installed,
|
||
|
-- note that on macos we do not use the system-installed compression
|
||
|
-- tools, instead preferring to use homebrew-installed (gnu) ones
|
||
|
|
||
|
local BIN_BROTLI = if process.os == "macos" then "/opt/homebrew/bin/brotli" else "brotli"
|
||
|
local BIN_GZIP = if process.os == "macos" then "/opt/homebrew/bin/gzip" else "gzip"
|
||
|
local BIN_LZ4 = if process.os == "macos" then "/opt/homebrew/bin/lz4" else "lz4"
|
||
|
local BIN_ZLIB = if process.os == "macos" then "/opt/homebrew/bin/pigz" else "pigz"
|
||
|
|
||
|
local function checkInstalled(program: string, args: { string }?)
|
||
|
print("Checking if", program, "is installed")
|
||
|
local result = process.spawn(program, args)
|
||
|
if not result.ok then
|
||
|
stdio.ewrite(string.format("Program '%s' is not installed\n", program))
|
||
|
process.exit(1)
|
||
|
end
|
||
|
end
|
||
|
|
||
|
checkInstalled(BIN_BROTLI, { "--version" })
|
||
|
checkInstalled(BIN_GZIP, { "--version" })
|
||
|
checkInstalled(BIN_LZ4, { "--version" })
|
||
|
checkInstalled(BIN_ZLIB, { "--version" })
|
||
|
|
||
|
-- Run them to generate files
|
||
|
|
||
|
local function run(program: string, args: { string }): string
|
||
|
local result = process.spawn(program, args)
|
||
|
if not result.ok then
|
||
|
stdio.ewrite(string.format("Command '%s' failed\n", program))
|
||
|
if #result.stdout > 0 then
|
||
|
stdio.ewrite("stdout:\n")
|
||
|
stdio.ewrite(result.stdout)
|
||
|
stdio.ewrite("\n")
|
||
|
end
|
||
|
if #result.stderr > 0 then
|
||
|
stdio.ewrite("stderr:\n")
|
||
|
stdio.ewrite(result.stderr)
|
||
|
stdio.ewrite("\n")
|
||
|
end
|
||
|
process.exit(1)
|
||
|
else
|
||
|
if #result.stdout > 0 then
|
||
|
stdio.ewrite("stdout:\n")
|
||
|
stdio.ewrite(result.stdout)
|
||
|
stdio.ewrite("\n")
|
||
|
end
|
||
|
end
|
||
|
return result.stdout
|
||
|
end
|
||
|
|
||
|
local OUTPUT_FILES = {
|
||
|
{
|
||
|
command = BIN_BROTLI,
|
||
|
format = "brotli" :: serde.CompressDecompressFormat,
|
||
|
args = { "--best", "-w", "22", TEMP_FILE },
|
||
|
output = TEMP_FILE .. ".br",
|
||
|
process = processNoop,
|
||
|
final = INPUT_FILE .. ".br",
|
||
|
},
|
||
|
{
|
||
|
command = BIN_GZIP,
|
||
|
format = "gzip" :: serde.CompressDecompressFormat,
|
||
|
args = { "--best", "--no-name", "--synchronous", TEMP_FILE },
|
||
|
output = TEMP_FILE .. ".gz",
|
||
|
process = processGzipSetOsUnknown,
|
||
|
final = INPUT_FILE .. ".gz",
|
||
|
},
|
||
|
{
|
||
|
command = BIN_LZ4,
|
||
|
format = "lz4" :: serde.CompressDecompressFormat,
|
||
|
args = { "--best", TEMP_FILE, TEMP_FILE .. ".lz4" },
|
||
|
output = TEMP_FILE .. ".lz4",
|
||
|
process = processLz4PrependSize,
|
||
|
final = INPUT_FILE .. ".lz4",
|
||
|
},
|
||
|
{
|
||
|
command = BIN_ZLIB,
|
||
|
format = "zlib" :: serde.CompressDecompressFormat,
|
||
|
args = { "--best", "--zlib", TEMP_FILE },
|
||
|
output = TEMP_FILE .. ".zz",
|
||
|
process = processNoop,
|
||
|
final = INPUT_FILE .. ".z",
|
||
|
},
|
||
|
}
|
||
|
|
||
|
for _, spec in OUTPUT_FILES do
|
||
|
-- Write the temp file for the compression tool to read and use, then
|
||
|
-- remove it, some tools may remove it on their own, so we ignore errors
|
||
|
fs.writeFile(TEMP_FILE, INPUT_FILE_CONTENTS)
|
||
|
local argsToDisplay = {}
|
||
|
for _, arg in spec.args do
|
||
|
table.insert(argsToDisplay, stripCwdIfPresent(arg))
|
||
|
end
|
||
|
print(
|
||
|
"\nRunning compression\n Cmd: ",
|
||
|
spec.command,
|
||
|
"\n Args: ",
|
||
|
stdio.format(table.unpack(argsToDisplay))
|
||
|
)
|
||
|
local output = run(spec.command, spec.args)
|
||
|
if #output > 0 then
|
||
|
print("Output:", output)
|
||
|
end
|
||
|
pcall(fs.removeFile, TEMP_FILE)
|
||
|
|
||
|
-- Read the compressed output file that is now supposed to exist
|
||
|
local compressedContents
|
||
|
pcall(function()
|
||
|
compressedContents = fs.readFile(spec.output)
|
||
|
compressedContents = spec.process(compressedContents)
|
||
|
fs.removeFile(spec.output)
|
||
|
end)
|
||
|
if not compressedContents then
|
||
|
error(
|
||
|
string.format(
|
||
|
"Nothing was written to output file while running %s:\n%s",
|
||
|
spec.command,
|
||
|
spec.output
|
||
|
)
|
||
|
)
|
||
|
end
|
||
|
|
||
|
-- If the newly compressed contents do not match the existing contents,
|
||
|
-- warn the user about this and ask if they want to overwrite the file
|
||
|
local existingContents = fs.readFile(spec.final)
|
||
|
if compressedContents ~= existingContents then
|
||
|
stdio.ewrite("\nCompressed file does not match existing contents!")
|
||
|
stdio.ewrite("\n\nExisting:\n")
|
||
|
stdio.ewrite(stringAsHex(existingContents))
|
||
|
stdio.ewrite("\n\nCompressed:\n")
|
||
|
stdio.ewrite(hexDiff(existingContents, compressedContents))
|
||
|
stdio.ewrite("\n\n")
|
||
|
local confirm = stdio.prompt("confirm", "Do you want to continue?")
|
||
|
if confirm == true then
|
||
|
print("Overwriting file!")
|
||
|
else
|
||
|
stdio.ewrite("\n\nAborting...\n")
|
||
|
process.exit(1)
|
||
|
return
|
||
|
end
|
||
|
end
|
||
|
|
||
|
-- Check if the compressed contents can be decompressed using serde
|
||
|
local decompressSuccess, decompressedContents =
|
||
|
pcall(serde.decompress, spec.format, compressedContents)
|
||
|
if not decompressSuccess then
|
||
|
stdio.ewrite("\nCompressed contents could not be decompressed using serde!")
|
||
|
stdio.ewrite("\n\nCompressed:\n")
|
||
|
stdio.ewrite(stringAsHex(compressedContents))
|
||
|
stdio.ewrite("\n\nError:\n")
|
||
|
stdio.ewrite(tostring(decompressedContents))
|
||
|
stdio.ewrite("\n\n")
|
||
|
local confirm = stdio.prompt("confirm", "Do you want to continue?")
|
||
|
if confirm == true then
|
||
|
print("Ignoring decompression error!")
|
||
|
else
|
||
|
stdio.ewrite("\n\nAborting...\n")
|
||
|
process.exit(1)
|
||
|
return
|
||
|
end
|
||
|
end
|
||
|
if decompressedContents ~= INPUT_FILE_CONTENTS then
|
||
|
stdio.ewrite("\nCompressed contents were not decompressable properly using serde!")
|
||
|
stdio.ewrite("\n\nOriginal:\n")
|
||
|
stdio.ewrite(INPUT_FILE_CONTENTS)
|
||
|
stdio.ewrite("\n\nDecompressed:\n")
|
||
|
stdio.ewrite(decompressedContents)
|
||
|
stdio.ewrite("\n\n")
|
||
|
local confirm = stdio.prompt("confirm", "Do you want to continue?")
|
||
|
if confirm == true then
|
||
|
print("Ignoring decompression mismatch!")
|
||
|
else
|
||
|
stdio.ewrite("\n\nAborting...\n")
|
||
|
process.exit(1)
|
||
|
return
|
||
|
end
|
||
|
end
|
||
|
|
||
|
-- Check if the compressed contents match the serde compressed contents,
|
||
|
-- if they don't this will 100% make the tests fail, but maybe we are doing
|
||
|
-- it because we are updating the serde library and need to update test files
|
||
|
local serdeContents = serde.compress(spec.format, INPUT_FILE_CONTENTS)
|
||
|
if compressedContents ~= serdeContents then
|
||
|
stdio.ewrite("\nTemp file does not match contents compressed with serde!")
|
||
|
stdio.ewrite("\nThis will caused the new compressed file to fail tests.")
|
||
|
stdio.ewrite("\n\nSerde:\n")
|
||
|
stdio.ewrite(stringAsHex(serdeContents))
|
||
|
stdio.ewrite("\n\nCompressed:\n")
|
||
|
stdio.ewrite(hexDiff(serdeContents, compressedContents))
|
||
|
stdio.ewrite("\n\n")
|
||
|
local confirm = stdio.prompt("confirm", "Do you want to continue?")
|
||
|
if confirm == true then
|
||
|
print("Writing new file!")
|
||
|
else
|
||
|
stdio.ewrite("\n\nAborting...\n")
|
||
|
process.exit(1)
|
||
|
return
|
||
|
end
|
||
|
end
|
||
|
|
||
|
-- Finally, write the new compressed file
|
||
|
fs.writeFile(spec.final, compressedContents)
|
||
|
print("Wrote new file successfully to", stripCwdIfPresent(spec.final))
|
||
|
end
|