luau/tests/conformance/utf8.lua
aaron 7105c81579
Sync to upstream/release/602 (#1089)
# What's changed?

* Fixed a bug in type cloning by maintaining persistent types.
* We now parse imprecise integer literals to report the imprecision as a
warning to developers.
* Add a compiler flag to specify the name of the statistics output file.

### New type solver

* Renamed `ConstraintGraphBuilder` to `ConstraintGenerator`
* LValues now take into account the type being assigned during
constraint generation.
* Normalization performance has been improved by 33% by replacing the an
internal usage of `std::unordered_set` with `DenseHashMap`.
* Normalization now has a helper to identify types that are equivalent
to `unknown`, which is being used to fix some bugs in subtyping.
* Uses of the old unifier in the new type solver have been eliminated.
* Improved error explanations for subtyping errors in `TypeChecker2`.

### Native code generation

* Expanded some of the statistics recorded during compilation to include
the number of instructions and blocks.
* Introduce instruction and block count limiters for controlling what
bytecode is translated into native code.
* Implement code generation for byteswap instruction.

### Internal Contributors

Co-authored-by: Aaron Weiss <aaronweiss@roblox.com>
Co-authored-by: Alexander McCord <amccord@roblox.com>
Co-authored-by: Andy Friesen <afriesen@roblox.com>
Co-authored-by: Aviral Goel <agoel@roblox.com>
Co-authored-by: Lily Brown <lbrown@roblox.com>
2023-11-03 16:45:04 -07:00

237 lines
7 KiB
Lua
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

-- This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
-- This file is based on Lua 5.x tests -- https://github.com/lua/lua/tree/master/testes
print "testing UTF-8 library"
local function checkerror (msg, f, ...)
local s, err = pcall(f, ...)
assert(not s and string.find(err, msg))
end
local function len (s)
return #string.gsub(s, "[\x80-\xBF]", "")
end
local justone = "^" .. utf8.charpattern .. "$"
-- 't' is the list of codepoints of 's'
local function checksyntax (s, t)
-- creates a string "return '\u{t[1]}...\u{t[n]}'"
local ts = {"return '"}
for i = 1, #t do ts[i + 1] = string.format("\\u{%x}", t[i]) end
ts[#t + 2] = "'"
ts = table.concat(ts)
-- its execution should result in 's'
assert(assert(loadstring(ts))() == s)
end
assert(not utf8.offset("alo", 5))
assert(not utf8.offset("alo", -4))
-- 'check' makes several tests over the validity of string 's'.
-- 't' is the list of codepoints of 's'.
local function check (s, t)
local l = utf8.len(s, 1, -1)
assert(#t == l and len(s) == l)
assert(utf8.char(table.unpack(t)) == s) -- 't' and 's' are equivalent
assert(utf8.offset(s, 0) == 1)
checksyntax(s, t)
-- creates new table with all codepoints of 's'
local t1 = {utf8.codepoint(s, 1, -1)}
assert(#t == #t1)
for i = 1, #t do assert(t[i] == t1[i]) end -- 't' is equal to 't1'
for i = 1, l do -- for all codepoints
local pi = utf8.offset(s, i) -- position of i-th char
local pi1 = utf8.offset(s, 2, pi) -- position of next char
assert(string.find(string.sub(s, pi, pi1 - 1), justone))
assert(utf8.offset(s, -1, pi1) == pi)
assert(utf8.offset(s, i - l - 1) == pi)
assert(pi1 - pi == #utf8.char(utf8.codepoint(s, pi, pi)))
for j = pi, pi1 - 1 do
assert(utf8.offset(s, 0, j) == pi)
end
for j = pi + 1, pi1 - 1 do
assert(not utf8.len(s, j))
end
assert(utf8.len(s, pi, pi) == 1)
assert(utf8.len(s, pi, pi1 - 1) == 1)
assert(utf8.len(s, pi, -1) == l - i + 1)
assert(utf8.len(s, pi1, -1) == l - i)
assert(utf8.len(s, 1, pi) == i)
end
local i = 0
for p, c in utf8.codes(s) do
i = i + 1
assert(c == t[i] and p == utf8.offset(s, i))
assert(utf8.codepoint(s, p, p) == c)
end
assert(i == #t)
i = 0
for c in string.gmatch(s, utf8.charpattern) do
i = i + 1
assert(c == utf8.char(t[i]))
end
assert(i == #t)
for i = 1, l do
assert(utf8.offset(s, i) == utf8.offset(s, i - l - 1, #s + 1))
end
end
do -- error indication in utf8.len
local function check (s, p)
local a, b = utf8.len(s)
assert(not a and b == p)
end
check("abc\xE3def", 4)
check("\xF4\x9F\xBF", 1)
check("\xF4\x9F\xBF\xBF", 1)
-- spurious continuation bytes
check("汉字\x80", #("汉字") + 1)
check("\x80hello", 1)
check("hel\x80lo", 4)
check("汉字\xBF", #("汉字") + 1)
check("\xBFhello", 1)
check("hel\xBFlo", 4)
end
-- errors in utf8.codes
do
local function errorcodes (s)
checkerror("invalid UTF%-8 code",
function ()
for c in utf8.codes(s) do assert(c) end
end)
end
errorcodes("ab\xff")
errorcodes("\244\144\128\128") -- "\u{110000}" in Lua 5.4
errorcodes("in\x80valid")
errorcodes("\xbfinvalid")
errorcodes("αλφ\xBFα")
-- calling interation function with invalid arguments
local f = utf8.codes("")
assert(f("", 2) == nil)
assert(f("", -1) == nil)
assert(f("", math.mininteger) == nil)
end
-- error in initial position for offset
checkerror("position out of range", utf8.offset, "abc", 1, 5)
checkerror("position out of range", utf8.offset, "abc", 1, -4)
checkerror("position out of range", utf8.offset, "", 1, 2)
checkerror("position out of range", utf8.offset, "", 1, -1)
checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
checkerror("continuation byte", utf8.offset, "\x80", 1)
-- error in indices for len
checkerror("out of string", utf8.len, "abc", 0, 2)
checkerror("out of string", utf8.len, "abc", 1, 4)
local s = "hello World"
local t = {string.byte(s, 1, -1)}
for i = 1, utf8.len(s) do assert(t[i] == string.byte(s, i)) end
check(s, t)
check("汉字/漢字", {27721, 23383, 47, 28450, 23383,})
do
local s = "áéí\128"
local t = {utf8.codepoint(s,1,#s - 1)}
assert(#t == 3 and t[1] == 225 and t[2] == 233 and t[3] == 237)
checkerror("invalid UTF%-8 code", utf8.codepoint, s, 1, #s)
checkerror("out of range", utf8.codepoint, s, #s + 1)
t = {utf8.codepoint(s, 4, 3)}
assert(#t == 0)
checkerror("out of range", utf8.codepoint, s, -(#s + 1), 1)
checkerror("out of range", utf8.codepoint, s, 1, #s + 1)
-- surrogates
assert(utf8.codepoint("\u{D7FF}") == 0xD800 - 1)
assert(utf8.codepoint("\u{E000}") == 0xDFFF + 1)
assert(utf8.codepoint("\u{D800}", 1, 1) == 0xD800) -- TODO: this is an error in Lua 5.4
assert(utf8.codepoint("\u{DFFF}", 1, 1) == 0xDFFF) -- TODO: this is an error in Lua 5.4
assert(pcall(utf8.codepoint, "\253\191\191\191\191\191") == false) -- 0x7FFFFFFF in Lua 5.4 when called with lax=true
end
assert(utf8.char() == "")
assert(utf8.char(0, 97, 98, 99, 1) == "\0abc\1")
assert(utf8.codepoint(utf8.char(0x10FFFF)) == 0x10FFFF)
assert(pcall(utf8.char, 0x7FFFFFFF) == false) -- valid in Lua 5.4
checkerror("value out of range", utf8.char, 0x7FFFFFFF + 1)
checkerror("value out of range", utf8.char, -1)
local function invalid (s)
checkerror("invalid UTF%-8 code", utf8.codepoint, s)
assert(not utf8.len(s))
end
-- UTF-8 representation for 0x11ffff (value out of valid range)
invalid("\xF4\x9F\xBF\xBF")
-- surrogates
-- invalid("\u{D800}") TODO: this is an error in Lua 5.4
-- invalid("\u{DFFF}") TODO: this is an error in Lua 5.4
-- overlong sequences
invalid("\xC0\x80") -- zero
invalid("\xC1\xBF") -- 0x7F (should be coded in 1 byte)
invalid("\xE0\x9F\xBF") -- 0x7FF (should be coded in 2 bytes)
invalid("\xF0\x8F\xBF\xBF") -- 0xFFFF (should be coded in 3 bytes)
-- invalid bytes
invalid("\x80") -- continuation byte
invalid("\xBF") -- continuation byte
invalid("\xFE") -- invalid byte
invalid("\xFF") -- invalid byte
-- empty string
check("", {})
-- minimum and maximum values for each sequence size
s = "\0 \x7F\z
\xC2\x80 \xDF\xBF\z
\xE0\xA0\x80 \xEF\xBF\xBF\z
\xF0\x90\x80\x80 \xF4\x8F\xBF\xBF"
s = string.gsub(s, " ", "")
check(s, {0,0x7F, 0x80,0x7FF, 0x800,0xFFFF, 0x10000,0x10FFFF})
local x = "日本語a-4\0éó"
check(x, {26085, 26412, 35486, 97, 45, 52, 0, 233, 243})
-- Supplementary Characters
check("𣲷𠜎𠱓𡁻𠵼ab𠺢",
{0x23CB7, 0x2070E, 0x20C53, 0x2107B, 0x20D7C, 0x61, 0x62, 0x20EA2,})
check("𨳊𩶘𦧺𨳒𥄫𤓓\xF4\x8F\xBF\xBF",
{0x28CCA, 0x29D98, 0x269FA, 0x28CD2, 0x2512B, 0x244D3, 0x10ffff})
local i = 0
for p, c in string.gmatch(x, "()(" .. utf8.charpattern .. ")") do
i = i + 1
assert(utf8.offset(x, i) == p)
assert(utf8.len(x, p) == utf8.len(x) - i + 1)
assert(utf8.len(c) == 1)
for j = 1, #c - 1 do
assert(utf8.offset(x, 0, p + j - 1) == p)
end
end
return 'OK'