mirror of
https://github.com/luau-lang/luau.git
synced 2024-12-13 05:20:38 +00:00
245 lines
7.7 KiB
C++
245 lines
7.7 KiB
C++
// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
|
|
#include "Luau/Lexer.h"
|
|
|
|
#include "Fixture.h"
|
|
#include "ScopedFlags.h"
|
|
|
|
#include "doctest.h"
|
|
|
|
using namespace Luau;
|
|
|
|
TEST_SUITE_BEGIN("LexerTests");
|
|
|
|
TEST_CASE("broken_string_works")
|
|
{
|
|
const std::string testInput = "[[";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
Lexeme lexeme = lexer.next();
|
|
CHECK_EQ(lexeme.type, Lexeme::Type::BrokenString);
|
|
CHECK_EQ(lexeme.location, Luau::Location(Luau::Position(0, 0), Luau::Position(0, 2)));
|
|
}
|
|
|
|
TEST_CASE("broken_comment")
|
|
{
|
|
const std::string testInput = "--[[ ";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
Lexeme lexeme = lexer.next();
|
|
CHECK_EQ(lexeme.type, Lexeme::Type::BrokenComment);
|
|
CHECK_EQ(lexeme.location, Luau::Location(Luau::Position(0, 0), Luau::Position(0, 6)));
|
|
}
|
|
|
|
TEST_CASE("broken_comment_kept")
|
|
{
|
|
const std::string testInput = "--[[ ";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
lexer.setSkipComments(true);
|
|
CHECK_EQ(lexer.next().type, Lexeme::Type::BrokenComment);
|
|
}
|
|
|
|
TEST_CASE("comment_skipped")
|
|
{
|
|
const std::string testInput = "-- ";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
lexer.setSkipComments(true);
|
|
CHECK_EQ(lexer.next().type, Lexeme::Type::Eof);
|
|
}
|
|
|
|
TEST_CASE("multilineCommentWithLexemeInAndAfter")
|
|
{
|
|
const std::string testInput = "--[[ function \n"
|
|
"]] end";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
Lexeme comment = lexer.next();
|
|
Lexeme end = lexer.next();
|
|
|
|
CHECK_EQ(comment.type, Lexeme::Type::BlockComment);
|
|
CHECK_EQ(comment.location, Luau::Location(Luau::Position(0, 0), Luau::Position(1, 2)));
|
|
CHECK_EQ(end.type, Lexeme::Type::ReservedEnd);
|
|
CHECK_EQ(end.location, Luau::Location(Luau::Position(1, 3), Luau::Position(1, 6)));
|
|
}
|
|
|
|
TEST_CASE("testBrokenEscapeTolerant")
|
|
{
|
|
const std::string testInput = "'\\3729472897292378'";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
Lexeme item = lexer.next();
|
|
|
|
CHECK_EQ(item.type, Lexeme::QuotedString);
|
|
CHECK_EQ(item.location, Luau::Location(Luau::Position(0, 0), Luau::Position(0, int(testInput.size()))));
|
|
}
|
|
|
|
TEST_CASE("testBigDelimiters")
|
|
{
|
|
const std::string testInput = "--[===[\n"
|
|
"\n"
|
|
"\n"
|
|
"\n"
|
|
"]===]";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
Lexeme item = lexer.next();
|
|
|
|
CHECK_EQ(item.type, Lexeme::Type::BlockComment);
|
|
CHECK_EQ(item.location, Luau::Location(Luau::Position(0, 0), Luau::Position(4, 5)));
|
|
}
|
|
|
|
TEST_CASE("lookahead")
|
|
{
|
|
const std::string testInput = "foo --[[ comment ]] bar : nil end";
|
|
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
lexer.setSkipComments(true);
|
|
lexer.next(); // must call next() before reading data from lexer at least once
|
|
|
|
CHECK_EQ(lexer.current().type, Lexeme::Name);
|
|
CHECK_EQ(lexer.current().name, std::string("foo"));
|
|
CHECK_EQ(lexer.lookahead().type, Lexeme::Name);
|
|
CHECK_EQ(lexer.lookahead().name, std::string("bar"));
|
|
|
|
lexer.next();
|
|
|
|
CHECK_EQ(lexer.current().type, Lexeme::Name);
|
|
CHECK_EQ(lexer.current().name, std::string("bar"));
|
|
CHECK_EQ(lexer.lookahead().type, ':');
|
|
|
|
lexer.next();
|
|
|
|
CHECK_EQ(lexer.current().type, ':');
|
|
CHECK_EQ(lexer.lookahead().type, Lexeme::ReservedNil);
|
|
|
|
lexer.next();
|
|
|
|
CHECK_EQ(lexer.current().type, Lexeme::ReservedNil);
|
|
CHECK_EQ(lexer.lookahead().type, Lexeme::ReservedEnd);
|
|
|
|
lexer.next();
|
|
|
|
CHECK_EQ(lexer.current().type, Lexeme::ReservedEnd);
|
|
CHECK_EQ(lexer.lookahead().type, Lexeme::Eof);
|
|
|
|
lexer.next();
|
|
|
|
CHECK_EQ(lexer.current().type, Lexeme::Eof);
|
|
CHECK_EQ(lexer.lookahead().type, Lexeme::Eof);
|
|
}
|
|
|
|
TEST_CASE("string_interpolation_basic")
|
|
{
|
|
const std::string testInput = R"(`foo {"bar"}`)";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
|
|
Lexeme interpBegin = lexer.next();
|
|
CHECK_EQ(interpBegin.type, Lexeme::InterpStringBegin);
|
|
|
|
Lexeme quote = lexer.next();
|
|
CHECK_EQ(quote.type, Lexeme::QuotedString);
|
|
|
|
Lexeme interpEnd = lexer.next();
|
|
CHECK_EQ(interpEnd.type, Lexeme::InterpStringEnd);
|
|
}
|
|
|
|
TEST_CASE("string_interpolation_full")
|
|
{
|
|
const std::string testInput = R"(`foo {"bar"} {"baz"} end`)";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
|
|
Lexeme interpBegin = lexer.next();
|
|
CHECK_EQ(interpBegin.type, Lexeme::InterpStringBegin);
|
|
CHECK_EQ(interpBegin.toString(), "`foo {");
|
|
|
|
Lexeme quote1 = lexer.next();
|
|
CHECK_EQ(quote1.type, Lexeme::QuotedString);
|
|
CHECK_EQ(quote1.toString(), "\"bar\"");
|
|
|
|
Lexeme interpMid = lexer.next();
|
|
CHECK_EQ(interpMid.type, Lexeme::InterpStringMid);
|
|
CHECK_EQ(interpMid.toString(), "} {");
|
|
|
|
Lexeme quote2 = lexer.next();
|
|
CHECK_EQ(quote2.type, Lexeme::QuotedString);
|
|
CHECK_EQ(quote2.toString(), "\"baz\"");
|
|
|
|
Lexeme interpEnd = lexer.next();
|
|
CHECK_EQ(interpEnd.type, Lexeme::InterpStringEnd);
|
|
CHECK_EQ(interpEnd.toString(), "} end`");
|
|
}
|
|
|
|
TEST_CASE("string_interpolation_double_brace")
|
|
{
|
|
const std::string testInput = R"(`foo{{bad}}bar`)";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
|
|
auto brokenInterpBegin = lexer.next();
|
|
CHECK_EQ(brokenInterpBegin.type, Lexeme::BrokenInterpDoubleBrace);
|
|
CHECK_EQ(std::string(brokenInterpBegin.data, brokenInterpBegin.length), std::string("foo"));
|
|
|
|
CHECK_EQ(lexer.next().type, Lexeme::Name);
|
|
|
|
auto interpEnd = lexer.next();
|
|
CHECK_EQ(interpEnd.type, Lexeme::InterpStringEnd);
|
|
CHECK_EQ(std::string(interpEnd.data, interpEnd.length), std::string("}bar"));
|
|
}
|
|
|
|
TEST_CASE("string_interpolation_double_but_unmatched_brace")
|
|
{
|
|
const std::string testInput = R"(`{{oops}`, 1)";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
|
|
CHECK_EQ(lexer.next().type, Lexeme::BrokenInterpDoubleBrace);
|
|
CHECK_EQ(lexer.next().type, Lexeme::Name);
|
|
CHECK_EQ(lexer.next().type, Lexeme::InterpStringEnd);
|
|
CHECK_EQ(lexer.next().type, ',');
|
|
CHECK_EQ(lexer.next().type, Lexeme::Number);
|
|
}
|
|
|
|
TEST_CASE("string_interpolation_unmatched_brace")
|
|
{
|
|
const std::string testInput = R"({
|
|
`hello {"world"}
|
|
} -- this might be incorrectly parsed as a string)";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
|
|
CHECK_EQ(lexer.next().type, '{');
|
|
CHECK_EQ(lexer.next().type, Lexeme::InterpStringBegin);
|
|
CHECK_EQ(lexer.next().type, Lexeme::QuotedString);
|
|
CHECK_EQ(lexer.next().type, Lexeme::BrokenString);
|
|
CHECK_EQ(lexer.next().type, '}');
|
|
}
|
|
|
|
TEST_CASE("string_interpolation_with_unicode_escape")
|
|
{
|
|
const std::string testInput = R"(`\u{1F41B}`)";
|
|
Luau::Allocator alloc;
|
|
AstNameTable table(alloc);
|
|
Lexer lexer(testInput.c_str(), testInput.size(), table);
|
|
|
|
CHECK_EQ(lexer.next().type, Lexeme::InterpStringSimple);
|
|
CHECK_EQ(lexer.next().type, Lexeme::Eof);
|
|
}
|
|
|
|
TEST_SUITE_END();
|