luau-lang/luau
1
0
Fork 0
mirror of https://github.com/luau-lang/luau.git synced 2025-04-04 10:50:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

Revert skipComments -> skipTrivia

Browse source
This commit is contained in:
JohnnyMorganz 2024-12-16 21:11:04 +00:00
parent 158e56956b
commit a14ee98529
4 changed files with 16 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Ast/include/Luau/Lexer.h
View file

@ -156,7 +156,7 @@ class Lexer
public: public:
Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names, Position startPosition = {0, 0}); Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names, Position startPosition = {0, 0});
void setSkipTrivia(bool skip); void setSkipComments(bool skip);
void setReadNames(bool read); void setReadNames(bool read);
const Location& previousLocation() const const Location& previousLocation() const
@ -165,7 +165,7 @@ public:
} }
const Lexeme& next(); const Lexeme& next();
const Lexeme& next(bool skipTrivia, bool updatePrevLocation); const Lexeme& next(bool skipComments, bool updatePrevLocation);
void nextline(); void nextline();
Lexeme lookahead(); Lexeme lookahead();
@ -228,7 +228,7 @@ private:
AstNameTable& names; AstNameTable& names;
bool skipTrivia; bool skipComments;
bool readNames; bool readNames;
enum class BraceType enum class BraceType

14
Ast/src/Lexer.cpp
View file

@ -316,14 +316,14 @@ Lexer::Lexer(const char* buffer, size_t bufferSize, AstNameTable& names, Positio
Lexeme::Eof Lexeme::Eof
) )
, names(names) , names(names)
, skipTrivia(false) , skipComments(false)
, readNames(true) , readNames(true)
{ {
} }
void Lexer::setSkipTrivia(bool skip) void Lexer::setSkipComments(bool skip)
{ {
skipTrivia = skip; skipComments = skip;
} }
void Lexer::setReadNames(bool read) void Lexer::setReadNames(bool read)
@ -333,12 +333,12 @@ void Lexer::setReadNames(bool read)
const Lexeme& Lexer::next() const Lexeme& Lexer::next()
{ {
return next(this->skipTrivia, true); return next(this->skipComments, true);
} }
const Lexeme& Lexer::next(bool skipTrivia, bool updatePrevLocation) const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation)
{ {
// in skipTrivia mode we reject valid comments // in skipComments mode we reject valid comments
do do
{ {
if (!FFlag::LuauLexerTokenizesWhitespace) if (!FFlag::LuauLexerTokenizesWhitespace)
@ -353,7 +353,7 @@ const Lexeme& Lexer::next(bool skipTrivia, bool updatePrevLocation)
lexeme = readNext(); lexeme = readNext();
updatePrevLocation = false; updatePrevLocation = false;
} while (skipTrivia && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment || lexeme.type == Lexeme::Whitespace)); } while (skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment || lexeme.type == Lexeme::Whitespace));
return lexeme; return lexeme;
} }

6
Ast/src/Parser.cpp
View file

@ -208,7 +208,7 @@ Parser::Parser(const char* buffer, size_t bufferSize, AstNameTable& names, Alloc
matchRecoveryStopOnToken[Lexeme::Type::Eof] = 1; matchRecoveryStopOnToken[Lexeme::Type::Eof] = 1;
// required for lookahead() to work across a comment boundary and for nextLexeme() to work when captureComments is false // required for lookahead() to work across a comment boundary and for nextLexeme() to work when captureComments is false
lexer.setSkipTrivia(true); lexer.setSkipComments(true);
// read first lexeme (any hot comments get .header = true) // read first lexeme (any hot comments get .header = true)
LUAU_ASSERT(hotcommentHeader); LUAU_ASSERT(hotcommentHeader);
@ -3572,7 +3572,7 @@ AstTypeError* Parser::reportMissingTypeError(const Location& parseErrorLocation,
void Parser::nextLexeme() void Parser::nextLexeme()
{ {
Lexeme::Type type = lexer.next(/* skipTrivia= */ false, true).type; Lexeme::Type type = lexer.next(/* skipComments= */ false, true).type;
while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment || type == Lexeme::Whitespace) while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment || type == Lexeme::Whitespace)
{ {
@ -3598,7 +3598,7 @@ void Parser::nextLexeme()
hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)}); hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)});
} }
type = lexer.next(/* skipTrivia= */ false, /* updatePrevLocation= */ false).type; type = lexer.next(/* skipComments= */ false, /* updatePrevLocation= */ false).type;
} }
} }

6
tests/Lexer.test.cpp
View file

@ -40,7 +40,7 @@ TEST_CASE("broken_comment_kept")
Luau::Allocator alloc; Luau::Allocator alloc;
AstNameTable table(alloc); AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table); Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipTrivia(true); lexer.setSkipComments(true);
CHECK_EQ(lexer.next().type, Lexeme::Type::BrokenComment); CHECK_EQ(lexer.next().type, Lexeme::Type::BrokenComment);
} }
@ -50,7 +50,7 @@ TEST_CASE("comment_skipped")
Luau::Allocator alloc; Luau::Allocator alloc;
AstNameTable table(alloc); AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table); Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipTrivia(true); lexer.setSkipComments(true);
CHECK_EQ(lexer.next().type, Lexeme::Type::Eof); CHECK_EQ(lexer.next().type, Lexeme::Type::Eof);
} }
@ -105,7 +105,7 @@ TEST_CASE("lookahead")
Luau::Allocator alloc; Luau::Allocator alloc;
AstNameTable table(alloc); AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table); Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipTrivia(true); lexer.setSkipComments(true);
lexer.next(); // must call next() before reading data from lexer at least once lexer.next(); // must call next() before reading data from lexer at least once
CHECK_EQ(lexer.current().type, Lexeme::Name); CHECK_EQ(lexer.current().type, Lexeme::Name);