luau-lang/luau
1
0
Fork 0
mirror of https://github.com/luau-lang/luau.git synced 2025-04-03 18:30:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

Introduce skipWhitespace

Browse source
This commit is contained in:
JohnnyMorganz 2024-12-16 21:18:14 +00:00
parent a14ee98529
commit bba1afce66
4 changed files with 16 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Ast/include/Luau/Lexer.h
View file

@ -157,6 +157,7 @@ public:
Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names, Position startPosition = {0, 0}); Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names, Position startPosition = {0, 0});
void setSkipComments(bool skip); void setSkipComments(bool skip);
void setSkipWhitespace(bool skip);
void setReadNames(bool read); void setReadNames(bool read);
const Location& previousLocation() const const Location& previousLocation() const
@ -165,7 +166,7 @@ public:
} }
const Lexeme& next(); const Lexeme& next();
const Lexeme& next(bool skipComments, bool updatePrevLocation); const Lexeme& next(bool skipComments, bool skipWhitespace, bool updatePrevLocation);
void nextline(); void nextline();
Lexeme lookahead(); Lexeme lookahead();
@ -229,6 +230,7 @@ private:
AstNameTable& names; AstNameTable& names;
bool skipComments; bool skipComments;
bool skipWhitespace;
bool readNames; bool readNames;
enum class BraceType enum class BraceType

12
Ast/src/Lexer.cpp
View file

@ -317,6 +317,7 @@ Lexer::Lexer(const char* buffer, size_t bufferSize, AstNameTable& names, Positio
) )
, names(names) , names(names)
, skipComments(false) , skipComments(false)
, skipWhitespace(true)
, readNames(true) , readNames(true)
{ {
} }
@ -326,6 +327,11 @@ void Lexer::setSkipComments(bool skip)
skipComments = skip; skipComments = skip;
} }
void Lexer::setSkipWhitespace(bool skip)
{
skipWhitespace = skip;
}
void Lexer::setReadNames(bool read) void Lexer::setReadNames(bool read)
{ {
readNames = read; readNames = read;
@ -333,10 +339,10 @@ void Lexer::setReadNames(bool read)
const Lexeme& Lexer::next() const Lexeme& Lexer::next()
{ {
return next(this->skipComments, true); return next(this->skipComments, this->skipWhitespace, true);
} }
const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation) const Lexeme& Lexer::next(bool skipComments, bool skipWhitespace, bool updatePrevLocation)
{ {
// in skipComments mode we reject valid comments // in skipComments mode we reject valid comments
do do
@ -353,7 +359,7 @@ const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation)
lexeme = readNext(); lexeme = readNext();
updatePrevLocation = false; updatePrevLocation = false;
} while (skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment || lexeme.type == Lexeme::Whitespace)); } while ((skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment)) || (skipWhitespace && lexeme.type == Lexeme::Whitespace));
return lexeme; return lexeme;
} }

4
Ast/src/Parser.cpp
View file

@ -3572,7 +3572,7 @@ AstTypeError* Parser::reportMissingTypeError(const Location& parseErrorLocation,
void Parser::nextLexeme() void Parser::nextLexeme()
{ {
Lexeme::Type type = lexer.next(/* skipComments= */ false, true).type; Lexeme::Type type = lexer.next(/* skipComments= */ false, /* skipWhitespace= */ false, true).type;
while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment || type == Lexeme::Whitespace) while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment || type == Lexeme::Whitespace)
{ {
@ -3598,7 +3598,7 @@ void Parser::nextLexeme()
hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)}); hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)});
} }
type = lexer.next(/* skipComments= */ false, /* updatePrevLocation= */ false).type; type = lexer.next(/* skipComments= */ false, /* skipWhitespace= */ false, /* updatePrevLocation= */ false).type;
} }
} }

2
tests/Lexer.test.cpp
View file

@ -252,6 +252,7 @@ TEST_CASE("lexer_tokenizes_whitespace")
Luau::Allocator alloc; Luau::Allocator alloc;
AstNameTable table(alloc); AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table); Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipWhitespace(false);
CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal); CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal);
CHECK_EQ(lexer.next().type, Lexeme::Whitespace); CHECK_EQ(lexer.next().type, Lexeme::Whitespace);
@ -282,6 +283,7 @@ TEST_CASE("lexer_tokenizes_multiline_whitespace")
Luau::Allocator alloc; Luau::Allocator alloc;
AstNameTable table(alloc); AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table); Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipWhitespace(false);
CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal); CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal);
CHECK_EQ(lexer.next().type, Lexeme::Whitespace); CHECK_EQ(lexer.next().type, Lexeme::Whitespace);