luau-lang/luau
1
0
Fork 0
mirror of https://github.com/luau-lang/luau.git synced 2025-04-03 02:10:53 +01:00
Code Issues Projects Releases Packages Wiki Activity

Introduce skipWhitespace

Browse source
This commit is contained in:
JohnnyMorganz 2024-12-16 21:18:14 +00:00
parent a14ee98529
commit bba1afce66
4 changed files with 16 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Ast/include/Luau/Lexer.h
View file

@ -157,6 +157,7 @@ public:
Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names, Position startPosition = {0, 0});
void setSkipComments(bool skip);
void setSkipWhitespace(bool skip);
void setReadNames(bool read);
const Location& previousLocation() const
@ -165,7 +166,7 @@ public:
}
const Lexeme& next();
const Lexeme& next(bool skipComments, bool updatePrevLocation);
const Lexeme& next(bool skipComments, bool skipWhitespace, bool updatePrevLocation);
void nextline();
Lexeme lookahead();
@ -229,6 +230,7 @@ private:
AstNameTable& names;
bool skipComments;
bool skipWhitespace;
bool readNames;
enum class BraceType

12
Ast/src/Lexer.cpp
View file

@ -317,6 +317,7 @@ Lexer::Lexer(const char* buffer, size_t bufferSize, AstNameTable& names, Positio
)
, names(names)
, skipComments(false)
, skipWhitespace(true)
, readNames(true)
{
}
@ -326,6 +327,11 @@ void Lexer::setSkipComments(bool skip)
skipComments = skip;
}
void Lexer::setSkipWhitespace(bool skip)
{
skipWhitespace = skip;
}
void Lexer::setReadNames(bool read)
{
readNames = read;
@ -333,10 +339,10 @@ void Lexer::setReadNames(bool read)
const Lexeme& Lexer::next()
{
return next(this->skipComments, true);
return next(this->skipComments, this->skipWhitespace, true);
}
const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation)
const Lexeme& Lexer::next(bool skipComments, bool skipWhitespace, bool updatePrevLocation)
{
// in skipComments mode we reject valid comments
do
@ -353,7 +359,7 @@ const Lexeme& Lexer::next(bool skipComments, bool updatePrevLocation)
lexeme = readNext();
updatePrevLocation = false;
} while (skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment || lexeme.type == Lexeme::Whitespace));
} while ((skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment)) || (skipWhitespace && lexeme.type == Lexeme::Whitespace));
return lexeme;
}

4
Ast/src/Parser.cpp
View file

@ -3572,7 +3572,7 @@ AstTypeError* Parser::reportMissingTypeError(const Location& parseErrorLocation,
void Parser::nextLexeme()
{
Lexeme::Type type = lexer.next(/* skipComments= */ false, true).type;
Lexeme::Type type = lexer.next(/* skipComments= */ false, /* skipWhitespace= */ false, true).type;
while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment || type == Lexeme::Whitespace)
{
@ -3598,7 +3598,7 @@ void Parser::nextLexeme()
hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)});
}
type = lexer.next(/* skipComments= */ false, /* updatePrevLocation= */ false).type;
type = lexer.next(/* skipComments= */ false, /* skipWhitespace= */ false, /* updatePrevLocation= */ false).type;
}
}

2
tests/Lexer.test.cpp
View file

@ -252,6 +252,7 @@ TEST_CASE("lexer_tokenizes_whitespace")
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipWhitespace(false);
CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal);
CHECK_EQ(lexer.next().type, Lexeme::Whitespace);
@ -282,6 +283,7 @@ TEST_CASE("lexer_tokenizes_multiline_whitespace")
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
lexer.setSkipWhitespace(false);
CHECK_EQ(lexer.next().type, Lexeme::ReservedLocal);
CHECK_EQ(lexer.next().type, Lexeme::Whitespace);