diff --git a/Ast/include/Luau/Lexer.h b/Ast/include/Luau/Lexer.h index d6fe24f8..7e1c0007 100644 --- a/Ast/include/Luau/Lexer.h +++ b/Ast/include/Luau/Lexer.h @@ -64,6 +64,8 @@ struct Lexeme InterpStringBegin, InterpStringMid, InterpStringEnd, + // An interpolated string with no expressions (like `x`) + InterpStringSimple, AddAssign, SubAssign, @@ -173,8 +175,6 @@ public: void setSkipComments(bool skip); void setReadNames(bool read); - const Lexeme& nextInterpolatedString(); - const Location& previousLocation() const { return prevLocation; @@ -244,6 +244,14 @@ private: bool skipComments; bool readNames; + + enum BraceType + { + InterpolatedString, + Normal + }; + + std::vector braceStack; }; inline bool isSpace(char ch) diff --git a/Ast/src/Lexer.cpp b/Ast/src/Lexer.cpp index f24ff332..79ba02e8 100644 --- a/Ast/src/Lexer.cpp +++ b/Ast/src/Lexer.cpp @@ -97,6 +97,7 @@ Lexeme::Lexeme(const Location& location, Type type, const char* data, size_t siz || type == InterpStringBegin || type == InterpStringMid || type == InterpStringEnd + || type == InterpStringSimple || type == Number || type == Comment || type == BlockComment @@ -171,6 +172,18 @@ std::string Lexeme::toString() const case QuotedString: return data ? format("\"%.*s\"", length, data) : "string"; + case InterpStringBegin: + return data ? format("the beginning of an interpolated string (`%.*s`)", length, data) : "the beginning of an interpolated string"; + + case InterpStringMid: + return data ? format("the middle of an interpolated string (`%.*s`)", length, data) : "the middle of an interpolated string"; + + case InterpStringEnd: + return data ? format("the end of an interpolated string (`%.*s`)", length, data) : "the end of an interpolated string"; + + case InterpStringSimple: + return data ? format("`%.*s`", length, data) : "interpolated string"; + case Number: return data ? format("'%.*s'", length, data) : "number"; @@ -587,14 +600,6 @@ Lexeme Lexer::readQuotedString() return Lexeme(Location(start, position()), Lexeme::QuotedString, &buffer[startOffset], offset - startOffset - 1); } -const Lexeme& Lexer::nextInterpolatedString() -{ - Position start = position(); - - lexeme = readInterpolatedStringSection(start, Lexeme::InterpStringMid, Lexeme::InterpStringEnd); - return lexeme; -} - Lexeme Lexer::readInterpolatedStringBegin() { LUAU_ASSERT(peekch() == '`'); @@ -602,7 +607,7 @@ Lexeme Lexer::readInterpolatedStringBegin() Position start = position(); consume(); - return readInterpolatedStringSection(start, Lexeme::InterpStringBegin, Lexeme::InterpStringEnd); + return readInterpolatedStringSection(start, Lexeme::InterpStringBegin, Lexeme::InterpStringSimple); } Lexeme Lexer::readInterpolatedStringSection(Position start, Lexeme::Type formatType, Lexeme::Type endType) @@ -616,6 +621,7 @@ Lexeme Lexer::readInterpolatedStringSection(Position start, Lexeme::Type formatT case 0: case '\r': case '\n': + // INTERP TODO: Clear anything we've added to the brace stack, and write a test to see what happens if we don't return Lexeme(Location(start, position()), Lexeme::BrokenString); case '\\': @@ -629,6 +635,7 @@ Lexeme Lexer::readInterpolatedStringSection(Position start, Lexeme::Type formatT return Lexeme(Location(start, position()), Lexeme::BrokenInterpDoubleBrace); } + braceStack.push_back(BraceType::InterpolatedString); Lexeme lexemeOutput(Location(start, position()), Lexeme::InterpStringBegin, &buffer[startOffset], offset - startOffset); consume(); return lexemeOutput; @@ -735,6 +742,34 @@ Lexeme Lexer::readNext() } } + case '{': + { + consume(); + + braceStack.push_back(BraceType::Normal); + return Lexeme(Location(start, 1), '{'); + } + + case '}': + { + consume(); + + if (braceStack.empty()) + { + return Lexeme(Location(start, 1), '}'); + } + + const BraceType braceStackTop = braceStack.back(); + braceStack.pop_back(); + + if (braceStackTop != BraceType::InterpolatedString) + { + return Lexeme(Location(start, 1), '}'); + } + + return readInterpolatedStringSection(position(), Lexeme::InterpStringMid, Lexeme::InterpStringEnd); + } + case '=': { consume(); @@ -901,8 +936,6 @@ Lexeme Lexer::readNext() case '(': case ')': - case '{': - case '}': case ']': case ';': case ',': diff --git a/Ast/src/Parser.cpp b/Ast/src/Parser.cpp index 390a103b..b7ef27ed 100644 --- a/Ast/src/Parser.cpp +++ b/Ast/src/Parser.cpp @@ -1573,7 +1573,7 @@ AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack) else return {reportTypeAnnotationError(begin, {}, /*isMissing*/ false, "String literal contains malformed escape sequence")}; } - else if (lexer.current().type == Lexeme::InterpStringBegin || lexer.current().type == Lexeme::InterpStringEnd) + else if (lexer.current().type == Lexeme::InterpStringBegin || lexer.current().type == Lexeme::InterpStringSimple) { parseInterpString(); @@ -2014,7 +2014,7 @@ AstExpr* Parser::parsePrimaryExpr(bool asStatement) || lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::InterpStringBegin - || lexer.current().type == Lexeme::InterpStringEnd + || lexer.current().type == Lexeme::InterpStringSimple ) { expr = parseFunctionArgs(expr, false, Location()); @@ -2286,7 +2286,7 @@ AstExpr* Parser::parseSimpleExpr() } } } - else if (lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::QuotedString || (FFlag::LuauInterpolatedStringBaseSupport && lexer.current().type == Lexeme::InterpStringEnd)) + else if (lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::QuotedString || (FFlag::LuauInterpolatedStringBaseSupport && lexer.current().type == Lexeme::InterpStringSimple)) { return parseString(); } @@ -2375,7 +2375,7 @@ AstExpr* Parser::parseFunctionArgs(AstExpr* func, bool self, const Location& sel return allocator.alloc(Location(func->location, expr->location), func, copy(&expr, 1), self, argLocation); } - else if (FFlag::LuauInterpolatedStringBaseSupport && (lexer.current().type == Lexeme::InterpStringBegin || lexer.current().type == Lexeme::InterpStringEnd)) + else if (FFlag::LuauInterpolatedStringBaseSupport && (lexer.current().type == Lexeme::InterpStringBegin || lexer.current().type == Lexeme::InterpStringSimple)) { Position argStart = lexer.current().location.end; AstExpr* expr = parseInterpString(); @@ -2694,11 +2694,11 @@ AstArray Parser::parseTypeParams() std::optional> Parser::parseCharArray() { - LUAU_ASSERT(lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::InterpStringEnd); + LUAU_ASSERT(lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::InterpStringSimple); scratchData.assign(lexer.current().data, lexer.current().length); - if (lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::InterpStringEnd) + if (lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::InterpStringSimple) { if (!Lexer::fixupQuotedString(scratchData)) { @@ -2734,7 +2734,12 @@ AstExpr* Parser::parseInterpString() do { Lexeme currentLexeme = lexer.current(); - LUAU_ASSERT(currentLexeme.type == Lexeme::InterpStringBegin || currentLexeme.type == Lexeme::InterpStringMid || currentLexeme.type == Lexeme::InterpStringEnd); + LUAU_ASSERT( + currentLexeme.type == Lexeme::InterpStringBegin + || currentLexeme.type == Lexeme::InterpStringMid + || currentLexeme.type == Lexeme::InterpStringEnd + || currentLexeme.type == Lexeme::InterpStringSimple + ); Location location = currentLexeme.location; @@ -2754,7 +2759,7 @@ AstExpr* Parser::parseInterpString() strings.push_back(chars); - if (currentLexeme.type == Lexeme::InterpStringEnd) + if (currentLexeme.type == Lexeme::InterpStringEnd || currentLexeme.type == Lexeme::InterpStringSimple) { AstArray> stringsArray = copy(strings); AstArray expressionsArray = copy(expressions); @@ -2764,24 +2769,20 @@ AstExpr* Parser::parseInterpString() AstExpr* expression = parseExpr(); - // INTERP CODE REVIEW: I want to use expectMatchAndConsume, but using that - // consumes the rest of the string, not the `}` - if (lexer.current().type != static_cast(static_cast('}'))) { - return reportExprError(startOfBrace, {}, "Expected '}' after interpolated string expression"); - } - expressions.push_back(expression); - Lexeme next = lexer.nextInterpolatedString(); - - switch (next.type) + switch (lexer.current().type) { - case Lexeme::BrokenString: - return reportExprError(location, {}, "Malformed interpolated string"); + case Lexeme::InterpStringBegin: + case Lexeme::InterpStringMid: + case Lexeme::InterpStringEnd: + break; case Lexeme::BrokenInterpDoubleBrace: return reportExprError(location, {}, ERROR_INVALID_INTERP_DOUBLE_BRACE); + case Lexeme::BrokenString: + return reportExprError(location, {}, "Malformed interpolated string, did you forget to add a '}'?"); default: - break; + return reportExprError(location, {}, "Malformed interpolated string, got %s", lexer.current().toString().c_str()); } } while (true); } diff --git a/tests/Lexer.test.cpp b/tests/Lexer.test.cpp index 20d8d0d5..54a55400 100644 --- a/tests/Lexer.test.cpp +++ b/tests/Lexer.test.cpp @@ -138,4 +138,23 @@ TEST_CASE("lookahead") CHECK_EQ(lexer.lookahead().type, Lexeme::Eof); } +TEST_CASE("stringInterpolation") +{ + ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true}; + + const std::string testInput = R"(`foo {"bar"}`)"; + Luau::Allocator alloc; + AstNameTable table(alloc); + Lexer lexer(testInput.c_str(), testInput.size(), table); + + Lexeme interpBegin = lexer.next(); + CHECK_EQ(interpBegin.type, Lexeme::InterpStringBegin); + + Lexeme quote = lexer.next(); + CHECK_EQ(quote.type, Lexeme::QuotedString); + + Lexeme interpEnd = lexer.next(); + CHECK_EQ(interpEnd.type, Lexeme::InterpStringEnd); +} + TEST_SUITE_END(); diff --git a/tests/Parser.test.cpp b/tests/Parser.test.cpp index c975a375..b6c096ed 100644 --- a/tests/Parser.test.cpp +++ b/tests/Parser.test.cpp @@ -954,7 +954,7 @@ TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_without_end_brace") catch (const ParseErrors& e) { auto error = e.getErrors().front(); - CHECK_EQ("Expected '}' after interpolated string expression", error.getMessage()); + CHECK_EQ("Malformed interpolated string, did you forget to add a '}'?", error.getMessage()); return error.getLocation().begin.column; } };