From 36ba1dd9acdc54138bb8cfb2175d5c10285c3850 Mon Sep 17 00:00:00 2001 From: Babyhamsta Date: Sat, 30 Jul 2022 22:29:41 -0500 Subject: [PATCH] Add files via upload --- Ast/include/lluz/Ast.h | 1268 +++++++++++++++++++++++++++++++ Ast/include/lluz/Confusables.h | 9 + Ast/include/lluz/DenseHash.h | 524 +++++++++++++ Ast/include/lluz/Lexer.h | 241 ++++++ Ast/include/lluz/Location.h | 109 +++ Ast/include/lluz/ParseOptions.h | 23 + Ast/include/lluz/ParseResult.h | 69 ++ Ast/include/lluz/Parser.h | 383 ++++++++++ Ast/include/lluz/StringUtils.h | 40 + Ast/include/lluz/TimeTrace.h | 230 ++++++ Ast/src/Ast.cpp | 58 +- Ast/src/Confusables.cpp | 8 +- Ast/src/Lexer.cpp | 100 +-- Ast/src/Location.cpp | 14 +- Ast/src/Parser.cpp | 404 +++++----- Ast/src/StringUtils.cpp | 18 +- Ast/src/TimeTrace.cpp | 47 +- 17 files changed, 3230 insertions(+), 315 deletions(-) create mode 100644 Ast/include/lluz/Ast.h create mode 100644 Ast/include/lluz/Confusables.h create mode 100644 Ast/include/lluz/DenseHash.h create mode 100644 Ast/include/lluz/Lexer.h create mode 100644 Ast/include/lluz/Location.h create mode 100644 Ast/include/lluz/ParseOptions.h create mode 100644 Ast/include/lluz/ParseResult.h create mode 100644 Ast/include/lluz/Parser.h create mode 100644 Ast/include/lluz/StringUtils.h create mode 100644 Ast/include/lluz/TimeTrace.h diff --git a/Ast/include/lluz/Ast.h b/Ast/include/lluz/Ast.h new file mode 100644 index 00000000..c9cee693 --- /dev/null +++ b/Ast/include/lluz/Ast.h @@ -0,0 +1,1268 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "lluz/Location.h" + +#include +#include + +#include + +namespace lluz +{ + +struct AstName +{ + const char* value; + + AstName() + : value(nullptr) + { + } + + explicit AstName(const char* value) + : value(value) + { + } + + bool operator==(const AstName& rhs) const + { + return value == rhs.value; + } + + bool operator!=(const AstName& rhs) const + { + return value != rhs.value; + } + + bool operator==(const char* rhs) const + { + return value && strcmp(value, rhs) == 0; + } + + bool operator!=(const char* rhs) const + { + return !value || strcmp(value, rhs) != 0; + } + + bool operator<(const AstName& rhs) const + { + return (value && rhs.value) ? strcmp(value, rhs.value) < 0 : value < rhs.value; + } +}; + +class AstVisitor +{ +public: + virtual ~AstVisitor() {} + + virtual bool visit(class AstNode*) + { + return true; + } + + virtual bool visit(class AstExpr* node) + { + return visit((class AstNode*)node); + } + + virtual bool visit(class AstExprGroup* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprConstantNil* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprConstantBool* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprConstantNumber* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprConstantString* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprLocal* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprGlobal* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprVarargs* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprCall* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprIndexName* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprIndexExpr* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprFunction* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprTable* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprUnary* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprBinary* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprTypeAssertion* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprIfElse* node) + { + return visit((class AstExpr*)node); + } + virtual bool visit(class AstExprError* node) + { + return visit((class AstExpr*)node); + } + + virtual bool visit(class AstStat* node) + { + return visit((class AstNode*)node); + } + + virtual bool visit(class AstStatBlock* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatIf* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatWhile* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatRepeat* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatBreak* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatContinue* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatReturn* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatExpr* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatLocal* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatFor* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatForIn* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatAssign* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatCompoundAssign* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatFunction* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatLocalFunction* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatTypeAlias* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatDeclareFunction* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatDeclareGlobal* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatDeclareClass* node) + { + return visit((class AstStat*)node); + } + virtual bool visit(class AstStatError* node) + { + return visit((class AstStat*)node); + } + + // By default visiting type annotations is disabled; override this in your visitor if you need to! + virtual bool visit(class AstType* node) + { + return false; + } + + virtual bool visit(class AstTypeReference* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeTable* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeFunction* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeTypeof* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeUnion* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeIntersection* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeSingletonBool* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeSingletonString* node) + { + return visit((class AstType*)node); + } + virtual bool visit(class AstTypeError* node) + { + return visit((class AstType*)node); + } + + virtual bool visit(class AstTypePack* node) + { + return false; + } + virtual bool visit(class AstTypePackExplicit* node) + { + return visit((class AstTypePack*)node); + } + virtual bool visit(class AstTypePackVariadic* node) + { + return visit((class AstTypePack*)node); + } + virtual bool visit(class AstTypePackGeneric* node) + { + return visit((class AstTypePack*)node); + } +}; + +class AstType; + +struct AstLocal +{ + AstName name; + Location location; + AstLocal* shadow; + size_t functionDepth; + size_t loopDepth; + + AstType* annotation; + + AstLocal(const AstName& name, const Location& location, AstLocal* shadow, size_t functionDepth, size_t loopDepth, AstType* annotation) + : name(name) + , location(location) + , shadow(shadow) + , functionDepth(functionDepth) + , loopDepth(loopDepth) + , annotation(annotation) + { + } +}; + +template +struct AstArray +{ + T* data; + size_t size; + + const T* begin() const + { + return data; + } + const T* end() const + { + return data + size; + } +}; + +struct AstTypeList +{ + AstArray types; + // Null indicates no tail, not an untyped tail. + AstTypePack* tailType = nullptr; +}; + +using AstArgumentName = std::pair; // TODO: remove and replace when we get a common struct for this pair instead of AstName + +struct AstGenericType +{ + AstName name; + Location location; + AstType* defaultValue = nullptr; +}; + +struct AstGenericTypePack +{ + AstName name; + Location location; + AstTypePack* defaultValue = nullptr; +}; + +extern int gAstRttiIndex; + +template +struct AstRtti +{ + static const int value; +}; + +template +const int AstRtti::value = ++gAstRttiIndex; + +#define lluz_RTTI(Class) \ + static int ClassIndex() \ + { \ + return AstRtti::value; \ + } + +class AstNode +{ +public: + explicit AstNode(int classIndex, const Location& location) + : classIndex(classIndex) + , location(location) + { + } + + virtual void visit(AstVisitor* visitor) = 0; + + virtual AstExpr* asExpr() + { + return nullptr; + } + virtual AstStat* asStat() + { + return nullptr; + } + virtual AstType* asType() + { + return nullptr; + } + + template + bool is() const + { + return classIndex == T::ClassIndex(); + } + template + T* as() + { + return classIndex == T::ClassIndex() ? static_cast(this) : nullptr; + } + template + const T* as() const + { + return classIndex == T::ClassIndex() ? static_cast(this) : nullptr; + } + + const int classIndex; + Location location; +}; + +class AstExpr : public AstNode +{ +public: + explicit AstExpr(int classIndex, const Location& location) + : AstNode(classIndex, location) + { + } + + AstExpr* asExpr() override + { + return this; + } +}; + +class AstStat : public AstNode +{ +public: + explicit AstStat(int classIndex, const Location& location) + : AstNode(classIndex, location) + , hasSemicolon(false) + { + } + + AstStat* asStat() override + { + return this; + } + + bool hasSemicolon; +}; + +class AstExprGroup : public AstExpr +{ +public: + lluz_RTTI(AstExprGroup) + + explicit AstExprGroup(const Location& location, AstExpr* expr); + + void visit(AstVisitor* visitor) override; + + AstExpr* expr; +}; + +class AstExprConstantNil : public AstExpr +{ +public: + lluz_RTTI(AstExprConstantNil) + + explicit AstExprConstantNil(const Location& location); + + void visit(AstVisitor* visitor) override; +}; + +class AstExprConstantBool : public AstExpr +{ +public: + lluz_RTTI(AstExprConstantBool) + + AstExprConstantBool(const Location& location, bool value); + + void visit(AstVisitor* visitor) override; + + bool value; +}; + +class AstExprConstantNumber : public AstExpr +{ +public: + lluz_RTTI(AstExprConstantNumber) + + AstExprConstantNumber(const Location& location, double value); + + void visit(AstVisitor* visitor) override; + + double value; +}; + +class AstExprConstantString : public AstExpr +{ +public: + lluz_RTTI(AstExprConstantString) + + AstExprConstantString(const Location& location, const AstArray& value); + + void visit(AstVisitor* visitor) override; + + AstArray value; +}; + +class AstExprLocal : public AstExpr +{ +public: + lluz_RTTI(AstExprLocal) + + AstExprLocal(const Location& location, AstLocal* local, bool upvalue); + + void visit(AstVisitor* visitor) override; + + AstLocal* local; + bool upvalue; +}; + +class AstExprGlobal : public AstExpr +{ +public: + lluz_RTTI(AstExprGlobal) + + AstExprGlobal(const Location& location, const AstName& name); + + void visit(AstVisitor* visitor) override; + + AstName name; +}; + +class AstExprVarargs : public AstExpr +{ +public: + lluz_RTTI(AstExprVarargs) + + AstExprVarargs(const Location& location); + + void visit(AstVisitor* visitor) override; +}; + +class AstExprCall : public AstExpr +{ +public: + lluz_RTTI(AstExprCall) + + AstExprCall(const Location& location, AstExpr* func, const AstArray& args, bool self, const Location& argLocation); + + void visit(AstVisitor* visitor) override; + + AstExpr* func; + AstArray args; + bool self; + Location argLocation; +}; + +class AstExprIndexName : public AstExpr +{ +public: + lluz_RTTI(AstExprIndexName) + + AstExprIndexName( + const Location& location, AstExpr* expr, const AstName& index, const Location& indexLocation, const Position& opPosition, char op); + + void visit(AstVisitor* visitor) override; + + AstExpr* expr; + AstName index; + Location indexLocation; + Position opPosition; + char op = '.'; +}; + +class AstExprIndexExpr : public AstExpr +{ +public: + lluz_RTTI(AstExprIndexExpr) + + AstExprIndexExpr(const Location& location, AstExpr* expr, AstExpr* index); + + void visit(AstVisitor* visitor) override; + + AstExpr* expr; + AstExpr* index; +}; + +class AstExprFunction : public AstExpr +{ +public: + lluz_RTTI(AstExprFunction) + + AstExprFunction(const Location& location, const AstArray& generics, const AstArray& genericPacks, + AstLocal* self, const AstArray& args, std::optional vararg, AstStatBlock* body, size_t functionDepth, + const AstName& debugname, std::optional returnAnnotation = {}, AstTypePack* varargAnnotation = nullptr, bool hasEnd = false, + std::optional argLocation = std::nullopt); + + void visit(AstVisitor* visitor) override; + + AstArray generics; + AstArray genericPacks; + AstLocal* self; + AstArray args; + std::optional returnAnnotation; + bool vararg = false; + Location varargLocation; + AstTypePack* varargAnnotation; + + AstStatBlock* body; + + size_t functionDepth; + + AstName debugname; + + bool hasEnd = false; + std::optional argLocation; +}; + +class AstExprTable : public AstExpr +{ +public: + lluz_RTTI(AstExprTable) + + struct Item + { + enum Kind + { + List, // foo, in which case key is a nullptr + Record, // foo=bar, in which case key is a AstExprConstantString + General, // [foo]=bar + }; + + Kind kind; + + AstExpr* key; // can be nullptr! + AstExpr* value; + }; + + AstExprTable(const Location& location, const AstArray& items); + + void visit(AstVisitor* visitor) override; + + AstArray items; +}; + +class AstExprUnary : public AstExpr +{ +public: + lluz_RTTI(AstExprUnary) + + enum Op + { + Not, + Minus, + Len + }; + + AstExprUnary(const Location& location, Op op, AstExpr* expr); + + void visit(AstVisitor* visitor) override; + + Op op; + AstExpr* expr; +}; + +std::string toString(AstExprUnary::Op op); + +class AstExprBinary : public AstExpr +{ +public: + lluz_RTTI(AstExprBinary) + + enum Op + { + Add, + Sub, + Mul, + Div, + Mod, + Pow, + Concat, + CompareNe, + CompareEq, + CompareLt, + CompareLe, + CompareGt, + CompareGe, + And, + Or + }; + + AstExprBinary(const Location& location, Op op, AstExpr* left, AstExpr* right); + + void visit(AstVisitor* visitor) override; + + Op op; + AstExpr* left; + AstExpr* right; +}; + +std::string toString(AstExprBinary::Op op); + +class AstExprTypeAssertion : public AstExpr +{ +public: + lluz_RTTI(AstExprTypeAssertion) + + AstExprTypeAssertion(const Location& location, AstExpr* expr, AstType* annotation); + + void visit(AstVisitor* visitor) override; + + AstExpr* expr; + AstType* annotation; +}; + +class AstExprIfElse : public AstExpr +{ +public: + lluz_RTTI(AstExprIfElse) + + AstExprIfElse(const Location& location, AstExpr* condition, bool hasThen, AstExpr* trueExpr, bool hasElse, AstExpr* falseExpr); + + void visit(AstVisitor* visitor) override; + + AstExpr* condition; + bool hasThen; + AstExpr* trueExpr; + bool hasElse; + AstExpr* falseExpr; +}; + +class AstStatBlock : public AstStat +{ +public: + lluz_RTTI(AstStatBlock) + + AstStatBlock(const Location& location, const AstArray& body); + + void visit(AstVisitor* visitor) override; + + AstArray body; +}; + +class AstStatIf : public AstStat +{ +public: + lluz_RTTI(AstStatIf) + + AstStatIf(const Location& location, AstExpr* condition, AstStatBlock* thenbody, AstStat* elsebody, const std::optional& thenLocation, + const std::optional& elseLocation, bool hasEnd); + + void visit(AstVisitor* visitor) override; + + AstExpr* condition; + AstStatBlock* thenbody; + AstStat* elsebody; + + std::optional thenLocation; + + // Active for 'elseif' as well + std::optional elseLocation; + + bool hasEnd = false; +}; + +class AstStatWhile : public AstStat +{ +public: + lluz_RTTI(AstStatWhile) + + AstStatWhile(const Location& location, AstExpr* condition, AstStatBlock* body, bool hasDo, const Location& doLocation, bool hasEnd); + + void visit(AstVisitor* visitor) override; + + AstExpr* condition; + AstStatBlock* body; + + bool hasDo = false; + Location doLocation; + + bool hasEnd = false; +}; + +class AstStatRepeat : public AstStat +{ +public: + lluz_RTTI(AstStatRepeat) + + AstStatRepeat(const Location& location, AstExpr* condition, AstStatBlock* body, bool hasUntil); + + void visit(AstVisitor* visitor) override; + + AstExpr* condition; + AstStatBlock* body; + + bool hasUntil = false; +}; + +class AstStatBreak : public AstStat +{ +public: + lluz_RTTI(AstStatBreak) + + AstStatBreak(const Location& location); + + void visit(AstVisitor* visitor) override; +}; + +class AstStatContinue : public AstStat +{ +public: + lluz_RTTI(AstStatContinue) + + AstStatContinue(const Location& location); + + void visit(AstVisitor* visitor) override; +}; + +class AstStatReturn : public AstStat +{ +public: + lluz_RTTI(AstStatReturn) + + AstStatReturn(const Location& location, const AstArray& list); + + void visit(AstVisitor* visitor) override; + + AstArray list; +}; + +class AstStatExpr : public AstStat +{ +public: + lluz_RTTI(AstStatExpr) + + AstStatExpr(const Location& location, AstExpr* expr); + + void visit(AstVisitor* visitor) override; + + AstExpr* expr; +}; + +class AstStatLocal : public AstStat +{ +public: + lluz_RTTI(AstStatLocal) + + AstStatLocal(const Location& location, const AstArray& vars, const AstArray& values, + const std::optional& equalsSignLocation); + + void visit(AstVisitor* visitor) override; + + AstArray vars; + AstArray values; + + std::optional equalsSignLocation; +}; + +class AstStatFor : public AstStat +{ +public: + lluz_RTTI(AstStatFor) + + AstStatFor(const Location& location, AstLocal* var, AstExpr* from, AstExpr* to, AstExpr* step, AstStatBlock* body, bool hasDo, + const Location& doLocation, bool hasEnd); + + void visit(AstVisitor* visitor) override; + + AstLocal* var; + AstExpr* from; + AstExpr* to; + AstExpr* step; + AstStatBlock* body; + + bool hasDo = false; + Location doLocation; + + bool hasEnd = false; +}; + +class AstStatForIn : public AstStat +{ +public: + lluz_RTTI(AstStatForIn) + + AstStatForIn(const Location& location, const AstArray& vars, const AstArray& values, AstStatBlock* body, bool hasIn, + const Location& inLocation, bool hasDo, const Location& doLocation, bool hasEnd); + + void visit(AstVisitor* visitor) override; + + AstArray vars; + AstArray values; + AstStatBlock* body; + + bool hasIn = false; + Location inLocation; + + bool hasDo = false; + Location doLocation; + + bool hasEnd = false; +}; + +class AstStatAssign : public AstStat +{ +public: + lluz_RTTI(AstStatAssign) + + AstStatAssign(const Location& location, const AstArray& vars, const AstArray& values); + + void visit(AstVisitor* visitor) override; + + AstArray vars; + AstArray values; +}; + +class AstStatCompoundAssign : public AstStat +{ +public: + lluz_RTTI(AstStatCompoundAssign) + + AstStatCompoundAssign(const Location& location, AstExprBinary::Op op, AstExpr* var, AstExpr* value); + + void visit(AstVisitor* visitor) override; + + AstExprBinary::Op op; + AstExpr* var; + AstExpr* value; +}; + +class AstStatFunction : public AstStat +{ +public: + lluz_RTTI(AstStatFunction) + + AstStatFunction(const Location& location, AstExpr* name, AstExprFunction* func); + + void visit(AstVisitor* visitor) override; + + AstExpr* name; + AstExprFunction* func; +}; + +class AstStatLocalFunction : public AstStat +{ +public: + lluz_RTTI(AstStatLocalFunction) + + AstStatLocalFunction(const Location& location, AstLocal* name, AstExprFunction* func); + + void visit(AstVisitor* visitor) override; + + AstLocal* name; + AstExprFunction* func; +}; + +class AstStatTypeAlias : public AstStat +{ +public: + lluz_RTTI(AstStatTypeAlias) + + AstStatTypeAlias(const Location& location, const AstName& name, const AstArray& generics, + const AstArray& genericPacks, AstType* type, bool exported); + + void visit(AstVisitor* visitor) override; + + AstName name; + AstArray generics; + AstArray genericPacks; + AstType* type; + bool exported; +}; + +class AstStatDeclareGlobal : public AstStat +{ +public: + lluz_RTTI(AstStatDeclareGlobal) + + AstStatDeclareGlobal(const Location& location, const AstName& name, AstType* type); + + void visit(AstVisitor* visitor) override; + + AstName name; + AstType* type; +}; + +class AstStatDeclareFunction : public AstStat +{ +public: + lluz_RTTI(AstStatDeclareFunction) + + AstStatDeclareFunction(const Location& location, const AstName& name, const AstArray& generics, + const AstArray& genericPacks, const AstTypeList& params, const AstArray& paramNames, + const AstTypeList& retTypes); + + void visit(AstVisitor* visitor) override; + + AstName name; + AstArray generics; + AstArray genericPacks; + AstTypeList params; + AstArray paramNames; + AstTypeList retTypes; +}; + +struct AstDeclaredClassProp +{ + AstName name; + AstType* ty = nullptr; + bool isMethod = false; +}; + +class AstStatDeclareClass : public AstStat +{ +public: + lluz_RTTI(AstStatDeclareClass) + + AstStatDeclareClass(const Location& location, const AstName& name, std::optional superName, const AstArray& props); + + void visit(AstVisitor* visitor) override; + + AstName name; + std::optional superName; + + AstArray props; +}; + +class AstType : public AstNode +{ +public: + AstType(int classIndex, const Location& location) + : AstNode(classIndex, location) + { + } + + AstType* asType() override + { + return this; + } +}; + +// Don't have lluz::Variant available, it's a bit of an overhead, but a plain struct is nice to use +struct AstTypeOrPack +{ + AstType* type = nullptr; + AstTypePack* typePack = nullptr; +}; + +class AstTypeReference : public AstType +{ +public: + lluz_RTTI(AstTypeReference) + + AstTypeReference(const Location& location, std::optional prefix, AstName name, bool hasParameterList = false, + const AstArray& parameters = {}); + + void visit(AstVisitor* visitor) override; + + bool hasParameterList; + std::optional prefix; + AstName name; + AstArray parameters; +}; + +struct AstTableProp +{ + AstName name; + Location location; + AstType* type; +}; + +struct AstTableIndexer +{ + AstType* indexType; + AstType* resultType; + Location location; +}; + +class AstTypeTable : public AstType +{ +public: + lluz_RTTI(AstTypeTable) + + AstTypeTable(const Location& location, const AstArray& props, AstTableIndexer* indexer = nullptr); + + void visit(AstVisitor* visitor) override; + + AstArray props; + AstTableIndexer* indexer; +}; + +class AstTypeFunction : public AstType +{ +public: + lluz_RTTI(AstTypeFunction) + + AstTypeFunction(const Location& location, const AstArray& generics, const AstArray& genericPacks, + const AstTypeList& argTypes, const AstArray>& argNames, const AstTypeList& returnTypes); + + void visit(AstVisitor* visitor) override; + + AstArray generics; + AstArray genericPacks; + AstTypeList argTypes; + AstArray> argNames; + AstTypeList returnTypes; +}; + +class AstTypeTypeof : public AstType +{ +public: + lluz_RTTI(AstTypeTypeof) + + AstTypeTypeof(const Location& location, AstExpr* expr); + + void visit(AstVisitor* visitor) override; + + AstExpr* expr; +}; + +class AstTypeUnion : public AstType +{ +public: + lluz_RTTI(AstTypeUnion) + + AstTypeUnion(const Location& location, const AstArray& types); + + void visit(AstVisitor* visitor) override; + + AstArray types; +}; + +class AstTypeIntersection : public AstType +{ +public: + lluz_RTTI(AstTypeIntersection) + + AstTypeIntersection(const Location& location, const AstArray& types); + + void visit(AstVisitor* visitor) override; + + AstArray types; +}; + +class AstExprError : public AstExpr +{ +public: + lluz_RTTI(AstExprError) + + AstExprError(const Location& location, const AstArray& expressions, unsigned messageIndex); + + void visit(AstVisitor* visitor) override; + + AstArray expressions; + unsigned messageIndex; +}; + +class AstStatError : public AstStat +{ +public: + lluz_RTTI(AstStatError) + + AstStatError(const Location& location, const AstArray& expressions, const AstArray& statements, unsigned messageIndex); + + void visit(AstVisitor* visitor) override; + + AstArray expressions; + AstArray statements; + unsigned messageIndex; +}; + +class AstTypeError : public AstType +{ +public: + lluz_RTTI(AstTypeError) + + AstTypeError(const Location& location, const AstArray& types, bool isMissing, unsigned messageIndex); + + void visit(AstVisitor* visitor) override; + + AstArray types; + bool isMissing; + unsigned messageIndex; +}; + +class AstTypeSingletonBool : public AstType +{ +public: + lluz_RTTI(AstTypeSingletonBool) + + AstTypeSingletonBool(const Location& location, bool value); + + void visit(AstVisitor* visitor) override; + + bool value; +}; + +class AstTypeSingletonString : public AstType +{ +public: + lluz_RTTI(AstTypeSingletonString) + + AstTypeSingletonString(const Location& location, const AstArray& value); + + void visit(AstVisitor* visitor) override; + + const AstArray value; +}; + +class AstTypePack : public AstNode +{ +public: + AstTypePack(int classIndex, const Location& location) + : AstNode(classIndex, location) + { + } +}; + +class AstTypePackExplicit : public AstTypePack +{ +public: + lluz_RTTI(AstTypePackExplicit) + + AstTypePackExplicit(const Location& location, AstTypeList typeList); + + void visit(AstVisitor* visitor) override; + + AstTypeList typeList; +}; + +class AstTypePackVariadic : public AstTypePack +{ +public: + lluz_RTTI(AstTypePackVariadic) + + AstTypePackVariadic(const Location& location, AstType* variadicType); + + void visit(AstVisitor* visitor) override; + + AstType* variadicType; +}; + +class AstTypePackGeneric : public AstTypePack +{ +public: + lluz_RTTI(AstTypePackGeneric) + + AstTypePackGeneric(const Location& location, AstName name); + + void visit(AstVisitor* visitor) override; + + AstName genericName; +}; + +AstName getIdentifier(AstExpr*); + +#undef lluz_RTTI + +} // namespace lluz + +namespace std +{ + +template<> +struct hash +{ + size_t operator()(const lluz::AstName& value) const + { + // note: since operator== uses pointer identity, hashing function uses it as well + // the hasher is the same as DenseHashPointer (DenseHash.h) + return (uintptr_t(value.value) >> 4) ^ (uintptr_t(value.value) >> 9); + } +}; + +} // namespace std diff --git a/Ast/include/lluz/Confusables.h b/Ast/include/lluz/Confusables.h new file mode 100644 index 00000000..c1a20656 --- /dev/null +++ b/Ast/include/lluz/Confusables.h @@ -0,0 +1,9 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include + +namespace lluz +{ +const char* findConfusable(uint32_t codepoint); +} diff --git a/Ast/include/lluz/DenseHash.h b/Ast/include/lluz/DenseHash.h new file mode 100644 index 00000000..b9d40016 --- /dev/null +++ b/Ast/include/lluz/DenseHash.h @@ -0,0 +1,524 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "lluz/Common.h" + +#include +#include +#include +#include +#include + +namespace lluz +{ + +struct DenseHashPointer +{ + size_t operator()(const void* key) const + { + return (uintptr_t(key) >> 4) ^ (uintptr_t(key) >> 9); + } +}; + +// Internal implementation of DenseHashSet and DenseHashMap +namespace detail +{ + +template +using DenseHashDefault = std::conditional_t, DenseHashPointer, std::hash>; + +template +class DenseHashTable +{ +public: + class const_iterator; + class iterator; + + DenseHashTable(const Key& empty_key, size_t buckets = 0) + : count(0) + , empty_key(empty_key) + { + // buckets has to be power-of-two or zero + lluz_ASSERT((buckets & (buckets - 1)) == 0); + + // don't move this to initializer list! this works around an MSVC codegen issue on AMD CPUs: + // https://developercommunity.visualstudio.com/t/stdvector-constructor-from-size-t-is-25-times-slow/1546547 + if (buckets) + resize_data(buckets); + } + + void clear() + { + data.clear(); + count = 0; + } + + Item* insert_unsafe(const Key& key) + { + // It is invalid to insert empty_key into the table since it acts as a "entry does not exist" marker + lluz_ASSERT(!eq(key, empty_key)); + + size_t hashmod = data.size() - 1; + size_t bucket = hasher(key) & hashmod; + + for (size_t probe = 0; probe <= hashmod; ++probe) + { + Item& probe_item = data[bucket]; + + // Element does not exist, insert here + if (eq(ItemInterface::getKey(probe_item), empty_key)) + { + ItemInterface::setKey(probe_item, key); + count++; + return &probe_item; + } + + // Element already exists + if (eq(ItemInterface::getKey(probe_item), key)) + { + return &probe_item; + } + + // Hash collision, quadratic probing + bucket = (bucket + probe + 1) & hashmod; + } + + // Hash table is full - this should not happen + lluz_ASSERT(false); + return NULL; + } + + const Item* find(const Key& key) const + { + if (data.empty()) + return 0; + if (eq(key, empty_key)) + return 0; + + size_t hashmod = data.size() - 1; + size_t bucket = hasher(key) & hashmod; + + for (size_t probe = 0; probe <= hashmod; ++probe) + { + const Item& probe_item = data[bucket]; + + // Element exists + if (eq(ItemInterface::getKey(probe_item), key)) + return &probe_item; + + // Element does not exist + if (eq(ItemInterface::getKey(probe_item), empty_key)) + return NULL; + + // Hash collision, quadratic probing + bucket = (bucket + probe + 1) & hashmod; + } + + // Hash table is full - this should not happen + lluz_ASSERT(false); + return NULL; + } + + void rehash() + { + size_t newsize = data.empty() ? 16 : data.size() * 2; + + if (data.empty() && data.capacity() >= newsize) + { + lluz_ASSERT(count == 0); + resize_data(newsize); + return; + } + + DenseHashTable newtable(empty_key, newsize); + + for (size_t i = 0; i < data.size(); ++i) + { + const Key& key = ItemInterface::getKey(data[i]); + + if (!eq(key, empty_key)) + { + Item* item = newtable.insert_unsafe(key); + *item = std::move(data[i]); + } + } + + lluz_ASSERT(count == newtable.count); + data.swap(newtable.data); + } + + void rehash_if_full() + { + if (count >= data.size() * 3 / 4) + { + rehash(); + } + } + + const_iterator begin() const + { + size_t start = 0; + + while (start < data.size() && eq(ItemInterface::getKey(data[start]), empty_key)) + start++; + + return const_iterator(this, start); + } + + const_iterator end() const + { + return const_iterator(this, data.size()); + } + + iterator begin() + { + size_t start = 0; + + while (start < data.size() && eq(ItemInterface::getKey(data[start]), empty_key)) + start++; + + return iterator(this, start); + } + + iterator end() + { + return iterator(this, data.size()); + } + + size_t size() const + { + return count; + } + + class const_iterator + { + public: + const_iterator() + : set(0) + , index(0) + { + } + + const_iterator(const DenseHashTable* set, size_t index) + : set(set) + , index(index) + { + } + + const Item& operator*() const + { + return set->data[index]; + } + + const Item* operator->() const + { + return &set->data[index]; + } + + bool operator==(const const_iterator& other) const + { + return set == other.set && index == other.index; + } + + bool operator!=(const const_iterator& other) const + { + return set != other.set || index != other.index; + } + + const_iterator& operator++() + { + size_t size = set->data.size(); + + do + { + index++; + } while (index < size && set->eq(ItemInterface::getKey(set->data[index]), set->empty_key)); + + return *this; + } + + const_iterator operator++(int) + { + const_iterator res = *this; + ++*this; + return res; + } + + private: + const DenseHashTable* set; + size_t index; + }; + + class iterator + { + public: + iterator() + : set(0) + , index(0) + { + } + + iterator(DenseHashTable* set, size_t index) + : set(set) + , index(index) + { + } + + MutableItem& operator*() const + { + return *reinterpret_cast(&set->data[index]); + } + + MutableItem* operator->() const + { + return reinterpret_cast(&set->data[index]); + } + + bool operator==(const iterator& other) const + { + return set == other.set && index == other.index; + } + + bool operator!=(const iterator& other) const + { + return set != other.set || index != other.index; + } + + iterator& operator++() + { + size_t size = set->data.size(); + + do + { + index++; + } while (index < size && set->eq(ItemInterface::getKey(set->data[index]), set->empty_key)); + + return *this; + } + + iterator operator++(int) + { + iterator res = *this; + ++*this; + return res; + } + + private: + DenseHashTable* set; + size_t index; + }; + +private: + template + void resize_data(size_t count, typename std::enable_if_t>* dummy = nullptr) + { + data.resize(count, ItemInterface::create(empty_key)); + } + + template + void resize_data(size_t count, typename std::enable_if_t>* dummy = nullptr) + { + size_t size = data.size(); + data.resize(count); + + for (size_t i = size; i < count; i++) + data[i].first = empty_key; + } + + std::vector data; + size_t count; + Key empty_key; + Hash hasher; + Eq eq; +}; + +template +struct ItemInterfaceSet +{ + static const Key& getKey(const Key& item) + { + return item; + } + + static void setKey(Key& item, const Key& key) + { + item = key; + } + + static Key create(const Key& key) + { + return key; + } +}; + +template +struct ItemInterfaceMap +{ + static const Key& getKey(const std::pair& item) + { + return item.first; + } + + static void setKey(std::pair& item, const Key& key) + { + item.first = key; + } + + static std::pair create(const Key& key) + { + return std::pair(key, Value()); + } +}; + +} // namespace detail + +// This is a faster alternative of unordered_set, but it does not implement the same interface (i.e. it does not support erasing) +template, typename Eq = std::equal_to> +class DenseHashSet +{ + typedef detail::DenseHashTable, Hash, Eq> Impl; + Impl impl; + +public: + typedef typename Impl::const_iterator const_iterator; + typedef typename Impl::iterator iterator; + + DenseHashSet(const Key& empty_key, size_t buckets = 0) + : impl(empty_key, buckets) + { + } + + void clear() + { + impl.clear(); + } + + const Key& insert(const Key& key) + { + impl.rehash_if_full(); + return *impl.insert_unsafe(key); + } + + const Key* find(const Key& key) const + { + return impl.find(key); + } + + bool contains(const Key& key) const + { + return impl.find(key) != 0; + } + + size_t size() const + { + return impl.size(); + } + + bool empty() const + { + return impl.size() == 0; + } + + const_iterator begin() const + { + return impl.begin(); + } + + const_iterator end() const + { + return impl.end(); + } + + iterator begin() + { + return impl.begin(); + } + + iterator end() + { + return impl.end(); + } +}; + +// This is a faster alternative of unordered_map, but it does not implement the same interface (i.e. it does not support erasing and has +// contains() instead of find()) +template, typename Eq = std::equal_to> +class DenseHashMap +{ + typedef detail::DenseHashTable, std::pair, detail::ItemInterfaceMap, Hash, Eq> Impl; + Impl impl; + +public: + typedef typename Impl::const_iterator const_iterator; + typedef typename Impl::iterator iterator; + + DenseHashMap(const Key& empty_key, size_t buckets = 0) + : impl(empty_key, buckets) + { + } + + void clear() + { + impl.clear(); + } + + // Note: this reference is invalidated by any insert operation (i.e. operator[]) + Value& operator[](const Key& key) + { + impl.rehash_if_full(); + return impl.insert_unsafe(key)->second; + } + + // Note: this pointer is invalidated by any insert operation (i.e. operator[]) + const Value* find(const Key& key) const + { + const std::pair* result = impl.find(key); + + return result ? &result->second : NULL; + } + + // Note: this pointer is invalidated by any insert operation (i.e. operator[]) + Value* find(const Key& key) + { + const std::pair* result = impl.find(key); + + return result ? const_cast(&result->second) : NULL; + } + + bool contains(const Key& key) const + { + return impl.find(key) != 0; + } + + size_t size() const + { + return impl.size(); + } + + bool empty() const + { + return impl.size() == 0; + } + + const_iterator begin() const + { + return impl.begin(); + } + + const_iterator end() const + { + return impl.end(); + } + + iterator begin() + { + return impl.begin(); + } + + iterator end() + { + return impl.end(); + } +}; + +} // namespace lluz diff --git a/Ast/include/lluz/Lexer.h b/Ast/include/lluz/Lexer.h new file mode 100644 index 00000000..36e09a12 --- /dev/null +++ b/Ast/include/lluz/Lexer.h @@ -0,0 +1,241 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "lluz/Ast.h" +#include "lluz/Location.h" +#include "lluz/DenseHash.h" +#include "lluz/Common.h" + +namespace lluz +{ + +class Allocator +{ +public: + Allocator(); + Allocator(Allocator&&); + + Allocator& operator=(Allocator&&) = delete; + + ~Allocator(); + + void* allocate(size_t size); + + template + T* alloc(Args&&... args) + { + static_assert(std::is_trivially_destructible::value, "Objects allocated with this allocator will never have their destructors run!"); + + T* t = static_cast(allocate(sizeof(T))); + new (t) T(std::forward(args)...); + return t; + } + +private: + struct Page + { + Page* next; + + char data[8192]; + }; + + Page* root; + size_t offset; +}; + +struct Lexeme +{ + enum Type + { + Eof = 0, + + // 1..255 means actual character values + Char_END = 256, + + Equal, + LessEqual, + GreaterEqual, + NotEqual, + Dot2, + Dot3, + SkinnyArrow, + DoubleColon, + + AddAssign, + SubAssign, + MulAssign, + DivAssign, + ModAssign, + PowAssign, + ConcatAssign, + + RawString, + QuotedString, + Number, + Name, + + Comment, + BlockComment, + + BrokenString, + BrokenComment, + BrokenUnicode, + Error, + + Reserved_BEGIN, + ReservedAnd = Reserved_BEGIN, + ReservedBreak, + ReservedDo, + ReservedElse, + ReservedElseif, + ReservedEnd, + ReservedFalse, + ReservedFor, + ReservedFunction, + ReservedIf, + ReservedIn, + ReservedLocal, + ReservedNil, + ReservedNot, + ReservedOr, + ReservedRepeat, + ReservedReturn, + ReservedThen, + ReservedTrue, + ReservedUntil, + ReservedWhile, + Reserved_END + }; + + Type type; + Location location; + unsigned int length; + + union + { + const char* data; // String, Number, Comment + const char* name; // Name + unsigned int codepoint; // BrokenUnicode + }; + + Lexeme(const Location& location, Type type); + Lexeme(const Location& location, char character); + Lexeme(const Location& location, Type type, const char* data, size_t size); + Lexeme(const Location& location, Type type, const char* name); + + std::string toString() const; +}; + +class AstNameTable +{ +public: + AstNameTable(Allocator& allocator); + + AstName addStatic(const char* name, Lexeme::Type type = Lexeme::Name); + + std::pair getOrAddWithType(const char* name, size_t length); + std::pair getWithType(const char* name, size_t length) const; + + AstName getOrAdd(const char* name); + AstName get(const char* name) const; + +private: + struct Entry + { + AstName value; + uint32_t length; + Lexeme::Type type; + + bool operator==(const Entry& other) const; + }; + + struct EntryHash + { + size_t operator()(const Entry& e) const; + }; + + DenseHashSet data; + + Allocator& allocator; +}; + +class Lexer +{ +public: + Lexer(const char* buffer, std::size_t bufferSize, AstNameTable& names); + + void setSkipComments(bool skip); + void setReadNames(bool read); + + const Location& previousLocation() const + { + return prevLocation; + } + + const Lexeme& next(); + const Lexeme& next(bool skipComments, bool updatePrevLocation); + void nextline(); + + Lexeme lookahead(); + + const Lexeme& current() const + { + return lexeme; + } + + static bool isReserved(const std::string& word); + + static bool fixupQuotedString(std::string& data); + static void fixupMultilineString(std::string& data); + +private: + char peekch() const; + char peekch(unsigned int lookahead) const; + + Position position() const; + + void consume(); + + Lexeme readCommentBody(); + + // Given a sequence [===[ or ]===], returns: + // 1. number of equal signs (or 0 if none present) between the brackets + // 2. -1 if this is not a long comment/string separator + // 3. -N if this is a malformed separator + // Does *not* consume the closing brace. + int skipLongSeparator(); + + Lexeme readLongString(const Position& start, int sep, Lexeme::Type ok, Lexeme::Type broken); + Lexeme readQuotedString(); + + std::pair readName(); + + Lexeme readNumber(const Position& start, unsigned int startOffset); + + Lexeme readUtf8Error(); + Lexeme readNext(); + + const char* buffer; + std::size_t bufferSize; + + unsigned int offset; + + unsigned int line; + unsigned int lineOffset; + + Lexeme lexeme; + + Location prevLocation; + + AstNameTable& names; + + bool skipComments; + bool readNames; +}; + +inline bool isSpace(char ch) +{ + return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\v' || ch == '\f'; +} + +} // namespace lluz diff --git a/Ast/include/lluz/Location.h b/Ast/include/lluz/Location.h new file mode 100644 index 00000000..9263b8cf --- /dev/null +++ b/Ast/include/lluz/Location.h @@ -0,0 +1,109 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include + +namespace lluz +{ + +struct Position +{ + unsigned int line, column; + + Position(unsigned int line, unsigned int column) + : line(line) + , column(column) + { + } + + bool operator==(const Position& rhs) const + { + return this->column == rhs.column && this->line == rhs.line; + } + bool operator!=(const Position& rhs) const + { + return !(*this == rhs); + } + + bool operator<(const Position& rhs) const + { + if (line == rhs.line) + return column < rhs.column; + else + return line < rhs.line; + } + + bool operator>(const Position& rhs) const + { + if (line == rhs.line) + return column > rhs.column; + else + return line > rhs.line; + } + + bool operator<=(const Position& rhs) const + { + return *this == rhs || *this < rhs; + } + + bool operator>=(const Position& rhs) const + { + return *this == rhs || *this > rhs; + } +}; + +struct Location +{ + Position begin, end; + + Location() + : begin(0, 0) + , end(0, 0) + { + } + + Location(const Position& begin, const Position& end) + : begin(begin) + , end(end) + { + } + + Location(const Position& begin, unsigned int length) + : begin(begin) + , end(begin.line, begin.column + length) + { + } + + Location(const Location& begin, const Location& end) + : begin(begin.begin) + , end(end.end) + { + } + + bool operator==(const Location& rhs) const + { + return this->begin == rhs.begin && this->end == rhs.end; + } + bool operator!=(const Location& rhs) const + { + return !(*this == rhs); + } + + bool encloses(const Location& l) const + { + return begin <= l.begin && end >= l.end; + } + bool contains(const Position& p) const + { + return begin <= p && p < end; + } + bool containsClosed(const Position& p) const + { + return begin <= p && p <= end; + } +}; + +std::string toString(const Position& position); +std::string toString(const Location& location); + +} // namespace lluz diff --git a/Ast/include/lluz/ParseOptions.h b/Ast/include/lluz/ParseOptions.h new file mode 100644 index 00000000..7f4c52d2 --- /dev/null +++ b/Ast/include/lluz/ParseOptions.h @@ -0,0 +1,23 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +namespace lluz +{ + +enum class Mode +{ + NoCheck, // Do not perform any inference + Nonstrict, // Unannotated symbols are any + Strict, // Unannotated symbols are inferred + Definition, // Type definition module, has special parsing rules +}; + +struct ParseOptions +{ + bool allowTypeAnnotations = true; + bool supportContinueStatement = true; + bool allowDeclarationSyntax = false; + bool captureComments = false; +}; + +} // namespace lluz diff --git a/Ast/include/lluz/ParseResult.h b/Ast/include/lluz/ParseResult.h new file mode 100644 index 00000000..b75597f5 --- /dev/null +++ b/Ast/include/lluz/ParseResult.h @@ -0,0 +1,69 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "lluz/Common.h" +#include "lluz/Location.h" +#include "lluz/Lexer.h" +#include "lluz/StringUtils.h" + +namespace lluz +{ + +class AstStatBlock; + +class ParseError : public std::exception +{ +public: + ParseError(const Location& location, const std::string& message); + + virtual const char* what() const throw(); + + const Location& getLocation() const; + const std::string& getMessage() const; + + static lluz_NORETURN void raise(const Location& location, const char* format, ...) lluz_PRINTF_ATTR(2, 3); + +private: + Location location; + std::string message; +}; + +class ParseErrors : public std::exception +{ +public: + ParseErrors(std::vector errors); + + virtual const char* what() const throw(); + + const std::vector& getErrors() const; + +private: + std::vector errors; + std::string message; +}; + +struct HotComment +{ + bool header; + Location location; + std::string content; +}; + +struct Comment +{ + Lexeme::Type type; // Comment, BlockComment, or BrokenComment + Location location; +}; + +struct ParseResult +{ + AstStatBlock* root; + std::vector hotcomments; + std::vector errors; + + std::vector commentLocations; +}; + +static constexpr const char* kParseNameError = "%error-id%"; + +} // namespace lluz diff --git a/Ast/include/lluz/Parser.h b/Ast/include/lluz/Parser.h new file mode 100644 index 00000000..8ca97543 --- /dev/null +++ b/Ast/include/lluz/Parser.h @@ -0,0 +1,383 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "lluz/Ast.h" +#include "lluz/Lexer.h" +#include "lluz/ParseOptions.h" +#include "lluz/ParseResult.h" +#include "lluz/StringUtils.h" +#include "lluz/DenseHash.h" +#include "lluz/Common.h" + +#include +#include + +namespace lluz +{ + +template +class TempVector +{ +public: + explicit TempVector(std::vector& storage); + + ~TempVector(); + + const T& operator[](std::size_t index) const; + + const T& front() const; + + const T& back() const; + + bool empty() const; + + std::size_t size() const; + + void push_back(const T& item); + + typename std::vector::const_iterator begin() const + { + return storage.begin() + offset; + } + typename std::vector::const_iterator end() const + { + return storage.begin() + offset + size_; + } + +private: + std::vector& storage; + size_t offset; + size_t size_; +}; + +class Parser +{ +public: + static ParseResult parse( + const char* buffer, std::size_t bufferSize, AstNameTable& names, Allocator& allocator, ParseOptions options = ParseOptions()); + +private: + struct Name; + struct Binding; + + Parser(const char* buffer, std::size_t bufferSize, AstNameTable& names, Allocator& allocator, const ParseOptions& options); + + bool blockFollow(const Lexeme& l); + + AstStatBlock* parseChunk(); + + // chunk ::= {stat [`;']} [laststat [`;']] + // block ::= chunk + AstStatBlock* parseBlock(); + + AstStatBlock* parseBlockNoScope(); + + // stat ::= + // varlist `=' explist | + // functioncall | + // do block end | + // while exp do block end | + // repeat block until exp | + // if exp then block {elseif exp then block} [else block] end | + // for Name `=' exp `,' exp [`,' exp] do block end | + // for namelist in explist do block end | + // function funcname funcbody | + // local function Name funcbody | + // local namelist [`=' explist] + // laststat ::= return [explist] | break + AstStat* parseStat(); + + // if exp then block {elseif exp then block} [else block] end + AstStat* parseIf(); + + // while exp do block end + AstStat* parseWhile(); + + // repeat block until exp + AstStat* parseRepeat(); + + // do block end + AstStat* parseDo(); + + // break + AstStat* parseBreak(); + + // continue + AstStat* parseContinue(const Location& start); + + // for Name `=' exp `,' exp [`,' exp] do block end | + // for namelist in explist do block end | + AstStat* parseFor(); + + // function funcname funcbody | + // funcname ::= Name {`.' Name} [`:' Name] + AstStat* parseFunctionStat(); + + // local function Name funcbody | + // local namelist [`=' explist] + AstStat* parseLocal(); + + // return [explist] + AstStat* parseReturn(); + + // type Name `=' typeannotation + AstStat* parseTypeAlias(const Location& start, bool exported); + + AstDeclaredClassProp parseDeclaredClassMethod(); + + // `declare global' Name: typeannotation | + // `declare function' Name`(' [parlist] `)' [`:` TypeAnnotation] + AstStat* parseDeclaration(const Location& start); + + // varlist `=' explist + AstStat* parseAssignment(AstExpr* initial); + + // var [`+=' | `-=' | `*=' | `/=' | `%=' | `^=' | `..='] exp + AstStat* parseCompoundAssignment(AstExpr* initial, AstExprBinary::Op op); + + // funcbody ::= `(' [parlist] `)' block end + // parlist ::= namelist [`,' `...'] | `...' + std::pair parseFunctionBody( + bool hasself, const Lexeme& matchFunction, const AstName& debugname, std::optional localName); + + // explist ::= {exp `,'} exp + void parseExprList(TempVector& result); + + // binding ::= Name [`:` TypeAnnotation] + Binding parseBinding(); + + // bindinglist ::= (binding | `...') {`,' bindinglist} + // Returns the location of the vararg ..., or std::nullopt if the function is not vararg. + std::pair, AstTypePack*> parseBindingList(TempVector& result, bool allowDot3 = false); + + AstType* parseOptionalTypeAnnotation(); + + // TypeList ::= TypeAnnotation [`,' TypeList] + // ReturnType ::= TypeAnnotation | `(' TypeList `)' + // TableProp ::= Name `:' TypeAnnotation + // TableIndexer ::= `[' TypeAnnotation `]' `:' TypeAnnotation + // PropList ::= (TableProp | TableIndexer) [`,' PropList] + // TypeAnnotation + // ::= Name + // | `nil` + // | `{' [PropList] `}' + // | `(' [TypeList] `)' `->` ReturnType + + // Returns the variadic annotation, if it exists. + AstTypePack* parseTypeList(TempVector& result, TempVector>& resultNames); + + std::optional parseOptionalReturnTypeAnnotation(); + std::pair parseReturnTypeAnnotation(); + + AstTableIndexer* parseTableIndexerAnnotation(); + + AstTypeOrPack parseFunctionTypeAnnotation(bool allowPack); + AstType* parseFunctionTypeAnnotationTail(const Lexeme& begin, AstArray generics, AstArray genericPacks, + AstArray& params, AstArray>& paramNames, AstTypePack* varargAnnotation); + + AstType* parseTableTypeAnnotation(); + AstTypeOrPack parseSimpleTypeAnnotation(bool allowPack); + + AstTypeOrPack parseTypeOrPackAnnotation(); + AstType* parseTypeAnnotation(TempVector& parts, const Location& begin); + AstType* parseTypeAnnotation(); + + AstTypePack* parseTypePackAnnotation(); + AstTypePack* parseVariadicArgumentAnnotation(); + + static std::optional parseUnaryOp(const Lexeme& l); + static std::optional parseBinaryOp(const Lexeme& l); + static std::optional parseCompoundOp(const Lexeme& l); + + struct BinaryOpPriority + { + unsigned char left, right; + }; + + std::optional checkUnaryConfusables(); + std::optional checkBinaryConfusables(const BinaryOpPriority binaryPriority[], unsigned int limit); + + // subexpr -> (asexp | unop subexpr) { binop subexpr } + // where `binop' is any binary operator with a priority higher than `limit' + AstExpr* parseExpr(unsigned int limit = 0); + + // NAME + AstExpr* parseNameExpr(const char* context = nullptr); + + // prefixexp -> NAME | '(' expr ')' + AstExpr* parsePrefixExpr(); + + // primaryexp -> prefixexp { `.' NAME | `[' exp `]' | `:' NAME funcargs | funcargs } + AstExpr* parsePrimaryExpr(bool asStatement); + + // asexp -> simpleexp [`::' typeAnnotation] + AstExpr* parseAssertionExpr(); + + // simpleexp -> NUMBER | STRING | NIL | true | false | ... | constructor | FUNCTION body | primaryexp + AstExpr* parseSimpleExpr(); + + // args ::= `(' [explist] `)' | tableconstructor | String + AstExpr* parseFunctionArgs(AstExpr* func, bool self, const Location& selfLocation); + + // tableconstructor ::= `{' [fieldlist] `}' + // fieldlist ::= field {fieldsep field} [fieldsep] + // field ::= `[' exp `]' `=' exp | Name `=' exp | exp + // fieldsep ::= `,' | `;' + AstExpr* parseTableConstructor(); + + // TODO: Add grammar rules here? + AstExpr* parseIfElseExpr(); + + // Name + std::optional parseNameOpt(const char* context = nullptr); + Name parseName(const char* context = nullptr); + Name parseIndexName(const char* context, const Position& previous); + + // `<' namelist `>' + std::pair, AstArray> parseGenericTypeList(bool withDefaultValues); + + // `<' typeAnnotation[, ...] `>' + AstArray parseTypeParams(); + + std::optional> parseCharArray(); + AstExpr* parseString(); + + AstLocal* pushLocal(const Binding& binding); + + unsigned int saveLocals(); + + void restoreLocals(unsigned int offset); + + // check that parser is at lexeme/symbol, move to next lexeme/symbol on success, report failure and continue on failure + bool expectAndConsume(char value, const char* context = nullptr); + bool expectAndConsume(Lexeme::Type type, const char* context = nullptr); + void expectAndConsumeFail(Lexeme::Type type, const char* context); + + bool expectMatchAndConsume(char value, const Lexeme& begin, bool searchForMissing = false); + void expectMatchAndConsumeFail(Lexeme::Type type, const Lexeme& begin, const char* extra = nullptr); + + bool expectMatchEndAndConsume(Lexeme::Type type, const Lexeme& begin); + void expectMatchEndAndConsumeFail(Lexeme::Type type, const Lexeme& begin); + + template + AstArray copy(const T* data, std::size_t size); + + template + AstArray copy(const TempVector& data); + + template + AstArray copy(std::initializer_list data); + + AstArray copy(const std::string& data); + + void incrementRecursionCounter(const char* context); + + void report(const Location& location, const char* format, va_list args); + void report(const Location& location, const char* format, ...) lluz_PRINTF_ATTR(3, 4); + + void reportNameError(const char* context); + + AstStatError* reportStatError(const Location& location, const AstArray& expressions, const AstArray& statements, + const char* format, ...) lluz_PRINTF_ATTR(5, 6); + AstExprError* reportExprError(const Location& location, const AstArray& expressions, const char* format, ...) lluz_PRINTF_ATTR(4, 5); + AstTypeError* reportTypeAnnotationError(const Location& location, const AstArray& types, bool isMissing, const char* format, ...) + lluz_PRINTF_ATTR(5, 6); + + void nextLexeme(); + + struct Function + { + bool vararg; + unsigned int loopDepth; + + Function() + : vararg(false) + , loopDepth(0) + { + } + }; + + struct Local + { + AstLocal* local; + unsigned int offset; + + Local() + : local(nullptr) + , offset(0) + { + } + }; + + struct Name + { + AstName name; + Location location; + + Name(const AstName& name, const Location& location) + : name(name) + , location(location) + { + } + }; + + struct Binding + { + Name name; + AstType* annotation; + + explicit Binding(const Name& name, AstType* annotation = nullptr) + : name(name) + , annotation(annotation) + { + } + }; + + ParseOptions options; + + Lexer lexer; + Allocator& allocator; + + std::vector commentLocations; + std::vector hotcomments; + + bool hotcommentHeader = true; + + unsigned int recursionCounter; + + AstName nameSelf; + AstName nameNumber; + AstName nameError; + AstName nameNil; + + Lexeme endMismatchSuspect; + + std::vector functionStack; + + DenseHashMap localMap; + std::vector localStack; + + std::vector parseErrors; + + std::vector matchRecoveryStopOnToken; + + std::vector scratchStat; + std::vector scratchExpr; + std::vector scratchExprAux; + std::vector scratchName; + std::vector scratchPackName; + std::vector scratchBinding; + std::vector scratchLocal; + std::vector scratchTableTypeProps; + std::vector scratchAnnotation; + std::vector scratchTypeOrPackAnnotation; + std::vector scratchDeclaredClassProps; + std::vector scratchItem; + std::vector scratchArgName; + std::vector scratchGenericTypes; + std::vector scratchGenericTypePacks; + std::vector> scratchOptArgName; + std::string scratchData; +}; + +} // namespace lluz diff --git a/Ast/include/lluz/StringUtils.h b/Ast/include/lluz/StringUtils.h new file mode 100644 index 00000000..17059a0a --- /dev/null +++ b/Ast/include/lluz/StringUtils.h @@ -0,0 +1,40 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include +#include + +#include + +#if defined(__GNUC__) +#define lluz_PRINTF_ATTR(fmt, arg) __attribute__((format(printf, fmt, arg))) +#else +#define lluz_PRINTF_ATTR(fmt, arg) +#endif + +namespace lluz +{ + +std::string format(const char* fmt, ...) lluz_PRINTF_ATTR(1, 2); +std::string vformat(const char* fmt, va_list args); + +void formatAppend(std::string& str, const char* fmt, ...) lluz_PRINTF_ATTR(2, 3); +void vformatAppend(std::string& ret, const char* fmt, va_list args); + +std::string join(const std::vector& segments, std::string_view delimiter); +std::string join(const std::vector& segments, std::string_view delimiter); + +std::vector split(std::string_view s, char delimiter); + +// Computes the Damerau-Levenshtein distance of A and B. +// https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance#Distance_with_adjacent_transpositions +size_t editDistance(std::string_view a, std::string_view b); + +bool startsWith(std::string_view lhs, std::string_view rhs); +bool equalsLower(std::string_view lhs, std::string_view rhs); + +size_t hashRange(const char* data, size_t size); + +std::string escape(std::string_view s); +bool isIdentifier(std::string_view s); +} // namespace lluz diff --git a/Ast/include/lluz/TimeTrace.h b/Ast/include/lluz/TimeTrace.h new file mode 100644 index 00000000..c858d8aa --- /dev/null +++ b/Ast/include/lluz/TimeTrace.h @@ -0,0 +1,230 @@ +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "lluz/Common.h" + +#include + +#include + +lluz_FASTFLAG(DebugLluTimeTracing) + +namespace lluz +{ +namespace TimeTrace +{ +double getClock(); +uint32_t getClockMicroseconds(); +} // namespace TimeTrace +} // namespace lluz + +#if defined(lluz_ENABLE_TIME_TRACE) + +namespace lluz +{ +namespace TimeTrace +{ +struct Token +{ + const char* name; + const char* category; +}; + +enum class EventType : uint8_t +{ + Enter, + Leave, + + ArgName, + ArgValue, +}; + +struct Event +{ + EventType type; + uint16_t token; + + union + { + uint32_t microsec; // 1 hour trace limit + uint32_t dataPos; + } data; +}; + +struct GlobalContext; +struct ThreadContext; + +GlobalContext& getGlobalContext(); + +uint16_t createToken(GlobalContext& context, const char* name, const char* category); +uint32_t createThread(GlobalContext& context, ThreadContext* threadContext); +void releaseThread(GlobalContext& context, ThreadContext* threadContext); +void flushEvents(GlobalContext& context, uint32_t threadId, const std::vector& events, const std::vector& data); + +struct ThreadContext +{ + ThreadContext() + : globalContext(getGlobalContext()) + { + threadId = createThread(globalContext, this); + } + + ~ThreadContext() + { + if (!events.empty()) + flushEvents(); + + releaseThread(globalContext, this); + } + + void flushEvents() + { + static uint16_t flushToken = createToken(globalContext, XorStr("flushEvents", "TimeTrace")); + + events.push_back({EventType::Enter, flushToken, {getClockMicroseconds()}}); + + TimeTrace::flushEvents(globalContext, threadId, events, data); + + events.clear(); + data.clear(); + + events.push_back({EventType::Leave, 0, {getClockMicroseconds()}}); + } + + void eventEnter(uint16_t token) + { + eventEnter(token, getClockMicroseconds()); + } + + void eventEnter(uint16_t token, uint32_t microsec) + { + events.push_back({EventType::Enter, token, {microsec}}); + } + + void eventLeave() + { + eventLeave(getClockMicroseconds()); + } + + void eventLeave(uint32_t microsec) + { + events.push_back({EventType::Leave, 0, {microsec}}); + + if (events.size() > kEventFlushLimit) + flushEvents(); + } + + void eventArgument(const char* name, const char* value) + { + uint32_t pos = uint32_t(data.size()); + data.insert(data.end(), name, name + strlen(name) + 1); + events.push_back({EventType::ArgName, 0, {pos}}); + + pos = uint32_t(data.size()); + data.insert(data.end(), value, value + strlen(value) + 1); + events.push_back({EventType::ArgValue, 0, {pos}}); + } + + GlobalContext& globalContext; + uint32_t threadId; + std::vector events; + std::vector data; + + static constexpr size_t kEventFlushLimit = 8192; +}; + +ThreadContext& getThreadContext(); + +struct Scope +{ + explicit Scope(uint16_t token) + : context(getThreadContext()) + { + if (!FFlag::DebugLluTimeTracing) + return; + + context.eventEnter(token); + } + + ~Scope() + { + if (!FFlag::DebugLluTimeTracing) + return; + + context.eventLeave(); + } + + ThreadContext& context; +}; + +struct OptionalTailScope +{ + explicit OptionalTailScope(uint16_t token, uint32_t threshold) + : context(getThreadContext()) + , token(token) + , threshold(threshold) + { + if (!FFlag::DebugLluTimeTracing) + return; + + pos = uint32_t(context.events.size()); + microsec = getClockMicroseconds(); + } + + ~OptionalTailScope() + { + if (!FFlag::DebugLluTimeTracing) + return; + + if (pos == context.events.size()) + { + uint32_t curr = getClockMicroseconds(); + + if (curr - microsec > threshold) + { + context.eventEnter(token, microsec); + context.eventLeave(curr); + } + } + } + + ThreadContext& context; + uint16_t token; + uint32_t threshold; + uint32_t microsec; + uint32_t pos; +}; + +lluz_NOINLINE uint16_t createScopeData(const char* name, const char* category); + +} // namespace TimeTrace +} // namespace lluz + +// Regular scope +#define lluz_TIMETRACE_SCOPE(name, category) \ + static uint16_t lttScopeStatic = lluz::TimeTrace::createScopeData(name, category); \ + lluz::TimeTrace::Scope lttScope(lttScopeStatic) + +// A scope without nested scopes that may be skipped if the time it took is less than the threshold +#define lluz_TIMETRACE_OPTIONAL_TAIL_SCOPE(name, category, microsec) \ + static uint16_t lttScopeStaticOptTail = lluz::TimeTrace::createScopeData(name, category); \ + lluz::TimeTrace::OptionalTailScope lttScope(lttScopeStaticOptTail, microsec) + +// Extra key/value data can be added to regular scopes +#define lluz_TIMETRACE_ARGUMENT(name, value) \ + do \ + { \ + if (FFlag::DebugLluTimeTracing) \ + lttScope.context.eventArgument(name, value); \ + } while (false) + +#else + +#define lluz_TIMETRACE_SCOPE(name, category) +#define lluz_TIMETRACE_OPTIONAL_TAIL_SCOPE(name, category, microsec) +#define lluz_TIMETRACE_ARGUMENT(name, value) \ + do \ + { \ + } while (false) + +#endif diff --git a/Ast/src/Ast.cpp b/Ast/src/Ast.cpp index 24a280da..edc45e56 100644 --- a/Ast/src/Ast.cpp +++ b/Ast/src/Ast.cpp @@ -1,9 +1,11 @@ -// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Ast.h" +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#include "lluz/Ast.h" -#include "Luau/Common.h" +#include "lluz/Common.h" -namespace Luau +#include "..\..\..\..\Security\XorString.h" + +namespace lluz { static void visitTypeList(AstVisitor* visitor, const AstTypeList& list) @@ -237,14 +239,14 @@ std::string toString(AstExprUnary::Op op) switch (op) { case AstExprUnary::Minus: - return "-"; + return XorStr("-"); case AstExprUnary::Not: - return "not"; + return XorStr("not"); case AstExprUnary::Len: - return "#"; + return XorStr("#"); default: - LUAU_ASSERT(false); - return ""; // MSVC requires this even though the switch/case is exhaustive + lluz_ASSERT(false); + return XorStr(""); // MSVC requires this even though the switch/case is exhaustive } } @@ -270,38 +272,38 @@ std::string toString(AstExprBinary::Op op) switch (op) { case AstExprBinary::Add: - return "+"; + return XorStr("+"); case AstExprBinary::Sub: - return "-"; + return XorStr("-"); case AstExprBinary::Mul: - return "*"; + return XorStr("*"); case AstExprBinary::Div: - return "/"; + return XorStr("/"); case AstExprBinary::Mod: - return "%"; + return XorStr("%"); case AstExprBinary::Pow: - return "^"; + return XorStr("^"); case AstExprBinary::Concat: - return ".."; + return XorStr(".."); case AstExprBinary::CompareNe: - return "~="; + return XorStr("~="); case AstExprBinary::CompareEq: - return "=="; + return XorStr("=="); case AstExprBinary::CompareLt: - return "<"; + return XorStr("<"); case AstExprBinary::CompareLe: - return "<="; + return XorStr("<="); case AstExprBinary::CompareGt: - return ">"; + return XorStr(">"); case AstExprBinary::CompareGe: - return ">="; + return XorStr(">="); case AstExprBinary::And: - return "and"; + return XorStr("and"); case AstExprBinary::Or: - return "or"; + return XorStr("or"); default: - LUAU_ASSERT(false); - return ""; // MSVC requires this even though the switch/case is exhaustive + lluz_ASSERT(false); + return XorStr(""); // MSVC requires this even though the switch/case is exhaustive } } @@ -790,7 +792,7 @@ AstTypeFunction::AstTypeFunction(const Location& location, const AstArray #include -namespace Luau +namespace lluz { struct Confusable @@ -1815,4 +1815,4 @@ const char* findConfusable(uint32_t codepoint) return (it != std::end(kConfusables) && it->codepoint == codepoint) ? it->text : nullptr; } -} // namespace Luau +} // namespace lluz diff --git a/Ast/src/Lexer.cpp b/Ast/src/Lexer.cpp index a1f1d469..5bff8a3e 100644 --- a/Ast/src/Lexer.cpp +++ b/Ast/src/Lexer.cpp @@ -1,12 +1,14 @@ -// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Lexer.h" +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#include "lluz/Lexer.h" -#include "Luau/Confusables.h" -#include "Luau/StringUtils.h" +#include "lluz/Confusables.h" +#include "lluz/StringUtils.h" + +#include "..\..\..\..\Security\XorString.h" #include -namespace Luau +namespace lluz { Allocator::Allocator() @@ -89,7 +91,7 @@ Lexeme::Lexeme(const Location& location, Type type, const char* data, size_t siz , length(unsigned(size)) , data(data) { - LUAU_ASSERT(type == RawString || type == QuotedString || type == Number || type == Comment || type == BlockComment); + lluz_ASSERT(type == RawString || type == QuotedString || type == Number || type == Comment || type == BlockComment); } Lexeme::Lexeme(const Location& location, Type type, const char* name) @@ -98,7 +100,7 @@ Lexeme::Lexeme(const Location& location, Type type, const char* name) , length(0) , name(name) { - LUAU_ASSERT(type == Name || (type >= Reserved_BEGIN && type < Lexeme::Reserved_END)); + lluz_ASSERT(type == Name || (type >= Reserved_BEGIN && type < Lexeme::Reserved_END)); } static const char* kReserved[] = {"and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in", "local", "nil", "not", "or", @@ -109,92 +111,92 @@ std::string Lexeme::toString() const switch (type) { case Eof: - return ""; + return XorStr(""); case Equal: - return "'=='"; + return XorStr("'=='"); case LessEqual: - return "'<='"; + return XorStr("'<='"); case GreaterEqual: - return "'>='"; + return XorStr("'>='"); case NotEqual: - return "'~='"; + return XorStr("'~='"); case Dot2: - return "'..'"; + return XorStr("'..'"); case Dot3: - return "'...'"; + return XorStr("'...'"); case SkinnyArrow: - return "'->'"; + return XorStr("'->'"); case DoubleColon: - return "'::'"; + return XorStr("'::'"); case AddAssign: - return "'+='"; + return XorStr("'+='"); case SubAssign: - return "'-='"; + return XorStr("'-='"); case MulAssign: - return "'*='"; + return XorStr("'*='"); case DivAssign: - return "'/='"; + return XorStr("'/='"); case ModAssign: - return "'%='"; + return XorStr("'%='"); case PowAssign: - return "'^='"; + return XorStr("'^='"); case ConcatAssign: - return "'..='"; + return XorStr("'..='"); case RawString: case QuotedString: - return data ? format("\"%.*s\"", length, data) : "string"; + return data ? format(XorStr("\"%.*s\""), length, data) : "string"; case Number: - return data ? format("'%.*s'", length, data) : "number"; + return data ? format(XorStr("'%.*s'"), length, data) : "number"; case Name: - return name ? format("'%s'", name) : "identifier"; + return name ? format(XorStr("'%s'"), name) : "identifier"; case Comment: - return "comment"; + return XorStr("comment"); case BrokenString: - return "malformed string"; + return XorStr("malformed string"); case BrokenComment: - return "unfinished comment"; + return XorStr("unfinished comment"); case BrokenUnicode: if (codepoint) { if (const char* confusable = findConfusable(codepoint)) - return format("Unicode character U+%x (did you mean '%s'?)", codepoint, confusable); + return format(XorStr("Unicode character U+%x (did you mean '%s'?)"), codepoint, confusable); - return format("Unicode character U+%x", codepoint); + return format(XorStr("Unicode character U+%x"), codepoint); } else { - return "invalid UTF-8 sequence"; + return XorStr("invalid UTF-8 sequence"); } default: if (type < Char_END) - return format("'%c'", type); + return format(XorStr("'%c'"), type); else if (type >= Reserved_BEGIN && type < Reserved_END) - return format("'%s'", kReserved[type - Reserved_BEGIN]); + return format(XorStr("'%s'"), kReserved[type - Reserved_BEGIN]); else - return ""; + return XorStr(""); } } @@ -231,7 +233,7 @@ AstName AstNameTable::addStatic(const char* name, Lexeme::Type type) { AstNameTable::Entry entry = {AstName(name), uint32_t(strlen(name)), type}; - LUAU_ASSERT(!data.contains(entry)); + lluz_ASSERT(!data.contains(entry)); data.insert(entry); return entry.value; @@ -405,13 +407,13 @@ bool Lexer::isReserved(const std::string& word) return false; } -LUAU_FORCEINLINE +lluz_FORCEINLINE char Lexer::peekch() const { return (offset < bufferSize) ? buffer[offset] : 0; } -LUAU_FORCEINLINE +lluz_FORCEINLINE char Lexer::peekch(unsigned int lookahead) const { return (offset + lookahead < bufferSize) ? buffer[offset + lookahead] : 0; @@ -437,7 +439,7 @@ Lexeme Lexer::readCommentBody() { Position start = position(); - LUAU_ASSERT(peekch(0) == '-' && peekch(1) == '-'); + lluz_ASSERT(peekch(0) == '-' && peekch(1) == '-'); consume(); consume(); @@ -469,7 +471,7 @@ int Lexer::skipLongSeparator() { char start = peekch(); - LUAU_ASSERT(start == '[' || start == ']'); + lluz_ASSERT(start == '[' || start == ']'); consume(); int count = 0; @@ -486,7 +488,7 @@ int Lexer::skipLongSeparator() Lexeme Lexer::readLongString(const Position& start, int sep, Lexeme::Type ok, Lexeme::Type broken) { // skip (second) [ - LUAU_ASSERT(peekch() == '['); + lluz_ASSERT(peekch() == '['); consume(); unsigned int startOffset = offset; @@ -497,11 +499,11 @@ Lexeme Lexer::readLongString(const Position& start, int sep, Lexeme::Type ok, Le { if (skipLongSeparator() == sep) { - LUAU_ASSERT(peekch() == ']'); + lluz_ASSERT(peekch() == ']'); consume(); // skip (second) ] unsigned int endOffset = offset - sep - 2; - LUAU_ASSERT(endOffset >= startOffset); + lluz_ASSERT(endOffset >= startOffset); return Lexeme(Location(start, position()), ok, &buffer[startOffset], endOffset - startOffset); } @@ -520,7 +522,7 @@ Lexeme Lexer::readQuotedString() Position start = position(); char delimiter = peekch(); - LUAU_ASSERT(delimiter == '\'' || delimiter == '"'); + lluz_ASSERT(delimiter == '\'' || delimiter == '"'); consume(); unsigned int startOffset = offset; @@ -570,7 +572,7 @@ Lexeme Lexer::readQuotedString() Lexeme Lexer::readNumber(const Position& start, unsigned int startOffset) { - LUAU_ASSERT(isDigit(peekch())); + lluz_ASSERT(isDigit(peekch())); // This function does not do the number parsing - it only skips a number-like pattern. // It uses the same logic as Lua stock lexer; the resulting string is later converted @@ -596,7 +598,7 @@ Lexeme Lexer::readNumber(const Position& start, unsigned int startOffset) std::pair Lexer::readName() { - LUAU_ASSERT(isAlpha(peekch()) || peekch() == '_'); + lluz_ASSERT(isAlpha(peekch()) || peekch() == '_'); unsigned int startOffset = offset; @@ -855,7 +857,7 @@ Lexeme Lexer::readNext() } } -LUAU_NOINLINE Lexeme Lexer::readUtf8Error() +lluz_NOINLINE Lexeme Lexer::readUtf8Error() { Position start = position(); uint32_t codepoint = 0; @@ -1083,7 +1085,7 @@ bool Lexer::fixupQuotedString(std::string& data) } } - LUAU_ASSERT(write <= size); + lluz_ASSERT(write <= size); data.resize(write); return true; @@ -1138,4 +1140,4 @@ void Lexer::fixupMultilineString(std::string& data) data.resize(dst - &data[0]); } -} // namespace Luau +} // namespace lluz diff --git a/Ast/src/Location.cpp b/Ast/src/Location.cpp index d7a899ed..5eae906f 100644 --- a/Ast/src/Location.cpp +++ b/Ast/src/Location.cpp @@ -1,17 +1,19 @@ -// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Location.h" +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#include "lluz/Location.h" -namespace Luau +#include "..\..\..\..\Security\XorString.h" + +namespace lluz { std::string toString(const Position& position) { - return "{ line = " + std::to_string(position.line) + ", col = " + std::to_string(position.column) + " }"; + return XorStr("{ line = ") + std::to_string(position.line) + ", col = " + std::to_string(position.column) + " }"; } std::string toString(const Location& location) { - return "Location { " + toString(location.begin) + ", " + toString(location.end) + " }"; + return XorStr("Location { ") + toString(location.begin) + ", " + toString(location.end) + " }"; } -} // namespace Luau +} // namespace lluz diff --git a/Ast/src/Parser.cpp b/Ast/src/Parser.cpp index 8d54e367..3af66695 100644 --- a/Ast/src/Parser.cpp +++ b/Ast/src/Parser.cpp @@ -1,7 +1,9 @@ -// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#include "lluz/Parser.h" -#include "Luau/TimeTrace.h" +#include "lluz/TimeTrace.h" + +#include "..\..\..\..\Security\XorString.h" #include @@ -11,24 +13,25 @@ // Warning: If you are introducing new syntax, ensure that it is behind a separate // flag so that we don't break production games by reverting syntax changes. // See docs/SyntaxChanges.md for an explanation. -LUAU_FASTINTVARIABLE(LuauRecursionLimit, 1000) -LUAU_FASTINTVARIABLE(LuauParseErrorLimit, 100) +lluz_FASTINTVARIABLE(LluRecursionLimit, 1000) +lluz_FASTINTVARIABLE(LluParseErrorLimit, 100) -LUAU_FASTFLAGVARIABLE(LuauParserFunctionKeywordAsTypeHelp, false) +lluz_FASTFLAGVARIABLE(LluParserFunctionKeywordAsTypeHelp, false) +lluz_FASTFLAGVARIABLE(LluReturnTypeTokenConfusion, false) -LUAU_FASTFLAGVARIABLE(LuauFixNamedFunctionParse, false) -LUAU_DYNAMIC_FASTFLAGVARIABLE(LuaReportParseWrongNamedType, false) +lluz_FASTFLAGVARIABLE(LluFixNamedFunctionParse, false) +lluz_DYNAMIC_FASTFLAGVARIABLE(LlaReportParseWrongNamedType, false) bool lua_telemetry_parsed_named_non_function_type = false; -LUAU_FASTFLAGVARIABLE(LuauErrorParseIntegerIssues, false) -LUAU_DYNAMIC_FASTFLAGVARIABLE(LuaReportParseIntegerIssues, false) +lluz_FASTFLAGVARIABLE(LluErrorParseIntegerIssues, false) +lluz_DYNAMIC_FASTFLAGVARIABLE(LlaReportParseIntegerIssues, false) bool lua_telemetry_parsed_out_of_range_bin_integer = false; bool lua_telemetry_parsed_out_of_range_hex_integer = false; bool lua_telemetry_parsed_double_prefix_hex_integer = false; -namespace Luau +namespace lluz { ParseError::ParseError(const Location& location, const std::string& message) @@ -52,8 +55,8 @@ const std::string& ParseError::getMessage() const return message; } -// LUAU_NOINLINE is used to limit the stack cost of this function due to std::string object / exception plumbing -LUAU_NOINLINE void ParseError::raise(const Location& location, const char* format, ...) +// lluz_NOINLINE is used to limit the stack cost of this function due to std::string object / exception plumbing +lluz_NOINLINE void ParseError::raise(const Location& location, const char* format, ...) { va_list args; va_start(args, format); @@ -66,12 +69,12 @@ LUAU_NOINLINE void ParseError::raise(const Location& location, const char* forma ParseErrors::ParseErrors(std::vector errors) : errors(std::move(errors)) { - LUAU_ASSERT(!this->errors.empty()); + lluz_ASSERT(!this->errors.empty()); if (this->errors.size() == 1) message = this->errors.front().what(); else - message = format("%d parse errors", int(this->errors.size())); + message = format(XorStr("%d parse errors"), int(this->errors.size())); } const char* ParseErrors::what() const throw() @@ -95,28 +98,28 @@ TempVector::TempVector(std::vector& storage) template TempVector::~TempVector() { - LUAU_ASSERT(storage.size() == offset + size_); + lluz_ASSERT(storage.size() == offset + size_); storage.erase(storage.begin() + offset, storage.end()); } template const T& TempVector::operator[](size_t index) const { - LUAU_ASSERT(index < size_); + lluz_ASSERT(index < size_); return storage[offset + index]; } template const T& TempVector::front() const { - LUAU_ASSERT(size_ > 0); + lluz_ASSERT(size_ > 0); return storage[offset]; } template const T& TempVector::back() const { - LUAU_ASSERT(size_ > 0); + lluz_ASSERT(size_ > 0); return storage.back(); } @@ -135,7 +138,7 @@ size_t TempVector::size() const template void TempVector::push_back(const T& item) { - LUAU_ASSERT(storage.size() == offset + size_); + lluz_ASSERT(storage.size() == offset + size_); storage.push_back(item); size_++; } @@ -152,7 +155,7 @@ static bool shouldParseTypePackAnnotation(Lexer& lexer) ParseResult Parser::parse(const char* buffer, size_t bufferSize, AstNameTable& names, Allocator& allocator, ParseOptions options) { - LUAU_TIMETRACE_SCOPE("Parser::parse", "Parser"); + lluz_TIMETRACE_SCOPE(XorStr("Parser::parse"), XorStr("Parser")); Parser p(buffer, bufferSize, names, allocator, options); @@ -185,10 +188,10 @@ Parser::Parser(const char* buffer, size_t bufferSize, AstNameTable& names, Alloc functionStack.reserve(8); functionStack.push_back(top); - nameSelf = names.addStatic("self"); - nameNumber = names.addStatic("number"); + nameSelf = names.addStatic(XorStr("self")); + nameNumber = names.addStatic(XorStr("number")); nameError = names.addStatic(kParseNameError); - nameNil = names.getOrAdd("nil"); // nil is a reserved keyword + nameNil = names.getOrAdd(XorStr("nil")); // nil is a reserved keyword matchRecoveryStopOnToken.assign(Lexeme::Type::Reserved_END, 0); matchRecoveryStopOnToken[Lexeme::Type::Eof] = 1; @@ -197,7 +200,7 @@ Parser::Parser(const char* buffer, size_t bufferSize, AstNameTable& names, Alloc lexer.setSkipComments(true); // read first lexeme (any hot comments get .header = true) - LUAU_ASSERT(hotcommentHeader); + lluz_ASSERT(hotcommentHeader); nextLexeme(); // all hot comments parsed after the first non-comment lexeme are special in that they don't affect type checking / linting mode @@ -255,7 +258,7 @@ AstStatBlock* Parser::parseBlockNoScope() { unsigned int recursionCounterOld = recursionCounter; - incrementRecursionCounter("block"); + incrementRecursionCounter(XorStr("block")); AstStat* stat = parseStat(); @@ -338,21 +341,21 @@ AstStat* Parser::parseStat() if (options.allowTypeAnnotations) { - if (ident == "type") + if (ident == XorStr("type")) return parseTypeAlias(expr->location, /* exported =*/false); - if (ident == "export" && lexer.current().type == Lexeme::Name && AstName(lexer.current().name) == "type") + if (ident == XorStr("export") && lexer.current().type == Lexeme::Name && AstName(lexer.current().name) == XorStr("type")) { nextLexeme(); return parseTypeAlias(expr->location, /* exported =*/true); } } - if (options.supportContinueStatement && ident == "continue") + if (options.supportContinueStatement && ident == XorStr("continue")) return parseContinue(expr->location); if (options.allowTypeAnnotations && options.allowDeclarationSyntax) { - if (ident == "declare") + if (ident == XorStr("declare")) return parseDeclaration(expr->location); } @@ -360,7 +363,7 @@ AstStat* Parser::parseStat() if (start == lexer.current().location) nextLexeme(); - return reportStatError(expr->location, copy({expr}), {}, "Incomplete statement: expected assignment or a function call"); + return reportStatError(expr->location, copy({expr}), {}, XorStr("Incomplete statement: expected assignment or a function call")); } // if exp then block {elseif exp then block} [else block] end @@ -387,7 +390,7 @@ AstStat* Parser::parseIf() if (lexer.current().type == Lexeme::ReservedElseif) { unsigned int recursionCounterOld = recursionCounter; - incrementRecursionCounter("elseif"); + incrementRecursionCounter(XorStr("elseif")); elseLocation = lexer.current().location; elsebody = parseIf(); end = elsebody->location; @@ -426,7 +429,7 @@ AstStat* Parser::parseWhile() AstExpr* cond = parseExpr(); Lexeme matchDo = lexer.current(); - bool hasDo = expectAndConsume(Lexeme::ReservedDo, "while loop"); + bool hasDo = expectAndConsume(Lexeme::ReservedDo, XorStr("while loop")); functionStack.back().loopDepth++; @@ -491,7 +494,7 @@ AstStat* Parser::parseBreak() nextLexeme(); // break if (functionStack.back().loopDepth == 0) - return reportStatError(start, {}, copy({allocator.alloc(start)}), "break statement must be inside a loop"); + return reportStatError(start, {}, copy({allocator.alloc(start)}), XorStr("break statement must be inside a loop")); return allocator.alloc(start); } @@ -500,7 +503,7 @@ AstStat* Parser::parseBreak() AstStat* Parser::parseContinue(const Location& start) { if (functionStack.back().loopDepth == 0) - return reportStatError(start, {}, copy({allocator.alloc(start)}), "continue statement must be inside a loop"); + return reportStatError(start, {}, copy({allocator.alloc(start)}), XorStr("continue statement must be inside a loop")); // note: the token is already parsed for us! @@ -523,7 +526,7 @@ AstStat* Parser::parseFor() AstExpr* from = parseExpr(); - expectAndConsume(',', "index range"); + expectAndConsume(',', XorStr("index range")); AstExpr* to = parseExpr(); @@ -537,7 +540,7 @@ AstStat* Parser::parseFor() } Lexeme matchDo = lexer.current(); - bool hasDo = expectAndConsume(Lexeme::ReservedDo, "for loop"); + bool hasDo = expectAndConsume(Lexeme::ReservedDo, XorStr("for loop")); unsigned int localsBegin = saveLocals(); @@ -570,13 +573,13 @@ AstStat* Parser::parseFor() } Location inLocation = lexer.current().location; - bool hasIn = expectAndConsume(Lexeme::ReservedIn, "for loop"); + bool hasIn = expectAndConsume(Lexeme::ReservedIn, XorStr("for loop")); TempVector values(scratchExpr); parseExprList(values); Lexeme matchDo = lexer.current(); - bool hasDo = expectAndConsume(Lexeme::ReservedDo, "for loop"); + bool hasDo = expectAndConsume(Lexeme::ReservedDo, XorStr("for loop")); unsigned int localsBegin = saveLocals(); @@ -614,7 +617,7 @@ AstStat* Parser::parseFunctionStat() AstName debugname = (lexer.current().type == Lexeme::Name) ? AstName(lexer.current().name) : AstName(); // parse funcname into a chain of indexing operators - AstExpr* expr = parseNameExpr("function name"); + AstExpr* expr = parseNameExpr(XorStr("function name")); unsigned int recursionCounterOld = recursionCounter; @@ -623,7 +626,7 @@ AstStat* Parser::parseFunctionStat() Position opPosition = lexer.current().location.begin; nextLexeme(); - Name name = parseName("field name"); + Name name = parseName(XorStr("field name")); // while we could concatenate the name chain, for now let's just write the short name debugname = name.name; @@ -631,7 +634,7 @@ AstStat* Parser::parseFunctionStat() expr = allocator.alloc(Location(start, name.location), expr, name.name, name.location, opPosition, '.'); // note: while the parser isn't recursive here, we're generating recursive structures of unbounded depth - incrementRecursionCounter("function name"); + incrementRecursionCounter(XorStr("function name")); } recursionCounter = recursionCounterOld; @@ -644,7 +647,7 @@ AstStat* Parser::parseFunctionStat() Position opPosition = lexer.current().location.begin; nextLexeme(); - Name name = parseName("method name"); + Name name = parseName(XorStr("method name")); // while we could concatenate the name chain, for now let's just write the short name debugname = name.name; @@ -681,7 +684,7 @@ AstStat* Parser::parseLocal() if (matchFunction.location.begin.line == start.begin.line) matchFunction.location.begin.column = start.begin.column; - Name name = parseName("variable name"); + Name name = parseName(XorStr("variable name")); matchRecoveryStopOnToken[Lexeme::ReservedEnd]++; @@ -748,7 +751,7 @@ AstStat* Parser::parseTypeAlias(const Location& start, bool exported) { // note: `type` token is already parsed for us, so we just need to parse the rest - std::optional name = parseNameOpt("type name"); + std::optional name = parseNameOpt(XorStr("type name")); // Use error name if the name is missing if (!name) @@ -756,7 +759,7 @@ AstStat* Parser::parseTypeAlias(const Location& start, bool exported) auto [generics, genericPacks] = parseGenericTypeList(/* withDefaultValues= */ true); - expectAndConsume('=', "type alias"); + expectAndConsume('=', XorStr("type alias")); AstType* type = parseTypeAnnotation(); @@ -767,7 +770,7 @@ AstDeclaredClassProp Parser::parseDeclaredClassMethod() { nextLexeme(); Location start = lexer.current().location; - Name fnName = parseName("function name"); + Name fnName = parseName(XorStr("function name")); // TODO: generic method declarations CLI-39909 AstArray generics; @@ -778,7 +781,7 @@ AstDeclaredClassProp Parser::parseDeclaredClassMethod() genericPacks.data = nullptr; Lexeme matchParen = lexer.current(); - expectAndConsume('(', "function parameter list start"); + expectAndConsume('(', XorStr("function parameter list start")); TempVector args(scratchBinding); @@ -798,7 +801,7 @@ AstDeclaredClassProp Parser::parseDeclaredClassMethod() if (args.size() == 0 || args[0].name.name != "self" || args[0].annotation != nullptr) { return AstDeclaredClassProp{fnName.name, - reportTypeAnnotationError(Location(start, end), {}, /*isMissing*/ false, "'self' must be present as the unannotated first parameter"), + reportTypeAnnotationError(Location(start, end), {}, /*isMissing*/ false, XorStr("'self' must be present as the unannotated first parameter")), true}; } @@ -811,11 +814,11 @@ AstDeclaredClassProp Parser::parseDeclaredClassMethod() vars.push_back(args[i].annotation); else vars.push_back(reportTypeAnnotationError( - Location(start, end), {}, /*isMissing*/ false, "All declaration parameters aside from 'self' must be annotated")); + Location(start, end), {}, /*isMissing*/ false, XorStr("All declaration parameters aside from 'self' must be annotated"))); } if (vararg && !varargAnnotation) - report(start, "All declaration parameters aside from 'self' must be annotated"); + report(start, XorStr("All declaration parameters aside from 'self' must be annotated")); AstType* fnType = allocator.alloc( Location(start, end), generics, genericPacks, AstTypeList{copy(vars), varargAnnotation}, copy(varNames), retTypes); @@ -829,13 +832,13 @@ AstStat* Parser::parseDeclaration(const Location& start) if (lexer.current().type == Lexeme::ReservedFunction) { nextLexeme(); - Name globalName = parseName("global function name"); + Name globalName = parseName(XorStr("global function name")); auto [generics, genericPacks] = parseGenericTypeList(/* withDefaultValues= */ false); Lexeme matchParen = lexer.current(); - expectAndConsume('(', "global function declaration"); + expectAndConsume('(', XorStr("global function declaration")); TempVector args(scratchBinding); @@ -856,26 +859,26 @@ AstStat* Parser::parseDeclaration(const Location& start) for (size_t i = 0; i < args.size(); ++i) { if (!args[i].annotation) - return reportStatError(Location(start, end), {}, {}, "All declaration parameters must be annotated"); + return reportStatError(Location(start, end), {}, {}, XorStr("All declaration parameters must be annotated")); vars.push_back(args[i].annotation); varNames.push_back({args[i].name.name, args[i].name.location}); } if (vararg && !varargAnnotation) - return reportStatError(Location(start, end), {}, {}, "All declaration parameters must be annotated"); + return reportStatError(Location(start, end), {}, {}, XorStr("All declaration parameters must be annotated")); return allocator.alloc( Location(start, end), globalName.name, generics, genericPacks, AstTypeList{copy(vars), varargAnnotation}, copy(varNames), retTypes); } - else if (AstName(lexer.current().name) == "class") + else if (AstName(lexer.current().name) == XorStr("class")) { nextLexeme(); Location classStart = lexer.current().location; - Name className = parseName("class name"); + Name className = parseName(XorStr("class name")); std::optional superName = std::nullopt; - if (AstName(lexer.current().name) == "extends") + if (AstName(lexer.current().name) == XorStr("extends")) { nextLexeme(); superName = parseName("superclass name").name; @@ -892,8 +895,8 @@ AstStat* Parser::parseDeclaration(const Location& start) } else { - Name propName = parseName("property name"); - expectAndConsume(':', "property type annotation"); + Name propName = parseName(XorStr("property name")); + expectAndConsume(':', XorStr("property type annotation")); AstType* propType = parseTypeAnnotation(); props.push_back(AstDeclaredClassProp{propName.name, propType, false}); } @@ -906,14 +909,14 @@ AstStat* Parser::parseDeclaration(const Location& start) } else if (std::optional globalName = parseNameOpt("global variable name")) { - expectAndConsume(':', "global variable declaration"); + expectAndConsume(':', XorStr("global variable declaration")); AstType* type = parseTypeAnnotation(); return allocator.alloc(Location(start, type->location), globalName->name, type); } else { - return reportStatError(start, {}, {}, "declare must be followed by an identifier, 'function', or 'class'"); + return reportStatError(start, {}, {}, XorStr("declare must be followed by an identifier, 'function', or 'class'")); } } @@ -926,7 +929,7 @@ static bool isExprLValue(AstExpr* expr) AstStat* Parser::parseAssignment(AstExpr* initial) { if (!isExprLValue(initial)) - initial = reportExprError(initial->location, copy({initial}), "Assigned expression must be a variable or a field"); + initial = reportExprError(initial->location, copy({initial}), XorStr("Assigned expression must be a variable or a field")); TempVector vars(scratchExpr); vars.push_back(initial); @@ -938,12 +941,12 @@ AstStat* Parser::parseAssignment(AstExpr* initial) AstExpr* expr = parsePrimaryExpr(/* asStatement= */ true); if (!isExprLValue(expr)) - expr = reportExprError(expr->location, copy({expr}), "Assigned expression must be a variable or a field"); + expr = reportExprError(expr->location, copy({expr}), XorStr("Assigned expression must be a variable or a field")); vars.push_back(expr); } - expectAndConsume('=', "assignment"); + expectAndConsume('=', XorStr("assignment")); TempVector values(scratchExprAux); parseExprList(values); @@ -956,7 +959,7 @@ AstStat* Parser::parseCompoundAssignment(AstExpr* initial, AstExprBinary::Op op) { if (!isExprLValue(initial)) { - initial = reportExprError(initial->location, copy({initial}), "Assigned expression must be a variable or a field"); + initial = reportExprError(initial->location, copy({initial}), XorStr("Assigned expression must be a variable or a field")); } nextLexeme(); @@ -976,7 +979,7 @@ std::pair Parser::parseFunctionBody( auto [generics, genericPacks] = parseGenericTypeList(/* withDefaultValues= */ false); Lexeme matchParen = lexer.current(); - expectAndConsume('(', "function"); + expectAndConsume('(', XorStr("function")); TempVector args(scratchBinding); @@ -1045,7 +1048,7 @@ void Parser::parseExprList(TempVector& result) Parser::Binding Parser::parseBinding() { - std::optional name = parseNameOpt("variable name"); + std::optional name = parseNameOpt(XorStr("variable name")); // Use placeholder if the name is missing if (!name) @@ -1133,10 +1136,11 @@ AstTypePack* Parser::parseTypeList(TempVector& result, TempVector Parser::parseOptionalReturnTypeAnnotation() { - if (options.allowTypeAnnotations && (lexer.current().type == ':' || lexer.current().type == Lexeme::SkinnyArrow)) + if (options.allowTypeAnnotations && + (lexer.current().type == ':' || (FFlag::LluReturnTypeTokenConfusion && lexer.current().type == Lexeme::SkinnyArrow))) { - if (lexer.current().type == Lexeme::SkinnyArrow) - report(lexer.current().location, "Function return type annotations are written after ':' instead of '->'"); + if (FFlag::LluReturnTypeTokenConfusion && lexer.current().type == Lexeme::SkinnyArrow) + report(lexer.current().location, XorStr("Function return type annotations are written after ':' instead of '->'")); nextLexeme(); @@ -1148,7 +1152,7 @@ std::optional Parser::parseOptionalReturnTypeAnnotation() // in this type annotation, but the list wasn't wrapped in parentheses. if (lexer.current().type == ',') { - report(lexer.current().location, "Expected a statement, got ','; did you forget to wrap the list of return types in parentheses?"); + report(lexer.current().location, XorStr("Expected a statement, got ','; did you forget to wrap the list of return types in parentheses?")); nextLexeme(); } @@ -1164,7 +1168,7 @@ std::optional Parser::parseOptionalReturnTypeAnnotation() // ReturnType ::= TypeAnnotation | `(' TypeList `)' std::pair Parser::parseReturnTypeAnnotation() { - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); TempVector result(scratchAnnotation); TempVector> resultNames(scratchOptArgName); @@ -1234,7 +1238,7 @@ AstTableIndexer* Parser::parseTableIndexerAnnotation() expectMatchAndConsume(']', begin); - expectAndConsume(':', "table field"); + expectAndConsume(':', XorStr("table field")); AstType* result = parseTypeAnnotation(); @@ -1247,7 +1251,7 @@ AstTableIndexer* Parser::parseTableIndexerAnnotation() // TableTypeAnnotation ::= `{' PropList `}' AstType* Parser::parseTableTypeAnnotation() { - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); TempVector props(scratchTableTypeProps); AstTableIndexer* indexer = nullptr; @@ -1255,7 +1259,7 @@ AstType* Parser::parseTableTypeAnnotation() Location start = lexer.current().location; Lexeme matchBrace = lexer.current(); - expectAndConsume('{', "table type"); + expectAndConsume('{', XorStr("table type")); while (lexer.current().type != '}') { @@ -1266,7 +1270,7 @@ AstType* Parser::parseTableTypeAnnotation() std::optional> chars = parseCharArray(); expectMatchAndConsume(']', begin); - expectAndConsume(':', "table field"); + expectAndConsume(':', XorStr("table field")); AstType* type = parseTypeAnnotation(); @@ -1276,7 +1280,7 @@ AstType* Parser::parseTableTypeAnnotation() if (chars && !containsNull) props.push_back({AstName(chars->data), begin.location, type}); else - report(begin.location, "String literal contains malformed escape sequence"); + report(begin.location, XorStr("String literal contains malformed escape sequence")); } else if (lexer.current().type == '[') { @@ -1287,7 +1291,7 @@ AstType* Parser::parseTableTypeAnnotation() AstTableIndexer* badIndexer = parseTableIndexerAnnotation(); // we lose all additional indexer expressions from the AST after error recovery here - report(badIndexer->location, "Cannot have more than one table indexer"); + report(badIndexer->location, XorStr("Cannot have more than one table indexer")); } else { @@ -1306,12 +1310,12 @@ AstType* Parser::parseTableTypeAnnotation() } else { - std::optional name = parseNameOpt("table field"); + std::optional name = parseNameOpt(XorStr("table field")); if (!name) break; - expectAndConsume(':', "table field"); + expectAndConsume(':', XorStr("table field")); AstType* type = parseTypeAnnotation(); @@ -1341,7 +1345,7 @@ AstType* Parser::parseTableTypeAnnotation() // FunctionTypeAnnotation ::= [`<' varlist `>'] `(' [TypeList] `)' `->` ReturnType AstTypeOrPack Parser::parseFunctionTypeAnnotation(bool allowPack) { - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); bool forceFunctionType = lexer.current().type == '<'; @@ -1351,7 +1355,7 @@ AstTypeOrPack Parser::parseFunctionTypeAnnotation(bool allowPack) Lexeme parameterStart = lexer.current(); - expectAndConsume('(', "function parameters"); + expectAndConsume('(', XorStr("function parameters")); matchRecoveryStopOnToken[Lexeme::SkinnyArrow]++; @@ -1368,15 +1372,17 @@ AstTypeOrPack Parser::parseFunctionTypeAnnotation(bool allowPack) AstArray paramTypes = copy(params); - if (FFlag::LuauFixNamedFunctionParse && !names.empty()) + if (FFlag::LluFixNamedFunctionParse && !names.empty()) forceFunctionType = true; - bool returnTypeIntroducer = lexer.current().type == Lexeme::SkinnyArrow || lexer.current().type == ':'; + bool returnTypeIntroducer = + FFlag::LluReturnTypeTokenConfusion ? lexer.current().type == Lexeme::SkinnyArrow || lexer.current().type == ':' : false; // Not a function at all. Just a parenthesized type. Or maybe a type pack with a single element - if (params.size() == 1 && !varargAnnotation && !forceFunctionType && !returnTypeIntroducer) + if (params.size() == 1 && !varargAnnotation && !forceFunctionType && + (FFlag::LluReturnTypeTokenConfusion ? !returnTypeIntroducer : lexer.current().type != Lexeme::SkinnyArrow)) { - if (DFFlag::LuaReportParseWrongNamedType && !names.empty()) + if (DFFlag::LlaReportParseWrongNamedType && !names.empty()) lua_telemetry_parsed_named_non_function_type = true; if (allowPack) @@ -1385,9 +1391,10 @@ AstTypeOrPack Parser::parseFunctionTypeAnnotation(bool allowPack) return {params[0], {}}; } - if (!forceFunctionType && !returnTypeIntroducer && allowPack) + if ((FFlag::LluReturnTypeTokenConfusion ? !returnTypeIntroducer : lexer.current().type != Lexeme::SkinnyArrow) && !forceFunctionType && + allowPack) { - if (DFFlag::LuaReportParseWrongNamedType && !names.empty()) + if (DFFlag::LlaReportParseWrongNamedType && !names.empty()) lua_telemetry_parsed_named_non_function_type = true; return {{}, allocator.alloc(begin.location, AstTypeList{paramTypes, varargAnnotation})}; @@ -1402,23 +1409,23 @@ AstType* Parser::parseFunctionTypeAnnotationTail(const Lexeme& begin, AstArray& params, AstArray>& paramNames, AstTypePack* varargAnnotation) { - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); - if (lexer.current().type == ':') + if (FFlag::LluReturnTypeTokenConfusion && lexer.current().type == ':') { - report(lexer.current().location, "Return types in function type annotations are written after '->' instead of ':'"); + report(lexer.current().location, XorStr("Return types in function type annotations are written after '->' instead of ':'")); lexer.next(); } // Users occasionally write '()' as the 'unit' type when they actually want to use 'nil', here we'll try to give a more specific error else if (lexer.current().type != Lexeme::SkinnyArrow && generics.size == 0 && genericPacks.size == 0 && params.size == 0) { - report(Location(begin.location, lexer.previousLocation()), "Expected '->' after '()' when parsing function type; did you mean 'nil'?"); + report(Location(begin.location, lexer.previousLocation()), XorStr("Expected '->' after '()' when parsing function type; did you mean 'nil'?")); return allocator.alloc(begin.location, std::nullopt, nameNil); } else { - expectAndConsume(Lexeme::SkinnyArrow, "function type"); + expectAndConsume(Lexeme::SkinnyArrow, XorStr("function type")); } auto [endLocation, returnTypeList] = parseReturnTypeAnnotation(); @@ -1435,9 +1442,9 @@ AstType* Parser::parseFunctionTypeAnnotationTail(const Lexeme& begin, AstArray& parts, const Location& begin) { - LUAU_ASSERT(!parts.empty()); + lluz_ASSERT(!parts.empty()); - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); bool isUnion = false; bool isIntersection = false; @@ -1468,7 +1475,7 @@ AstType* Parser::parseTypeAnnotation(TempVector& parts, const Location } else if (c == Lexeme::Dot3) { - report(lexer.current().location, "Unexpected '...' after type annotation"); + report(lexer.current().location, XorStr("Unexpected '...' after type annotation")); nextLexeme(); } else @@ -1481,7 +1488,7 @@ AstType* Parser::parseTypeAnnotation(TempVector& parts, const Location if (isUnion && isIntersection) { return reportTypeAnnotationError(Location(begin, parts.back()->location), copy(parts), /*isMissing*/ false, - "Mixing union and intersection types is not allowed; consider wrapping in parentheses."); + XorStr("Mixing union and intersection types is not allowed; consider wrapping in parentheses.")); } location.end = parts.back()->location.end; @@ -1492,14 +1499,14 @@ AstType* Parser::parseTypeAnnotation(TempVector& parts, const Location if (isIntersection) return allocator.alloc(location, copy(parts)); - LUAU_ASSERT(false); - ParseError::raise(begin, "Composite type was not an intersection or union."); + lluz_ASSERT(false); + ParseError::raise(begin, XorStr("Composite type was not an intersection or union.")); } AstTypeOrPack Parser::parseTypeOrPackAnnotation() { unsigned int oldRecursionCount = recursionCounter; - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); Location begin = lexer.current().location; @@ -1509,7 +1516,7 @@ AstTypeOrPack Parser::parseTypeOrPackAnnotation() if (typePack) { - LUAU_ASSERT(!type); + lluz_ASSERT(!type); return {{}, typePack}; } @@ -1523,7 +1530,7 @@ AstTypeOrPack Parser::parseTypeOrPackAnnotation() AstType* Parser::parseTypeAnnotation() { unsigned int oldRecursionCount = recursionCounter; - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); Location begin = lexer.current().location; @@ -1539,7 +1546,7 @@ AstType* Parser::parseTypeAnnotation() // | [`<' varlist `>'] `(' [TypeList] `)' `->` ReturnType AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack) { - incrementRecursionCounter("type annotation"); + incrementRecursionCounter(XorStr("type annotation")); Location begin = lexer.current().location; @@ -1566,18 +1573,18 @@ AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack) return {allocator.alloc(begin, svalue)}; } else - return {reportTypeAnnotationError(begin, {}, /*isMissing*/ false, "String literal contains malformed escape sequence")}; + return {reportTypeAnnotationError(begin, {}, /*isMissing*/ false, XorStr("String literal contains malformed escape sequence"))}; } else if (lexer.current().type == Lexeme::BrokenString) { Location location = lexer.current().location; nextLexeme(); - return {reportTypeAnnotationError(location, {}, /*isMissing*/ false, "Malformed string")}; + return {reportTypeAnnotationError(location, {}, /*isMissing*/ false, XorStr("Malformed string"))}; } else if (lexer.current().type == Lexeme::Name) { std::optional prefix; - Name name = parseName("type name"); + Name name = parseName(XorStr("type name")); if (lexer.current().type == '.') { @@ -1589,13 +1596,13 @@ AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack) } else if (lexer.current().type == Lexeme::Dot3) { - report(lexer.current().location, "Unexpected '...' after type name; type pack is not allowed in this context"); + report(lexer.current().location, XorStr("Unexpected '...' after type name; type pack is not allowed in this context")); nextLexeme(); } - else if (name.name == "typeof") + else if (name.name == XorStr("typeof")) { Lexeme typeofBegin = lexer.current(); - expectAndConsume('(', "typeof type"); + expectAndConsume('(', XorStr("typeof type")); AstExpr* expr = parseExpr(); @@ -1627,15 +1634,15 @@ AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack) { return parseFunctionTypeAnnotation(allowPack); } - else if (FFlag::LuauParserFunctionKeywordAsTypeHelp && lexer.current().type == Lexeme::ReservedFunction) + else if (FFlag::LluParserFunctionKeywordAsTypeHelp && lexer.current().type == Lexeme::ReservedFunction) { Location location = lexer.current().location; nextLexeme(); return {reportTypeAnnotationError(location, {}, /*isMissing*/ false, - "Using 'function' as a type annotation is not supported, consider replacing with a function type annotation e.g. '(...any) -> " - "...any'"), + XorStr("Using 'function' as a type annotation is not supported, consider replacing with a function type annotation e.g. '(...any) -> " + "...any'")), {}}; } else @@ -1645,7 +1652,7 @@ AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack) // For a missing type annotation, capture 'space' between last token and the next one location = Location(lexer.previousLocation().end, lexer.current().location.begin); - return {reportTypeAnnotationError(location, {}, /*isMissing*/ true, "Expected type, got %s", lexer.current().toString().c_str()), {}}; + return {reportTypeAnnotationError(location, {}, /*isMissing*/ true, XorStr("Expected type, got %s"), lexer.current().toString().c_str()), {}}; } } @@ -1654,11 +1661,11 @@ AstTypePack* Parser::parseVariadicArgumentAnnotation() // Generic: a... if (lexer.current().type == Lexeme::Name && lexer.lookahead().type == Lexeme::Dot3) { - Name name = parseName("generic name"); + Name name = parseName(XorStr("generic name")); Location end = lexer.current().location; // This will not fail because of the lookahead guard. - expectAndConsume(Lexeme::Dot3, "generic type pack annotation"); + expectAndConsume(Lexeme::Dot3, XorStr("generic type pack annotation")); return allocator.alloc(Location(name.location, end), name.name); } // Variadic: T @@ -1682,11 +1689,11 @@ AstTypePack* Parser::parseTypePackAnnotation() // Generic: a... else if (lexer.current().type == Lexeme::Name && lexer.lookahead().type == Lexeme::Dot3) { - Name name = parseName("generic name"); + Name name = parseName(XorStr("generic name")); Location end = lexer.current().location; // This will not fail because of the lookahead guard. - expectAndConsume(Lexeme::Dot3, "generic type pack annotation"); + expectAndConsume(Lexeme::Dot3, XorStr("generic type pack annotation")); return allocator.alloc(Location(name.location, end), name.name); } @@ -1775,7 +1782,7 @@ std::optional Parser::checkUnaryConfusables() if (curr.type == '!') { - report(start, "Unexpected '!', did you mean 'not'?"); + report(start, XorStr("Unexpected '!', did you mean 'not'?")); return AstExprUnary::Not; } @@ -1797,20 +1804,20 @@ std::optional Parser::checkBinaryConfusables(const BinaryOpPr if (curr.type == '&' && next.type == '&' && curr.location.end == next.location.begin && binaryPriority[AstExprBinary::And].left > limit) { nextLexeme(); - report(Location(start, next.location), "Unexpected '&&', did you mean 'and'?"); + report(Location(start, next.location), XorStr("Unexpected '&&', did you mean 'and'?")); return AstExprBinary::And; } else if (curr.type == '|' && next.type == '|' && curr.location.end == next.location.begin && binaryPriority[AstExprBinary::Or].left > limit) { nextLexeme(); - report(Location(start, next.location), "Unexpected '||', did you mean 'or'?"); + report(Location(start, next.location), XorStr("Unexpected '||', did you mean 'or'?")); return AstExprBinary::Or; } else if (curr.type == '!' && next.type == '=' && curr.location.end == next.location.begin && binaryPriority[AstExprBinary::CompareNe].left > limit) { nextLexeme(); - report(Location(start, next.location), "Unexpected '!=', did you mean '~='?"); + report(Location(start, next.location), XorStr("Unexpected '!=', did you mean '~='?")); return AstExprBinary::CompareNe; } @@ -1832,7 +1839,7 @@ AstExpr* Parser::parseExpr(unsigned int limit) unsigned int recursionCounterOld = recursionCounter; // this handles recursive calls to parseSubExpr/parseExpr - incrementRecursionCounter("expression"); + incrementRecursionCounter(XorStr("expression")); const unsigned int unaryPriority = 8; @@ -1878,7 +1885,7 @@ AstExpr* Parser::parseExpr(unsigned int limit) op = checkBinaryConfusables(binaryPriority, limit); // note: while the parser isn't recursive here, we're generating recursive structures of unbounded depth - incrementRecursionCounter("expression"); + incrementRecursionCounter(XorStr("expression")); } recursionCounter = recursionCounterOld; @@ -1922,7 +1929,7 @@ AstExpr* Parser::parsePrefixExpr() if (lexer.current().type != ')') { - const char* suggestion = (lexer.current().type == '=') ? "; did you mean to use '{' when defining a table?" : nullptr; + const char* suggestion = (lexer.current().type == '=') ? XorStr("; did you mean to use '{' when defining a table?") : nullptr; expectMatchAndConsumeFail(static_cast(')'), matchParen, suggestion); @@ -1937,7 +1944,7 @@ AstExpr* Parser::parsePrefixExpr() } else { - return parseNameExpr("expression"); + return parseNameExpr(XorStr("expression")); } } @@ -1979,7 +1986,7 @@ AstExpr* Parser::parsePrimaryExpr(bool asStatement) Position opPosition = lexer.current().location.begin; nextLexeme(); - Name index = parseIndexName("method name", opPosition); + Name index = parseIndexName(XorStr("method name"), opPosition); AstExpr* func = allocator.alloc(Location(start, index.location), expr, index.name, index.location, opPosition, ':'); expr = parseFunctionArgs(func, true, index.location); @@ -1990,8 +1997,8 @@ AstExpr* Parser::parsePrimaryExpr(bool asStatement) if (!asStatement && expr->location.end.line != lexer.current().location.begin.line) { report(lexer.current().location, - "Ambiguous syntax: this looks like an argument list for a function call, but could also be a start of " - "new statement; use ';' to separate statements"); + XorStr("Ambiguous syntax: this looks like an argument list for a function call, but could also be a start of " + "new statement; use ';' to separate statements")); break; } @@ -2008,7 +2015,7 @@ AstExpr* Parser::parsePrimaryExpr(bool asStatement) } // note: while the parser isn't recursive here, we're generating recursive structures of unbounded depth - incrementRecursionCounter("expression"); + incrementRecursionCounter(XorStr("expression")); } recursionCounter = recursionCounterOld; @@ -2046,7 +2053,7 @@ static const char* parseInteger(double& result, const char* data, int base) if (errno == ERANGE) { - if (DFFlag::LuaReportParseIntegerIssues) + if (DFFlag::LlaReportParseIntegerIssues) { if (base == 2) lua_telemetry_parsed_out_of_range_bin_integer = true; @@ -2054,8 +2061,8 @@ static const char* parseInteger(double& result, const char* data, int base) lua_telemetry_parsed_out_of_range_hex_integer = true; } - if (FFlag::LuauErrorParseIntegerIssues) - return "Integer number value is out of range"; + if (FFlag::LluErrorParseIntegerIssues) + return XorStr("Integer number value is out of range"); } } @@ -2072,10 +2079,10 @@ static const char* parseNumber(double& result, const char* data) // hexadecimal literal if (data[0] == '0' && (data[1] == 'x' || data[1] == 'X') && data[2]) { - if (DFFlag::LuaReportParseIntegerIssues && data[2] == '0' && (data[3] == 'x' || data[3] == 'X')) + if (DFFlag::LlaReportParseIntegerIssues && data[2] == '0' && (data[3] == 'x' || data[3] == 'X')) lua_telemetry_parsed_double_prefix_hex_integer = true; - if (FFlag::LuauErrorParseIntegerIssues) + if (FFlag::LluErrorParseIntegerIssues) return parseInteger(result, data, 16); // keep prefix, it's handled by 'strtoull' else return parseInteger(result, data + 2, 16); @@ -2158,7 +2165,7 @@ AstExpr* Parser::parseSimpleExpr() scratchData.erase(std::remove(scratchData.begin(), scratchData.end(), '_'), scratchData.end()); } - if (DFFlag::LuaReportParseIntegerIssues || FFlag::LuauErrorParseIntegerIssues) + if (DFFlag::LlaReportParseIntegerIssues || FFlag::LluErrorParseIntegerIssues) { double value = 0; if (const char* error = parseNumber(value, scratchData.c_str())) @@ -2187,7 +2194,7 @@ AstExpr* Parser::parseSimpleExpr() { nextLexeme(); - return reportExprError(start, {}, "Malformed number"); + return reportExprError(start, {}, XorStr("Malformed number")); } } } @@ -2198,7 +2205,7 @@ AstExpr* Parser::parseSimpleExpr() else if (lexer.current().type == Lexeme::BrokenString) { nextLexeme(); - return reportExprError(start, {}, "Malformed string"); + return reportExprError(start, {}, XorStr("Malformed string")); } else if (lexer.current().type == Lexeme::Dot3) { @@ -2212,7 +2219,7 @@ AstExpr* Parser::parseSimpleExpr() { nextLexeme(); - return reportExprError(start, {}, "Cannot use '...' outside of a vararg function"); + return reportExprError(start, {}, XorStr("Cannot use '...' outside of a vararg function")); } } else if (lexer.current().type == '{') @@ -2237,8 +2244,8 @@ AstExpr* Parser::parseFunctionArgs(AstExpr* func, bool self, const Location& sel Position argStart = lexer.current().location.end; if (func->location.end.line != lexer.current().location.begin.line) { - report(lexer.current().location, "Ambiguous syntax: this looks like an argument list for a function call, but could also be a start of " - "new statement; use ';' to separate statements"); + report(lexer.current().location, XorStr("Ambiguous syntax: this looks like an argument list for a function call, but could also be a start of " + "new statement; use ';' to separate statements")); } Lexeme matchParen = lexer.current(); @@ -2275,12 +2282,12 @@ AstExpr* Parser::parseFunctionArgs(AstExpr* func, bool self, const Location& sel { if (self && lexer.current().location.begin.line != func->location.end.line) { - return reportExprError(func->location, copy({func}), "Expected function call arguments after '('"); + return reportExprError(func->location, copy({func}), XorStr("Expected function call arguments after '('")); } else { return reportExprError(Location(func->location.begin, lexer.current().location.begin), copy({func}), - "Expected '(', '{' or when parsing function call, got %s", lexer.current().toString().c_str()); + XorStr("Expected '(', '{' or when parsing function call, got %s"), lexer.current().toString().c_str()); } } } @@ -2296,7 +2303,7 @@ AstExpr* Parser::parseTableConstructor() Location start = lexer.current().location; Lexeme matchBrace = lexer.current(); - expectAndConsume('{', "table literal"); + expectAndConsume('{', XorStr("table literal")); while (lexer.current().type != '}') { @@ -2309,7 +2316,7 @@ AstExpr* Parser::parseTableConstructor() expectMatchAndConsume(']', matchLocationBracket); - expectAndConsume('=', "table field"); + expectAndConsume('=', XorStr("table field")); AstExpr* value = parseExpr(); @@ -2317,9 +2324,9 @@ AstExpr* Parser::parseTableConstructor() } else if (lexer.current().type == Lexeme::Name && lexer.lookahead().type == '=') { - Name name = parseName("table field"); + Name name = parseName(XorStr("table field")); - expectAndConsume('=', "table field"); + expectAndConsume('=', XorStr("table field")); AstArray nameString; nameString.data = const_cast(name.name.value); @@ -2368,7 +2375,7 @@ AstExpr* Parser::parseIfElseExpr() AstExpr* condition = parseExpr(); - bool hasThen = expectAndConsume(Lexeme::ReservedThen, "if then else expression"); + bool hasThen = expectAndConsume(Lexeme::ReservedThen, XorStr("if then else expression")); AstExpr* trueExpr = parseExpr(); AstExpr* falseExpr = nullptr; @@ -2376,14 +2383,14 @@ AstExpr* Parser::parseIfElseExpr() if (lexer.current().type == Lexeme::ReservedElseif) { unsigned int oldRecursionCount = recursionCounter; - incrementRecursionCounter("expression"); + incrementRecursionCounter(XorStr("expression")); hasElse = true; falseExpr = parseIfElseExpr(); recursionCounter = oldRecursionCount; } else { - hasElse = expectAndConsume(Lexeme::ReservedElse, "if then else expression"); + hasElse = expectAndConsume(Lexeme::ReservedElse, XorStr("if then else expression")); falseExpr = parseExpr(); } @@ -2464,7 +2471,7 @@ std::pair, AstArray> Parser::parseG seenPack = true; if (lexer.current().type != Lexeme::Dot3) - report(lexer.current().location, "Generic types come before generic type packs"); + report(lexer.current().location, XorStr("Generic types come before generic type packs")); else nextLexeme(); @@ -2486,7 +2493,7 @@ std::pair, AstArray> Parser::parseG auto [type, typePack] = parseTypeOrPackAnnotation(); if (type) - report(Location(packBegin.location.begin, lexer.previousLocation().end), "Expected type pack after '=', got type"); + report(Location(packBegin.location.begin, lexer.previousLocation().end), XorStr("Expected type pack after '=', got type")); namePacks.push_back({name, nameLocation, typePack}); } @@ -2494,7 +2501,7 @@ std::pair, AstArray> Parser::parseG else { if (seenDefault) - report(lexer.current().location, "Expected default type pack after type pack name"); + report(lexer.current().location, XorStr("Expected default type pack after type pack name")); namePacks.push_back({name, nameLocation, nullptr}); } @@ -2513,7 +2520,7 @@ std::pair, AstArray> Parser::parseG else { if (seenDefault) - report(lexer.current().location, "Expected default type after type name"); + report(lexer.current().location, XorStr("Expected default type after type name")); names.push_back({name, nameLocation, nullptr}); } @@ -2582,7 +2589,7 @@ AstArray Parser::parseTypeParams() std::optional> Parser::parseCharArray() { - LUAU_ASSERT(lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::RawString); + lluz_ASSERT(lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::RawString); scratchData.assign(lexer.current().data, lexer.current().length); @@ -2610,7 +2617,7 @@ AstExpr* Parser::parseString() if (std::optional> value = parseCharArray()) return allocator.alloc(location, *value); else - return reportExprError(location, {}, "String literal contains malformed escape sequence"); + return reportExprError(location, {}, XorStr("String literal contains malformed escape sequence")); } AstLocal* Parser::pushLocal(const Binding& binding) @@ -2671,17 +2678,17 @@ bool Parser::expectAndConsume(Lexeme::Type type, const char* context) } } -// LUAU_NOINLINE is used to limit the stack cost of this function due to std::string objects, and to increase caller performance since this code is +// lluz_NOINLINE is used to limit the stack cost of this function due to std::string objects, and to increase caller performance since this code is // cold -LUAU_NOINLINE void Parser::expectAndConsumeFail(Lexeme::Type type, const char* context) +lluz_NOINLINE void Parser::expectAndConsumeFail(Lexeme::Type type, const char* context) { std::string typeString = Lexeme(Location(Position(0, 0), 0), type).toString(); std::string currLexemeString = lexer.current().toString(); if (context) - report(lexer.current().location, "Expected %s when parsing %s, got %s", typeString.c_str(), context, currLexemeString.c_str()); + report(lexer.current().location, XorStr("Expected %s when parsing %s, got %s"), typeString.c_str(), context, currLexemeString.c_str()); else - report(lexer.current().location, "Expected %s, got %s", typeString.c_str(), currLexemeString.c_str()); + report(lexer.current().location, XorStr("Expected %s, got %s"), typeString.c_str(), currLexemeString.c_str()); } bool Parser::expectMatchAndConsume(char value, const Lexeme& begin, bool searchForMissing) @@ -2737,18 +2744,18 @@ bool Parser::expectMatchAndConsume(char value, const Lexeme& begin, bool searchF } } -// LUAU_NOINLINE is used to limit the stack cost of this function due to std::string objects, and to increase caller performance since this code is +// lluz_NOINLINE is used to limit the stack cost of this function due to std::string objects, and to increase caller performance since this code is // cold -LUAU_NOINLINE void Parser::expectMatchAndConsumeFail(Lexeme::Type type, const Lexeme& begin, const char* extra) +lluz_NOINLINE void Parser::expectMatchAndConsumeFail(Lexeme::Type type, const Lexeme& begin, const char* extra) { std::string typeString = Lexeme(Location(Position(0, 0), 0), type).toString(); if (lexer.current().location.begin.line == begin.location.begin.line) - report(lexer.current().location, "Expected %s (to close %s at column %d), got %s%s", typeString.c_str(), begin.toString().c_str(), - begin.location.begin.column + 1, lexer.current().toString().c_str(), extra ? extra : ""); + report(lexer.current().location, XorStr("Expected %s (to close %s at column %d), got %s%s"), typeString.c_str(), begin.toString().c_str(), + begin.location.begin.column + 1, lexer.current().toString().c_str(), extra ? extra : XorStr("")); else - report(lexer.current().location, "Expected %s (to close %s at line %d), got %s%s", typeString.c_str(), begin.toString().c_str(), - begin.location.begin.line + 1, lexer.current().toString().c_str(), extra ? extra : ""); + report(lexer.current().location, XorStr("Expected %s (to close %s at line %d), got %s%s"), typeString.c_str(), begin.toString().c_str(), + begin.location.begin.line + 1, lexer.current().toString().c_str(), extra ? extra : XorStr("")); } bool Parser::expectMatchEndAndConsume(Lexeme::Type type, const Lexeme& begin) @@ -2786,14 +2793,14 @@ bool Parser::expectMatchEndAndConsume(Lexeme::Type type, const Lexeme& begin) } } -// LUAU_NOINLINE is used to limit the stack cost of this function due to std::string objects, and to increase caller performance since this code is +// lluz_NOINLINE is used to limit the stack cost of this function due to std::string objects, and to increase caller performance since this code is // cold -LUAU_NOINLINE void Parser::expectMatchEndAndConsumeFail(Lexeme::Type type, const Lexeme& begin) +lluz_NOINLINE void Parser::expectMatchEndAndConsumeFail(Lexeme::Type type, const Lexeme& begin) { if (endMismatchSuspect.type != Lexeme::Eof && endMismatchSuspect.location.begin.line > begin.location.begin.line) { std::string suggestion = - format("; did you forget to close %s at line %d?", endMismatchSuspect.toString().c_str(), endMismatchSuspect.location.begin.line + 1); + format(XorStr("; did you forget to close %s at line %d?"), endMismatchSuspect.toString().c_str(), endMismatchSuspect.location.begin.line + 1); expectMatchAndConsumeFail(type, begin, suggestion.c_str()); } @@ -2844,9 +2851,9 @@ void Parser::incrementRecursionCounter(const char* context) { recursionCounter++; - if (recursionCounter > unsigned(FInt::LuauRecursionLimit)) + if (recursionCounter > unsigned(FInt::LluRecursionLimit)) { - ParseError::raise(lexer.current().location, "Exceeded allowed recursion depth; simplify your %s to make the code compile", context); + ParseError::raise(lexer.current().location, XorStr("Exceeded allowed recursion depth; simplify your %s to make the code compile"), context); } } @@ -2860,13 +2867,13 @@ void Parser::report(const Location& location, const char* format, va_list args) std::string message = vformat(format, args); // when limited to a single error, behave as if the error recovery is disabled - if (FInt::LuauParseErrorLimit == 1) + if (FInt::LluParseErrorLimit == 1) throw ParseError(location, message); parseErrors.emplace_back(location, message); - if (parseErrors.size() >= unsigned(FInt::LuauParseErrorLimit)) - ParseError::raise(location, "Reached error limit (%d)", int(FInt::LuauParseErrorLimit)); + if (parseErrors.size() >= unsigned(FInt::LluParseErrorLimit)) + ParseError::raise(location, XorStr("Reached error limit (%d)"), int(FInt::LluParseErrorLimit)); } void Parser::report(const Location& location, const char* format, ...) @@ -2877,12 +2884,12 @@ void Parser::report(const Location& location, const char* format, ...) va_end(args); } -LUAU_NOINLINE void Parser::reportNameError(const char* context) +lluz_NOINLINE void Parser::reportNameError(const char* context) { if (context) - report(lexer.current().location, "Expected identifier when parsing %s, got %s", context, lexer.current().toString().c_str()); + report(lexer.current().location, XorStr("Expected identifier when parsing %s, got %s"), context, lexer.current().toString().c_str()); else - report(lexer.current().location, "Expected identifier, got %s", lexer.current().toString().c_str()); + report(lexer.current().location, XorStr("Expected identifier, got %s"), lexer.current().toString().c_str()); } AstStatError* Parser::reportStatError( @@ -2918,34 +2925,37 @@ AstTypeError* Parser::reportTypeAnnotationError(const Location& location, const void Parser::nextLexeme() { - Lexeme::Type type = lexer.next(/* skipComments= */ false, true).type; - - while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment) + if (options.captureComments) { - const Lexeme& lexeme = lexer.current(); + Lexeme::Type type = lexer.next(/* skipComments= */ false, true).type; - if (options.captureComments) + while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment) + { + const Lexeme& lexeme = lexer.current(); commentLocations.push_back(Comment{lexeme.type, lexeme.location}); - // Subtlety: Broken comments are weird because we record them as comments AND pass them to the parser as a lexeme. - // The parser will turn this into a proper syntax error. - if (lexeme.type == Lexeme::BrokenComment) - return; + // Subtlety: Broken comments are weird because we record them as comments AND pass them to the parser as a lexeme. + // The parser will turn this into a proper syntax error. + if (lexeme.type == Lexeme::BrokenComment) + return; - // Comments starting with ! are called "hot comments" and contain directives for type checking / linting / compiling - if (lexeme.type == Lexeme::Comment && lexeme.length && lexeme.data[0] == '!') - { - const char* text = lexeme.data; + // Comments starting with ! are called "hot comments" and contain directives for type checking / linting + if (lexeme.type == Lexeme::Comment && lexeme.length && lexeme.data[0] == '!') + { + const char* text = lexeme.data; - unsigned int end = lexeme.length; - while (end > 0 && isSpace(text[end - 1])) - --end; + unsigned int end = lexeme.length; + while (end > 0 && isSpace(text[end - 1])) + --end; - hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)}); + hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)}); + } + + type = lexer.next(/* skipComments= */ false, /* updatePrevLocation= */ false).type; } - - type = lexer.next(/* skipComments= */ false, /* updatePrevLocation= */ false).type; } + else + lexer.next(); } -} // namespace Luau +} // namespace lluz diff --git a/Ast/src/StringUtils.cpp b/Ast/src/StringUtils.cpp index 0dc3f3f5..9c527d3a 100644 --- a/Ast/src/StringUtils.cpp +++ b/Ast/src/StringUtils.cpp @@ -1,14 +1,16 @@ -// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/StringUtils.h" +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#include "lluz/StringUtils.h" -#include "Luau/Common.h" +#include "lluz/Common.h" + +#include "..\..\..\..\Security\XorString.h" #include #include #include #include -namespace Luau +namespace lluz { void vformatAppend(std::string& ret, const char* fmt, va_list args) @@ -59,7 +61,7 @@ template static std::string joinImpl(const std::vector& segments, std::string_view delimiter) { if (segments.empty()) - return ""; + return XorStr(""); size_t len = (segments.size() - 1) * delimiter.size(); for (const auto& sv : segments) @@ -81,7 +83,7 @@ static std::string joinImpl(const std::vector& segments, std::string_vie dest += it->size(); } - LUAU_ASSERT(dest == result.data() + len); + lluz_ASSERT(dest == result.data() + len); return result; } @@ -276,11 +278,11 @@ std::string escape(std::string_view s) r += '\\'; break; default: - Luau::formatAppend(r, "%03u", c); + lluz::formatAppend(r, "%03u", c); } } } return r; } -} // namespace Luau +} // namespace lluz diff --git a/Ast/src/TimeTrace.cpp b/Ast/src/TimeTrace.cpp index e3807683..274d5b10 100644 --- a/Ast/src/TimeTrace.cpp +++ b/Ast/src/TimeTrace.cpp @@ -1,7 +1,7 @@ -// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/TimeTrace.h" +// This file is part of the lluz programming language and is licensed under MIT License; see LICENSE.txt for details +#include "lluz/TimeTrace.h" -#include "Luau/StringUtils.h" +#include "lluz/StringUtils.h" #include #include @@ -24,9 +24,10 @@ #endif #include +#include "../../../../Security/Lazy_Importer.h" -LUAU_FASTFLAGVARIABLE(DebugLuauTimeTracing, false) -namespace Luau +lluz_FASTFLAGVARIABLE(DebugLluTimeTracing, false) +namespace lluz { namespace TimeTrace { @@ -34,7 +35,7 @@ static double getClockPeriod() { #if defined(_WIN32) LARGE_INTEGER result = {}; - QueryPerformanceFrequency(&result); + LI_FN(QueryPerformanceFrequency).in(LI_MODULE("kernel32.dll").cached())(&result); return 1.0 / double(result.QuadPart); #elif defined(__APPLE__) mach_timebase_info_data_t result = {}; @@ -51,7 +52,7 @@ static double getClockTimestamp() { #if defined(_WIN32) LARGE_INTEGER result = {}; - QueryPerformanceCounter(&result); + LI_FN(QueryPerformanceCounter).in(LI_MODULE("kernel32.dll").cached())(&result); return double(result.QuadPart); #elif defined(__APPLE__) return double(mach_absolute_time()); @@ -80,11 +81,11 @@ uint32_t getClockMicroseconds() return uint32_t((getClockTimestamp() - start) * period); } } // namespace TimeTrace -} // namespace Luau +} // namespace lluz -#if defined(LUAU_ENABLE_TIME_TRACE) +#if defined(lluz_ENABLE_TIME_TRACE) -namespace Luau +namespace lluz { namespace TimeTrace { @@ -122,7 +123,7 @@ uint16_t createToken(GlobalContext& context, const char* name, const char* categ { std::scoped_lock lock(context.mutex); - LUAU_ASSERT(context.tokens.size() < 64 * 1024); + lluz_ASSERT(context.tokens.size() < 64 * 1024); context.tokens.push_back({name, category}); return uint16_t(context.tokens.size() - 1); @@ -151,12 +152,12 @@ void flushEvents(GlobalContext& context, uint32_t threadId, const std::vector