From ec481695a3bc82c66477ee74206d77e8bc871f9b Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 9 Feb 2022 09:19:50 -0800 Subject: [PATCH 01/25] Update library.md (#352) Clarify the relationship between `typeof` and `newproxy`. As a sandboxing measure, `typeof` only uses `__type` on host-defined userdata. Fixes #351. --- docs/_pages/library.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/_pages/library.md b/docs/_pages/library.md index 834bd73b..7695762d 100644 --- a/docs/_pages/library.md +++ b/docs/_pages/library.md @@ -128,7 +128,8 @@ Returns the type of the object, which is one of `"nil"`, `"boolean"`, `"number"` function typeof(obj: any): string ``` -Returns the type of the object; for userdata objects that have a metatable with the `__type` field, returns the value for that key. +Returns the type of the object; for userdata objects that have a metatable with the `__type` field *and* are defined by the host with an internal tag, returns the value for that key. +For custom userdata objects, such as ones returned by `newproxy`, this function returns `"userdata"` to make sure host-defined types can not be spoofed. ``` function ipairs(t: table): From abe3f87b483b323e04a378c56b95a3757b8853e7 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 9 Feb 2022 09:27:01 -0800 Subject: [PATCH 02/25] docs: Add documentation for upcoming MisleadingAndOr lint (#349) This is going to be part of Luau 0.514 --- docs/_pages/lint.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/_pages/lint.md b/docs/_pages/lint.md index ff677829..de4d247e 100644 --- a/docs/_pages/lint.md +++ b/docs/_pages/lint.md @@ -294,6 +294,7 @@ table.insert(t, 0, 42) -- table.insert uses index 0 but arrays are 1-based; did table.insert(t, #t+1, 42) -- table.insert will append the value to the table; consider removing the second argument for efficiency ``` + ## DuplicateCondition (24) When checking multiple conditions via `and/or` or `if/elseif`, a copy & paste error may result in checking the same condition redundantly. This almost always indicates a bug, so a warning is emitted when use of a duplicate condition is detected. @@ -301,3 +302,18 @@ When checking multiple conditions via `and/or` or `if/elseif`, a copy & paste er ```lua assert(self._adorns[normID1] and self._adorns[normID1]) -- Condition has already been checked on column 8 ``` + +## MisleadingAndOr (25) + +In Lua, there is no first-class ternary operator but it can be emulated via `a and b or c` pattern. However, due to how boolean evaluation works, if `b` is `false` or `nil`, the resulting expression evaluates to `c` regardless of the value of `a`. Luau solves this problem with the `if a then b else c` expression; a warning is emitted for and-or expressions where the first alternative is `false` or `nil` because it's almost always a bug. + +```lua +-- The and-or expression always evaluates to the second alternative because the first alternative is false; consider using if-then-else expression instead +local x = flag and false or true +``` + +The code above can be rewritten as follows to avoid the warning and the associated bug: + +```lua +local x = if flag then false else true +``` From 5187e64f88953f34785ffe58acd0610ee5041f5f Mon Sep 17 00:00:00 2001 From: Alan Jeffrey <403333+asajeffrey@users.noreply.github.com> Date: Wed, 9 Feb 2022 17:14:29 -0600 Subject: [PATCH 03/25] Implement a prototype interpreter (#353) * First cut interpreter --- .github/workflows/prototyping.yml | 12 ++- prototyping/Examples.agda | 3 + prototyping/Examples/OpSem.agda | 11 +++ prototyping/Examples/Run.agda | 18 ++++ prototyping/Examples/SmokeTest.lua | 3 +- prototyping/Examples/SmokeTestOutput.lua | 12 +++ prototyping/Examples/Syntax.agda | 6 +- prototyping/FFI/Data/HaskellInt.agda | 14 ++++ prototyping/FFI/Data/Vector.agda | 15 +++- prototyping/Interpreter.agda | 39 +++++++++ prototyping/Luau/Addr.agda | 17 ++++ prototyping/Luau/Addr/ToString.agda | 8 ++ prototyping/Luau/Heap.agda | 48 +++++++++++ prototyping/Luau/OpSem.agda | 92 +++++++++++++++++++++ prototyping/Luau/Run.agda | 28 +++++++ prototyping/Luau/RuntimeError.agda | 21 +++++ prototyping/Luau/RuntimeError/ToString.agda | 18 ++++ prototyping/Luau/Substitution.agda | 30 +++++++ prototyping/Luau/Syntax.agda | 18 ++-- prototyping/Luau/Syntax/FromJSON.agda | 12 ++- prototyping/Luau/Syntax/ToString.agda | 16 +++- prototyping/Luau/Type.agda | 10 +++ prototyping/Luau/Value.agda | 15 ++++ prototyping/Luau/Value/ToString.agda | 10 +++ prototyping/Luau/Var.agda | 16 ++++ prototyping/Luau/Var/ToString.agda | 8 ++ prototyping/Properties.agda | 3 + prototyping/Properties/Dec.agda | 8 ++ prototyping/Properties/Remember.agda | 9 ++ prototyping/Properties/Step.agda | 58 +++++++++++++ 30 files changed, 546 insertions(+), 32 deletions(-) create mode 100644 prototyping/Examples/OpSem.agda create mode 100644 prototyping/Examples/Run.agda create mode 100644 prototyping/Examples/SmokeTestOutput.lua create mode 100644 prototyping/FFI/Data/HaskellInt.agda create mode 100644 prototyping/Interpreter.agda create mode 100644 prototyping/Luau/Addr.agda create mode 100644 prototyping/Luau/Addr/ToString.agda create mode 100644 prototyping/Luau/Heap.agda create mode 100644 prototyping/Luau/OpSem.agda create mode 100644 prototyping/Luau/Run.agda create mode 100644 prototyping/Luau/RuntimeError.agda create mode 100644 prototyping/Luau/RuntimeError/ToString.agda create mode 100644 prototyping/Luau/Substitution.agda create mode 100644 prototyping/Luau/Type.agda create mode 100644 prototyping/Luau/Value.agda create mode 100644 prototyping/Luau/Value/ToString.agda create mode 100644 prototyping/Luau/Var.agda create mode 100644 prototyping/Luau/Var/ToString.agda create mode 100644 prototyping/Properties.agda create mode 100644 prototyping/Properties/Dec.agda create mode 100644 prototyping/Properties/Remember.agda create mode 100644 prototyping/Properties/Step.agda diff --git a/.github/workflows/prototyping.yml b/.github/workflows/prototyping.yml index 1d5a36f5..02e021ee 100644 --- a/.github/workflows/prototyping.yml +++ b/.github/workflows/prototyping.yml @@ -7,10 +7,12 @@ on: paths: - '.github/workflows/**' - 'prototyping/**' + - 'Analysis/src/JsonEncoder.cpp' pull_request: paths: - '.github/workflows/**' - 'prototyping/**' + - 'Analysis/src/JsonEncoder.cpp' jobs: linux: @@ -40,16 +42,20 @@ jobs: - name: check examples working-directory: prototyping run: ~/.cabal/bin/agda Examples.agda - - name: build PrettyPrinter + - name: build executables working-directory: prototyping - run: ~/.cabal/bin/agda --compile --ghc-flag=-v PrettyPrinter.agda + run: | + ~/.cabal/bin/agda --compile PrettyPrinter.agda + ~/.cabal/bin/agda --compile Interpreter.agda - name: cmake configure run: cmake . - name: cmake build luau-ast run: cmake --build . --target Luau.Ast.CLI - name: run smoketest working-directory: prototyping - run: ../luau-ast Examples/SmokeTest.lua | ./PrettyPrinter > Examples/SmokeTestOutput.lua + run: | + ../luau-ast Examples/SmokeTest.lua | ./PrettyPrinter > Examples/SmokeTestOutput.lua + ../luau-ast Examples/SmokeTest.lua | ./Interpreter - name: diff smoketest working-directory: prototyping run: diff Examples/SmokeTest.lua Examples/SmokeTestOutput.lua diff --git a/prototyping/Examples.agda b/prototyping/Examples.agda index 4b576377..fe396eff 100644 --- a/prototyping/Examples.agda +++ b/prototyping/Examples.agda @@ -1,4 +1,7 @@ +{-# OPTIONS --rewriting #-} module Examples where import Examples.Syntax +import Examples.OpSem +import Examples.Run diff --git a/prototyping/Examples/OpSem.agda b/prototyping/Examples/OpSem.agda new file mode 100644 index 00000000..4043869e --- /dev/null +++ b/prototyping/Examples/OpSem.agda @@ -0,0 +1,11 @@ +module Examples.OpSem where + +open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; subst) +open import Luau.Syntax using (var; nil; local_←_; _∙_; done; return) +open import Luau.Heap using (emp) + +x = var "x" + +ex1 : emp ⊢ (local "x" ← nil ∙ return x ∙ done) ⟶ᴮ (return nil ∙ done) ⊣ emp +ex1 = subst + diff --git a/prototyping/Examples/Run.agda b/prototyping/Examples/Run.agda new file mode 100644 index 00000000..bfa79839 --- /dev/null +++ b/prototyping/Examples/Run.agda @@ -0,0 +1,18 @@ +{-# OPTIONS --rewriting #-} + +module Examples.Run where + +open import Agda.Builtin.Equality using (_≡_; refl) +open import Luau.Syntax using (nil; var; _$_; function_⟨_⟩_end; return; _∙_; done) +open import Luau.Value using (nil) +open import Luau.Run using (run; return) +open import Luau.Heap using (emp; lookup-next; next-emp; lookup-next-emp) + +import Agda.Builtin.Equality.Rewrite +{-# REWRITE lookup-next next-emp lookup-next-emp #-} + +x = var "x" +id = var "id" + +ex1 : (run (function "id" ⟨ "x" ⟩ return x ∙ done end ∙ return (id $ nil) ∙ done) ≡ return nil _) +ex1 = refl diff --git a/prototyping/Examples/SmokeTest.lua b/prototyping/Examples/SmokeTest.lua index b1b91e19..e1e37e32 100644 --- a/prototyping/Examples/SmokeTest.lua +++ b/prototyping/Examples/SmokeTest.lua @@ -8,5 +8,6 @@ local function comp(f) end end end -local id2 = id(id) +local id2 = comp(id)(id) local nil2 = id2(nil) +return id2(nil2) diff --git a/prototyping/Examples/SmokeTestOutput.lua b/prototyping/Examples/SmokeTestOutput.lua new file mode 100644 index 00000000..b1b91e19 --- /dev/null +++ b/prototyping/Examples/SmokeTestOutput.lua @@ -0,0 +1,12 @@ +local function id(x) + return x +end +local function comp(f) + return function(g) + return function(x) + return f(g(x)) + end + end +end +local id2 = id(id) +local nil2 = id2(nil) diff --git a/prototyping/Examples/Syntax.agda b/prototyping/Examples/Syntax.agda index d16dfebb..8af9bca8 100644 --- a/prototyping/Examples/Syntax.agda +++ b/prototyping/Examples/Syntax.agda @@ -2,7 +2,7 @@ module Examples.Syntax where open import Agda.Builtin.Equality using (_≡_; refl) open import FFI.Data.String using (_++_) -open import Luau.Syntax using (var; _$_; return; nil; function_⟨_⟩_end; _∙; _∙_) +open import Luau.Syntax using (var; _$_; return; nil; function_⟨_⟩_end; done; _∙_) open import Luau.Syntax.ToString using (exprToString; blockToString) f = var "f" @@ -12,11 +12,11 @@ ex1 : exprToString(f $ x) ≡ "f(x)" ex1 = refl -ex2 : blockToString(return nil ∙) ≡ +ex2 : blockToString(return nil ∙ done) ≡ "return nil" ex2 = refl -ex3 : blockToString(function "f" ⟨ "x" ⟩ return x ∙ end ∙ return f ∙) ≡ +ex3 : blockToString(function "f" ⟨ "x" ⟩ return x ∙ done end ∙ return f ∙ done) ≡ "local function f(x)\n" ++ " return x\n" ++ "end\n" ++ diff --git a/prototyping/FFI/Data/HaskellInt.agda b/prototyping/FFI/Data/HaskellInt.agda new file mode 100644 index 00000000..9ab0868e --- /dev/null +++ b/prototyping/FFI/Data/HaskellInt.agda @@ -0,0 +1,14 @@ +module FFI.Data.HaskellInt where + +open import Agda.Builtin.Int using (Int) + +{-# FOREIGN GHC import qualified Data.Int #-} + +postulate HaskellInt : Set +{-# COMPILE GHC HaskellInt = type Data.Int.Int #-} + +postulate + intToHaskellInt : Int → HaskellInt + haskellIntToInt : HaskellInt → Int +{-# COMPILE GHC intToHaskellInt = fromIntegral #-} +{-# COMPILE GHC haskellIntToInt = fromIntegral #-} diff --git a/prototyping/FFI/Data/Vector.agda b/prototyping/FFI/Data/Vector.agda index 8ef8d63c..2c5d1925 100644 --- a/prototyping/FFI/Data/Vector.agda +++ b/prototyping/FFI/Data/Vector.agda @@ -1,6 +1,10 @@ module FFI.Data.Vector where +open import Agda.Builtin.Equality using (_≡_) +open import Agda.Builtin.Int using (Int; pos; negsuc) +open import Agda.Builtin.Nat using (Nat) open import FFI.Data.Bool using (Bool; false; true) +open import FFI.Data.HaskellInt using (HaskellInt; haskellIntToInt; intToHaskellInt) open import FFI.Data.Maybe using (Maybe; just; nothing) {-# FOREIGN GHC import qualified Data.Vector #-} @@ -14,10 +18,20 @@ postulate null : ∀ {A} → (Vector A) → Bool unsafeHead : ∀ {A} → (Vector A) → A unsafeTail : ∀ {A} → (Vector A) → (Vector A) + length : ∀ {A} → (Vector A) → Nat + lookup : ∀ {A} → (Vector A) → Nat → (Maybe A) + snoc : ∀ {A} → (Vector A) → A → (Vector A) {-# COMPILE GHC empty = \_ -> Data.Vector.empty #-} {-# COMPILE GHC null = \_ -> Data.Vector.null #-} {-# COMPILE GHC unsafeHead = \_ -> Data.Vector.unsafeHead #-} {-# COMPILE GHC unsafeTail = \_ -> Data.Vector.unsafeTail #-} +{-# COMPILE GHC length = \_ -> (fromIntegral . Data.Vector.length) #-} +{-# COMPILE GHC lookup = \_ v -> ((v Data.Vector.!?) . fromIntegral) #-} +{-# COMPILE GHC snoc = \_ -> Data.Vector.snoc #-} + +postulate length-empty : ∀ {A} → (length (empty {A}) ≡ 0) +postulate lookup-snoc : ∀ {A} (x : A) (v : Vector A) → (lookup (snoc v x) (length v) ≡ just x) +postulate lookup-snoc-empty : ∀ {A} (x : A) → (lookup (snoc empty x) 0 ≡ just x) head : ∀ {A} → (Vector A) → (Maybe A) head vec with null vec @@ -28,4 +42,3 @@ tail : ∀ {A} → (Vector A) → Vector A tail vec with null vec tail vec | false = unsafeTail vec tail vec | true = empty - diff --git a/prototyping/Interpreter.agda b/prototyping/Interpreter.agda new file mode 100644 index 00000000..9184235e --- /dev/null +++ b/prototyping/Interpreter.agda @@ -0,0 +1,39 @@ +module Interpreter where + +open import Agda.Builtin.IO using (IO) +open import Agda.Builtin.Int using (pos) +open import Agda.Builtin.Unit using (⊤) + +open import FFI.IO using (getContents; putStrLn; _>>=_; _>>_) +open import FFI.Data.Aeson using (Value; eitherDecode) +open import FFI.Data.Either using (Left; Right) +open import FFI.Data.String using (String; _++_) +open import FFI.Data.Text.Encoding using (encodeUtf8) +open import FFI.System.Exit using (exitWith; ExitFailure) + +open import Luau.Syntax using (Block) +open import Luau.Syntax.FromJSON using (blockFromJSON) +open import Luau.Syntax.ToString using (blockToString) +open import Luau.Run using (run; return; done; error) +open import Luau.RuntimeError.ToString using (errToStringᴮ) +open import Luau.Value.ToString using (valueToString) + +runBlock : Block → IO ⊤ +runBlock block with run block +runBlock block | return V D = putStrLn (valueToString V) +runBlock block | done D = putStrLn "nil" +runBlock block | error E D = putStrLn (errToStringᴮ E) + +runJSON : Value → IO ⊤ +runJSON value with blockFromJSON(value) +runJSON value | (Left err) = putStrLn ("Luau error: " ++ err) >> exitWith (ExitFailure (pos 1)) +runJSON value | (Right block) = runBlock block + +runString : String → IO ⊤ +runString txt with eitherDecode (encodeUtf8 txt) +runString txt | (Left err) = putStrLn ("JSON error: " ++ err) >> exitWith (ExitFailure (pos 1)) +runString txt | (Right value) = runJSON value + +main : IO ⊤ +main = getContents >>= runString + diff --git a/prototyping/Luau/Addr.agda b/prototyping/Luau/Addr.agda new file mode 100644 index 00000000..c1978047 --- /dev/null +++ b/prototyping/Luau/Addr.agda @@ -0,0 +1,17 @@ +module Luau.Addr where + +open import Agda.Builtin.Bool using (true; false) +open import Agda.Builtin.Equality using (_≡_) +open import Agda.Builtin.Nat using (Nat; _==_) +open import Agda.Builtin.String using (String) +open import Agda.Builtin.TrustMe using (primTrustMe) +open import Properties.Dec using (Dec; yes; no; ⊥) + +Addr : Set +Addr = Nat + +_≡ᴬ_ : (a b : Addr) → Dec (a ≡ b) +a ≡ᴬ b with a == b +a ≡ᴬ b | false = no p where postulate p : (a ≡ b) → ⊥ +a ≡ᴬ b | true = yes primTrustMe + diff --git a/prototyping/Luau/Addr/ToString.agda b/prototyping/Luau/Addr/ToString.agda new file mode 100644 index 00000000..2fc38335 --- /dev/null +++ b/prototyping/Luau/Addr/ToString.agda @@ -0,0 +1,8 @@ +module Luau.Addr.ToString where + +open import Agda.Builtin.String using (String; primStringAppend) +open import Luau.Addr using (Addr) +open import Agda.Builtin.Int using (Int; primShowInteger; pos) + +addrToString : Addr → String +addrToString a = primStringAppend "a" (primShowInteger (pos a)) diff --git a/prototyping/Luau/Heap.agda b/prototyping/Luau/Heap.agda new file mode 100644 index 00000000..1a0416e4 --- /dev/null +++ b/prototyping/Luau/Heap.agda @@ -0,0 +1,48 @@ +module Luau.Heap where + +open import Agda.Builtin.Equality using (_≡_) +open import FFI.Data.Maybe using (Maybe; just) +open import FFI.Data.Vector using (Vector; length; snoc; empty) +open import Luau.Addr using (Addr) +open import Luau.Var using (Var) +open import Luau.Syntax using (Block; Expr; nil; addr; function⟨_⟩_end) + +data HeapValue : Set where + function_⟨_⟩_end : Var → Var → Block → HeapValue + +Heap = Vector HeapValue + +data _≡_⊕_↦_ : Heap → Heap → Addr → HeapValue → Set where + + defn : ∀ {H val} → + + ----------------------------------- + (snoc H val) ≡ H ⊕ (length H) ↦ val + +lookup : Heap → Addr → Maybe HeapValue +lookup = FFI.Data.Vector.lookup + +emp : Heap +emp = empty + +data AllocResult (H : Heap) (V : HeapValue) : Set where + ok : ∀ a H′ → (H′ ≡ H ⊕ a ↦ V) → AllocResult H V + +alloc : ∀ H V → AllocResult H V +alloc H V = ok (length H) (snoc H V) defn + +next : Heap → Addr +next = length + +allocated : Heap → HeapValue → Heap +allocated = snoc + +-- next-emp : (length empty ≡ 0) +next-emp = FFI.Data.Vector.length-empty + +-- lookup-next : ∀ V H → (lookup (allocated H V) (next H) ≡ just V) +lookup-next = FFI.Data.Vector.lookup-snoc + +-- lookup-next-emp : ∀ V → (lookup (allocated emp V) 0 ≡ just V) +lookup-next-emp = FFI.Data.Vector.lookup-snoc-empty + diff --git a/prototyping/Luau/OpSem.agda b/prototyping/Luau/OpSem.agda new file mode 100644 index 00000000..dcd474bf --- /dev/null +++ b/prototyping/Luau/OpSem.agda @@ -0,0 +1,92 @@ +module Luau.OpSem where + +open import Agda.Builtin.Equality using (_≡_) +open import FFI.Data.Maybe using (just) +open import Luau.Heap using (Heap; _≡_⊕_↦_; lookup; function_⟨_⟩_end) +open import Luau.Substitution using (_[_/_]ᴮ) +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function⟨_⟩_end; _$_; block_is_end; local_←_; _∙_; done; function_⟨_⟩_end; return) +open import Luau.Value using (addr; val) + +data _⊢_⟶ᴮ_⊣_ : Heap → Block → Block → Heap → Set +data _⊢_⟶ᴱ_⊣_ : Heap → Expr → Expr → Heap → Set + +data _⊢_⟶ᴱ_⊣_ where + + nil : ∀ {H} → + + ------------------- + H ⊢ nil ⟶ᴱ nil ⊣ H + + function : ∀ {H H′ a x B} → + + H′ ≡ H ⊕ a ↦ (function "anon" ⟨ x ⟩ B end) → + ------------------------------------------- + H ⊢ (function⟨ x ⟩ B end) ⟶ᴱ (addr a) ⊣ H′ + + app : ∀ {H H′ M M′ N} → + + H ⊢ M ⟶ᴱ M′ ⊣ H′ → + ----------------------------- + H ⊢ (M $ N) ⟶ᴱ (M′ $ N) ⊣ H′ + + beta : ∀ {H M a f x B} → + + (lookup H a) ≡ just(function f ⟨ x ⟩ B end) → + ----------------------------------------------------- + H ⊢ (addr a $ M) ⟶ᴱ (block f is local x ← M ∙ B end) ⊣ H + + block : ∀ {H H′ B B′ b} → + + H ⊢ B ⟶ᴮ B′ ⊣ H′ → + ---------------------------------------------------- + H ⊢ (block b is B end) ⟶ᴱ (block b is B′ end) ⊣ H′ + + return : ∀ {H V B b} → + + -------------------------------------------------------- + H ⊢ (block b is return (val V) ∙ B end) ⟶ᴱ (val V) ⊣ H + + done : ∀ {H b} → + + --------------------------------- + H ⊢ (block b is done end) ⟶ᴱ nil ⊣ H + +data _⊢_⟶ᴮ_⊣_ where + + local : ∀ {H H′ x M M′ B} → + + H ⊢ M ⟶ᴱ M′ ⊣ H′ → + ------------------------------------------------- + H ⊢ (local x ← M ∙ B) ⟶ᴮ (local x ← M′ ∙ B) ⊣ H′ + + subst : ∀ {H x v B} → + + ------------------------------------------------- + H ⊢ (local x ← val v ∙ B) ⟶ᴮ (B [ v / x ]ᴮ) ⊣ H + + function : ∀ {H H′ a f x B C} → + + H′ ≡ H ⊕ a ↦ (function f ⟨ x ⟩ C end) → + -------------------------------------------------------------- + H ⊢ (function f ⟨ x ⟩ C end ∙ B) ⟶ᴮ (B [ addr a / f ]ᴮ) ⊣ H′ + + return : ∀ {H H′ M M′ B} → + + H ⊢ M ⟶ᴱ M′ ⊣ H′ → + -------------------------------------------- + H ⊢ (return M ∙ B) ⟶ᴮ (return M′ ∙ B) ⊣ H′ + +data _⊢_⟶*_⊣_ : Heap → Block → Block → Heap → Set where + + refl : ∀ {H B} → + + ---------------- + H ⊢ B ⟶* B ⊣ H + + step : ∀ {H H′ H″ B B′ B″} → + H ⊢ B ⟶ᴮ B′ ⊣ H′ → + H′ ⊢ B′ ⟶* B″ ⊣ H″ → + ------------------ + H ⊢ B ⟶* B″ ⊣ H″ + + diff --git a/prototyping/Luau/Run.agda b/prototyping/Luau/Run.agda new file mode 100644 index 00000000..29709ceb --- /dev/null +++ b/prototyping/Luau/Run.agda @@ -0,0 +1,28 @@ +module Luau.Run where + +open import Agda.Builtin.Equality using (_≡_; refl) +open import Luau.Heap using (Heap; emp) +open import Luau.Syntax using (Block; return; _∙_; done) +open import Luau.OpSem using (_⊢_⟶*_⊣_; refl; step) +open import Luau.Value using (val) +open import Properties.Step using (stepᴮ; step; return; done; error) +open import Luau.RuntimeError using (RuntimeErrorᴮ) + +data RunResult (H : Heap) (B : Block) : Set where + return : ∀ V {B′ H′} → (H ⊢ B ⟶* (return (val V) ∙ B′) ⊣ H′) → RunResult H B + done : ∀ {H′} → (H ⊢ B ⟶* done ⊣ H′) → RunResult H B + error : ∀ {B′ H′} → (RuntimeErrorᴮ H′ B′) → (H ⊢ B ⟶* B′ ⊣ H′) → RunResult H B + +{-# TERMINATING #-} +run′ : ∀ H B → RunResult H B +run′ H B with stepᴮ H B +run′ H B | step H′ B′ D with run′ H′ B′ +run′ H B | step H′ B′ D | return V D′ = return V (step D D′) +run′ H B | step H′ B′ D | done D′ = done (step D D′) +run′ H B | step H′ B′ D | error E D′ = error E (step D D′) +run′ H _ | return V refl = return V refl +run′ H _ | done refl = done refl +run′ H B | error E = error E refl + +run : ∀ B → RunResult emp B +run = run′ emp diff --git a/prototyping/Luau/RuntimeError.agda b/prototyping/Luau/RuntimeError.agda new file mode 100644 index 00000000..86e7cf21 --- /dev/null +++ b/prototyping/Luau/RuntimeError.agda @@ -0,0 +1,21 @@ +module Luau.RuntimeError where + +open import Agda.Builtin.Equality using (_≡_) +open import Luau.Heap using (Heap; lookup) +open import FFI.Data.Maybe using (just; nothing) +open import Luau.Syntax using (Block; Expr; nil; var; addr; function⟨_⟩_end; block_is_end; _$_; local_←_; function_⟨_⟩_end; return; done; _∙_) + +data RuntimeErrorᴮ (H : Heap) : Block → Set +data RuntimeErrorᴱ (H : Heap) : Expr → Set + +data RuntimeErrorᴱ H where + NilIsNotAFunction : ∀ {M} → RuntimeErrorᴱ H (nil $ M) + UnboundVariable : ∀ x → RuntimeErrorᴱ H (var x) + SEGV : ∀ a → (lookup H a ≡ nothing) → RuntimeErrorᴱ H (addr a) + app : ∀ {M N} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (M $ N) + block : ∀ b {B} → RuntimeErrorᴮ H B → RuntimeErrorᴱ H (block b is B end) + +data RuntimeErrorᴮ H where + local : ∀ x {M B} → RuntimeErrorᴱ H M → RuntimeErrorᴮ H (local x ← M ∙ B) + return : ∀ {M B} → RuntimeErrorᴱ H M → RuntimeErrorᴮ H (return M ∙ B) + diff --git a/prototyping/Luau/RuntimeError/ToString.agda b/prototyping/Luau/RuntimeError/ToString.agda new file mode 100644 index 00000000..f11756f7 --- /dev/null +++ b/prototyping/Luau/RuntimeError/ToString.agda @@ -0,0 +1,18 @@ +module Luau.RuntimeError.ToString where + +open import FFI.Data.String using (String; _++_) +open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; UnboundVariable; SEGV; app; block) +open import Luau.Addr.ToString using (addrToString) +open import Luau.Var.ToString using (varToString) + +errToStringᴱ : ∀ {H B} → RuntimeErrorᴱ H B → String +errToStringᴮ : ∀ {H B} → RuntimeErrorᴮ H B → String + +errToStringᴱ NilIsNotAFunction = "nil is not a function" +errToStringᴱ (UnboundVariable x) = "variable " ++ varToString x ++ " is unbound" +errToStringᴱ (SEGV a x) = "address " ++ addrToString a ++ " is unallocated" +errToStringᴱ (app E) = errToStringᴱ E +errToStringᴱ (block b E) = errToStringᴮ E ++ "\n in call of function " ++ varToString b + +errToStringᴮ (local x E) = errToStringᴱ E ++ "\n in definition of " ++ varToString x +errToStringᴮ (return E) = errToStringᴱ E ++ "\n in return statement" diff --git a/prototyping/Luau/Substitution.agda b/prototyping/Luau/Substitution.agda new file mode 100644 index 00000000..1a361174 --- /dev/null +++ b/prototyping/Luau/Substitution.agda @@ -0,0 +1,30 @@ +module Luau.Substitution where + +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function⟨_⟩_end; _$_; block_is_end; local_←_; _∙_; done; function_⟨_⟩_end; return) +open import Luau.Value using (Value; val) +open import Luau.Var using (Var; _≡ⱽ_) +open import Properties.Dec using (Dec; yes; no) + +_[_/_]ᴱ : Expr → Value → Var → Expr +_[_/_]ᴮ : Block → Value → Var → Block +var_[_/_]ᴱwhenever_ : ∀ {P} → Var → Value → Var → (Dec P) → Expr +_[_/_]ᴮunless_ : ∀ {P} → Block → Value → Var → (Dec P) → Block + +nil [ v / x ]ᴱ = nil +var y [ v / x ]ᴱ = var y [ v / x ]ᴱwhenever (x ≡ⱽ y) +addr a [ v / x ]ᴱ = addr a +(M $ N) [ v / x ]ᴱ = (M [ v / x ]ᴱ) $ (N [ v / x ]ᴱ) +function⟨ y ⟩ C end [ v / x ]ᴱ = function⟨ y ⟩ C [ v / x ]ᴮunless (x ≡ⱽ y) end +block b is C end [ v / x ]ᴱ = block b is C [ v / x ]ᴮ end + +(function f ⟨ y ⟩ C end ∙ B) [ v / x ]ᴮ = function f ⟨ y ⟩ (C [ v / x ]ᴮunless (x ≡ⱽ y)) end ∙ (B [ v / x ]ᴮunless (x ≡ⱽ f)) +(local y ← M ∙ B) [ v / x ]ᴮ = local y ← (M [ v / x ]ᴱ) ∙ (B [ v / x ]ᴮunless (x ≡ⱽ y)) +(return M ∙ B) [ v / x ]ᴮ = return (M [ v / x ]ᴱ) ∙ (B [ v / x ]ᴮ) +done [ v / x ]ᴮ = done + +var y [ v / x ]ᴱwhenever yes p = val v +var y [ v / x ]ᴱwhenever no p = var y + +B [ v / x ]ᴮunless yes p = B +B [ v / x ]ᴮunless no p = B [ v / x ]ᴮ + diff --git a/prototyping/Luau/Syntax.agda b/prototyping/Luau/Syntax.agda index c1d9c951..3fe05bc8 100644 --- a/prototyping/Luau/Syntax.agda +++ b/prototyping/Luau/Syntax.agda @@ -1,27 +1,17 @@ module Luau.Syntax where -open import Agda.Builtin.String using (String) +open import Luau.Var using (Var) +open import Luau.Addr using (Addr) infixr 5 _∙_ -data Type : Set where - nil : Type - _⇒_ : Type → Type → Type - none : Type - any : Type - _∪_ : Type → Type → Type - _∩_ : Type → Type → Type - -Var : Set -Var = String - data Block : Set data Stat : Set data Expr : Set data Block where _∙_ : Stat → Block → Block - _∙ : Stat → Block + done : Block data Stat where function_⟨_⟩_end : Var → Var → Block → Stat @@ -31,5 +21,7 @@ data Stat where data Expr where nil : Expr var : Var → Expr + addr : Addr → Expr _$_ : Expr → Expr → Expr function⟨_⟩_end : Var → Block → Expr + block_is_end : Var → Block → Expr diff --git a/prototyping/Luau/Syntax/FromJSON.agda b/prototyping/Luau/Syntax/FromJSON.agda index 36432118..0a4164fb 100644 --- a/prototyping/Luau/Syntax/FromJSON.agda +++ b/prototyping/Luau/Syntax/FromJSON.agda @@ -1,6 +1,6 @@ module Luau.Syntax.FromJSON where -open import Luau.Syntax using (Type; Block; Stat ; Expr; nil; _$_; var; function⟨_⟩_end; local_←_; function_⟨_⟩_end; return; _∙; _∙_) +open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; function⟨_⟩_end; local_←_; function_⟨_⟩_end; return; done; _∙_) open import Agda.Builtin.List using (List; _∷_; []) @@ -117,11 +117,9 @@ blockFromJSON (object obj) | nothing | _ = Left "AstStatBlock missing type" blockFromJSON _ = Left "AstBlock not an array or AstStatBlock object" blockFromArray arr with head arr -blockFromArray arr | nothing = Left "Block should be a non-empty array" +blockFromArray arr | nothing = Right done blockFromArray arr | just value with statFromJSON value blockFromArray arr | just value | Left err = Left err -blockFromArray arr | just value | Right S with null (tail arr) -blockFromArray arr | just value | Right S | true = Right (S ∙) -blockFromArray arr | just value | Right S | false with blockFromArray(tail arr) -blockFromArray arr | just value | Right S | false | Left err = Left (err) -blockFromArray arr | just value | Right S | false | Right B = Right (S ∙ B) +blockFromArray arr | just value | Right S with blockFromArray(tail arr) +blockFromArray arr | just value | Right S | Left err = Left (err) +blockFromArray arr | just value | Right S | Right B = Right (S ∙ B) diff --git a/prototyping/Luau/Syntax/ToString.agda b/prototyping/Luau/Syntax/ToString.agda index 87876565..afec0935 100644 --- a/prototyping/Luau/Syntax/ToString.agda +++ b/prototyping/Luau/Syntax/ToString.agda @@ -1,8 +1,9 @@ module Luau.Syntax.ToString where -open import Luau.Syntax using (Type; Block; Stat; Expr; nil; var; _$_; function⟨_⟩_end; return; function_⟨_⟩_end ;local_←_; _∙_; _∙) - +open import Luau.Syntax using (Block; Stat; Expr; nil; var; addr; _$_; function⟨_⟩_end; return; function_⟨_⟩_end ;local_←_; _∙_; done; block_is_end) open import FFI.Data.String using (String; _++_) +open import Luau.Addr.ToString using (addrToString) +open import Luau.Var.ToString using (varToString) exprToString′ : String → Expr → String statToString′ : String → Stat → String @@ -10,14 +11,20 @@ blockToString′ : String → Block → String exprToString′ lb nil = "nil" +exprToString′ lb (addr a) = + addrToString(a) exprToString′ lb (var x) = - x + varToString(x) exprToString′ lb (M $ N) = (exprToString′ lb M) ++ "(" ++ (exprToString′ lb N) ++ ")" exprToString′ lb (function⟨ x ⟩ B end) = "function(" ++ x ++ ")" ++ lb ++ " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ "end" +exprToString′ lb (block b is B end) = + "(function " ++ b ++ "()" ++ lb ++ + " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ + "end)()" statToString′ lb (function f ⟨ x ⟩ B end) = "local function " ++ f ++ "(" ++ x ++ ")" ++ lb ++ @@ -28,8 +35,9 @@ statToString′ lb (local x ← M) = statToString′ lb (return M) = "return " ++ (exprToString′ lb M) +blockToString′ lb (S ∙ done) = statToString′ lb S blockToString′ lb (S ∙ B) = statToString′ lb S ++ lb ++ blockToString′ lb B -blockToString′ lb (S ∙) = statToString′ lb S +blockToString′ lb (done) = "" exprToString : Expr → String exprToString = exprToString′ "\n" diff --git a/prototyping/Luau/Type.agda b/prototyping/Luau/Type.agda new file mode 100644 index 00000000..8da3a985 --- /dev/null +++ b/prototyping/Luau/Type.agda @@ -0,0 +1,10 @@ +module Luau.Type where + +data Type : Set where + nil : Type + _⇒_ : Type → Type → Type + none : Type + any : Type + _∪_ : Type → Type → Type + _∩_ : Type → Type → Type + diff --git a/prototyping/Luau/Value.agda b/prototyping/Luau/Value.agda new file mode 100644 index 00000000..855e663e --- /dev/null +++ b/prototyping/Luau/Value.agda @@ -0,0 +1,15 @@ +module Luau.Value where + +open import Luau.Addr using (Addr) +open import Luau.Syntax using (Block; Expr; nil; addr; function⟨_⟩_end) +open import Luau.Var using (Var) + +data Value : Set where + nil : Value + addr : Addr → Value + +val : Value → Expr +val nil = nil +val (addr a) = addr a + + diff --git a/prototyping/Luau/Value/ToString.agda b/prototyping/Luau/Value/ToString.agda new file mode 100644 index 00000000..3cac3ee7 --- /dev/null +++ b/prototyping/Luau/Value/ToString.agda @@ -0,0 +1,10 @@ +module Luau.Value.ToString where + +open import Agda.Builtin.String using (String) +open import Luau.Value using (Value; nil; addr) +open import Luau.Addr.ToString using (addrToString) + +valueToString : Value → String +valueToString nil = "nil" +valueToString (addr a) = addrToString a + diff --git a/prototyping/Luau/Var.agda b/prototyping/Luau/Var.agda new file mode 100644 index 00000000..091ce125 --- /dev/null +++ b/prototyping/Luau/Var.agda @@ -0,0 +1,16 @@ +module Luau.Var where + +open import Agda.Builtin.Bool using (true; false) +open import Agda.Builtin.Equality using (_≡_) +open import Agda.Builtin.String using (String; primStringEquality) +open import Agda.Builtin.TrustMe using (primTrustMe) +open import Properties.Dec using (Dec; yes; no; ⊥) + +Var : Set +Var = String + +_≡ⱽ_ : (a b : Var) → Dec (a ≡ b) +a ≡ⱽ b with primStringEquality a b +a ≡ⱽ b | false = no p where postulate p : (a ≡ b) → ⊥ +a ≡ⱽ b | true = yes primTrustMe + diff --git a/prototyping/Luau/Var/ToString.agda b/prototyping/Luau/Var/ToString.agda new file mode 100644 index 00000000..10cd915b --- /dev/null +++ b/prototyping/Luau/Var/ToString.agda @@ -0,0 +1,8 @@ +module Luau.Var.ToString where + +open import Agda.Builtin.String using (String) +open import Luau.Var using (Var) + +varToString : Var → String +varToString x = x + diff --git a/prototyping/Properties.agda b/prototyping/Properties.agda new file mode 100644 index 00000000..a3c685d1 --- /dev/null +++ b/prototyping/Properties.agda @@ -0,0 +1,3 @@ +module Properties where + +import Properties.Dec diff --git a/prototyping/Properties/Dec.agda b/prototyping/Properties/Dec.agda new file mode 100644 index 00000000..d89fd754 --- /dev/null +++ b/prototyping/Properties/Dec.agda @@ -0,0 +1,8 @@ +module Properties.Dec where + +data ⊥ : Set where + +data Dec(A : Set) : Set where + yes : A → Dec A + no : (A → ⊥) → Dec A + diff --git a/prototyping/Properties/Remember.agda b/prototyping/Properties/Remember.agda new file mode 100644 index 00000000..5058d594 --- /dev/null +++ b/prototyping/Properties/Remember.agda @@ -0,0 +1,9 @@ +module Properties.Remember where + +open import Agda.Builtin.Equality using (_≡_; refl) + +data Remember {A : Set} (a : A) : Set where + _,_ : ∀ b → (a ≡ b) → Remember(a) + +remember : ∀ {A} (a : A) → Remember(a) +remember a = (a , refl) diff --git a/prototyping/Properties/Step.agda b/prototyping/Properties/Step.agda new file mode 100644 index 00000000..0eddd9fe --- /dev/null +++ b/prototyping/Properties/Step.agda @@ -0,0 +1,58 @@ +module Properties.Step where + +open import Agda.Builtin.Equality using (_≡_; refl) +open import FFI.Data.Maybe using (just; nothing) +open import Luau.Heap using (Heap; lookup; alloc; ok; function_⟨_⟩_end) +open import Luau.Syntax using (Block; Expr; nil; var; addr; function⟨_⟩_end; block_is_end; _$_; local_←_; function_⟨_⟩_end; return; done; _∙_) +open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app ; beta; function; block; return; done; local; subst) +open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; UnboundVariable; SEGV; app; block; local; return) +open import Luau.Substitution using (_[_/_]ᴮ) +open import Luau.Value using (nil; addr; val) +open import Properties.Remember using (remember; _,_) + +data StepResultᴮ (H : Heap) (B : Block) : Set +data StepResultᴱ (H : Heap) (M : Expr) : Set + +data StepResultᴮ H B where + step : ∀ H′ B′ → (H ⊢ B ⟶ᴮ B′ ⊣ H′) → StepResultᴮ H B + return : ∀ V {B′} → (B ≡ (return (val V) ∙ B′)) → StepResultᴮ H B + done : (B ≡ done) → StepResultᴮ H B + error : (RuntimeErrorᴮ H B) → StepResultᴮ H B + +data StepResultᴱ H M where + step : ∀ H′ M′ → (H ⊢ M ⟶ᴱ M′ ⊣ H′) → StepResultᴱ H M + value : ∀ V → (M ≡ val V) → StepResultᴱ H M + error : (RuntimeErrorᴱ H M) → StepResultᴱ H M + +stepᴱ : ∀ H M → StepResultᴱ H M +stepᴮ : ∀ H B → StepResultᴮ H B + +stepᴱ H nil = value nil refl +stepᴱ H (var x) = error (UnboundVariable x) +stepᴱ H (addr a) = value (addr a) refl +stepᴱ H (M $ N) with stepᴱ H M +stepᴱ H (M $ N) | step H′ M′ D = step H′ (M′ $ N) (app D) +stepᴱ H (nil $ N) | value nil refl = error NilIsNotAFunction +stepᴱ H (addr a $ N) | value (addr a) refl with remember (lookup H a) +stepᴱ H (addr a $ N) | value (addr a) refl | (nothing , p) = error (app (SEGV a p)) +stepᴱ H (addr a $ N) | value (addr a) refl | (just(function f ⟨ x ⟩ B end) , p) = step H (block f is local x ← N ∙ B end) (beta p) +stepᴱ H (M $ N) | error E = error (app E) +stepᴱ H (function⟨ x ⟩ B end) with alloc H (function "anon" ⟨ x ⟩ B end) +stepᴱ H (function⟨ x ⟩ B end) | ok a H′ p = step H′ (addr a) (function p) +stepᴱ H (block b is B end) with stepᴮ H B +stepᴱ H (block b is B end) | step H′ B′ D = step H′ (block b is B′ end) (block D) +stepᴱ H (block b is (return _ ∙ B′) end) | return V refl = step H (val V) return +stepᴱ H (block b is done end) | done refl = step H nil done +stepᴱ H (block b is B end) | error E = error (block b E) + +stepᴮ H (function f ⟨ x ⟩ C end ∙ B) with alloc H (function f ⟨ x ⟩ C end) +stepᴮ H (function f ⟨ x ⟩ C end ∙ B) | ok a H′ p = step H′ (B [ addr a / f ]ᴮ) (function p) +stepᴮ H (local x ← M ∙ B) with stepᴱ H M +stepᴮ H (local x ← M ∙ B) | step H′ M′ D = step H′ (local x ← M′ ∙ B) (local D) +stepᴮ H (local x ← _ ∙ B) | value V refl = step H (B [ V / x ]ᴮ) subst +stepᴮ H (local x ← M ∙ B) | error E = error (local x E) +stepᴮ H (return M ∙ B) with stepᴱ H M +stepᴮ H (return M ∙ B) | step H′ M′ D = step H′ (return M′ ∙ B) (return D) +stepᴮ H (return _ ∙ B) | value V refl = return V refl +stepᴮ H (return M ∙ B) | error E = error (return E) +stepᴮ H done = done refl From aecd60371be8e54470adeebb7476370b7541c7e0 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Fri, 11 Feb 2022 09:13:27 -0800 Subject: [PATCH 04/25] Update performance.md (#355) Document weak table shrinking and paged sweeper --- docs/_pages/performance.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/_pages/performance.md b/docs/_pages/performance.md index 4e87ec5b..b4fd3a7b 100644 --- a/docs/_pages/performance.md +++ b/docs/_pages/performance.md @@ -167,3 +167,11 @@ While Luau uses an incremental garbage collector, once per each collector cycle Normally objects that have been modified after the GC marked them in an incremental mark phase need to be rescanned during atomic phase, so frequent modifications of existing tables may result in a slow atomic step. To address this, we run a "remark" step where we traverse objects that have been modified after being marked once more (incrementally); additionally, the write barrier that triggers for object modifications changes the transition logic during remark phase to reduce the probability that the object will need to be rescanned. Another source of scalability challenges is coroutines. Writes to coroutine stacks don't use a write barrier, since that's prohibitively expensive as they are too frequent. This means that coroutine stacks need to be traversed during atomic step, so applications with many coroutines suffer large atomic pauses. To address this, we implement incremental marking of coroutines: marking a coroutine makes it "inactive" and resuming a coroutine (or pushing extra objects on the coroutine stack via C API) makes it "active". Atomic step only needs to traverse active coroutines again, which reduces the cost of atomic step by effectively making coroutine collection incremental as well. + +While large tables can be a problem for incremental GC in general since currently marking a single object is indivisible, large weak tables are a unique challenge because they also need to be processed during atomic phase, and the main use case for weak tables - object caches - may result in tables with large capacity but few live objects in long-running applications that exhibit bursts of activity. To address this, weak tables in Luau can be marked as "shrinkable" by including `s` as part of `__mode` string, which results in weak tables being resized to the optimal capacity during GC. This option may result in missing keys during table iteration if the table is resized while iteration is in progress and as such is only recommended for use in specific circumstances. + +## Optimized garbage collector sweeping + +The incremental garbage collector in Luau runs three phases for each cycle: mark, atomic and sweep. Mark incrementally traverses all live objects, atomic finishes various operations that need to happen without mutator intervention (see previous section), and sweep traverses all objects in the heap, reclaiming memory used by dead objects and performing minor fixup for live objects. While objects allocated during the mark phase are traversed in the same cycle and thus may get reclaimed, objects allocated during the sweep phase are considered live. Because of this, the faster the sweep phase completes, the less garbage will accumulate; and, of course, the less time sweeping takes the less overhead there is from this phase of garbage collection on the process. + +Since sweeping traverses the whole heap, we maximize the efficiency of this traversal by allocating garbage-collected objects of the same size in 16 KB pages, and traversing each page at a time, which is otherwise known as a paged sweeper. This ensures good locality of reference as consecutively swept objects are contiugous in memory, and allows us to spend no memory for each object on sweep-related data or allocation metadata, since paged sweeper doesn't need to be able to free objects without knowing which page they are in. Compared to linked list based sweeping that Lua/LuaJIT implement, paged sweeper is 2-3x faster, and saves 16 bytes per object on 64-bit platforms. From 63d5423bbb283e602dd396a693a90318f4902d23 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Fri, 11 Feb 2022 11:02:09 -0800 Subject: [PATCH 05/25] Sync to upstream/release/514 (#357) --- Analysis/include/Luau/Autocomplete.h | 2 + Analysis/include/Luau/Linter.h | 1 + Analysis/include/Luau/Substitution.h | 2 +- Analysis/include/Luau/TxnLog.h | 4 +- Analysis/include/Luau/TypeInfer.h | 11 ++ Analysis/include/Luau/TypePack.h | 3 - Analysis/include/Luau/TypeVar.h | 3 - Analysis/src/EmbeddedBuiltinDefinitions.cpp | 12 +- Analysis/src/Linter.cpp | 103 ++++++++++++-- Analysis/src/Module.cpp | 28 ++-- Analysis/src/Scope.cpp | 4 + Analysis/src/TxnLog.cpp | 8 ++ Analysis/src/TypeInfer.cpp | 100 ++++++++++--- Analysis/src/TypeVar.cpp | 4 +- Analysis/src/Unifier.cpp | 125 +++++++++++------ Ast/src/Parser.cpp | 2 +- CLI/Repl.cpp | 68 ++++++--- CLI/Repl.h | 7 +- Compiler/src/BytecodeBuilder.cpp | 8 +- Compiler/src/Compiler.cpp | 62 ++------- Sources.cmake | 5 +- VM/src/lbuiltins.cpp | 2 +- VM/src/lgcdebug.cpp | 4 + VM/src/lmem.cpp | 127 ++++++++++++----- VM/src/lobject.cpp | 4 +- VM/src/lvmexecute.cpp | 3 +- tests/Compiler.test.cpp | 4 - tests/Conformance.test.cpp | 2 - tests/Linter.test.cpp | 36 ++++- tests/Repl.test.cpp | 92 ++++++++++++ tests/TypeInfer.aliases.test.cpp | 61 ++++++++ tests/TypeInfer.builtins.test.cpp | 15 +- tests/TypeInfer.provisional.test.cpp | 74 +++++++++- tests/TypeInfer.refinements.test.cpp | 22 +-- tests/TypeInfer.test.cpp | 147 ++++++++++++++++++++ tests/TypeInfer.tryUnify.test.cpp | 17 +++ tests/conformance/basic.lua | 10 +- tests/conformance/debug.lua | 1 + tests/conformance/errors.lua | 16 ++- tests/conformance/gc.lua | 7 +- tests/conformance/math.lua | 1 + tests/conformance/vararg.lua | 6 + tests/conformance/vector.lua | 9 ++ tools/heapgraph.py | 33 +++-- tools/svg.py | 2 +- 45 files changed, 989 insertions(+), 268 deletions(-) diff --git a/Analysis/include/Luau/Autocomplete.h b/Analysis/include/Luau/Autocomplete.h index 58534293..65b788d3 100644 --- a/Analysis/include/Luau/Autocomplete.h +++ b/Analysis/include/Luau/Autocomplete.h @@ -86,6 +86,8 @@ struct OwningAutocompleteResult }; AutocompleteResult autocomplete(Frontend& frontend, const ModuleName& moduleName, Position position, StringCompletionCallback callback); + +// Deprecated, do not use in new work. OwningAutocompleteResult autocompleteSource(Frontend& frontend, std::string_view source, Position position, StringCompletionCallback callback); } // namespace Luau diff --git a/Analysis/include/Luau/Linter.h b/Analysis/include/Luau/Linter.h index 1f7f7f9d..ec3c124d 100644 --- a/Analysis/include/Luau/Linter.h +++ b/Analysis/include/Luau/Linter.h @@ -49,6 +49,7 @@ struct LintWarning Code_DeprecatedApi = 22, Code_TableOperations = 23, Code_DuplicateCondition = 24, + Code_MisleadingAndOr = 25, Code__Count }; diff --git a/Analysis/include/Luau/Substitution.h b/Analysis/include/Luau/Substitution.h index 4f3307cd..f85b4269 100644 --- a/Analysis/include/Luau/Substitution.h +++ b/Analysis/include/Luau/Substitution.h @@ -93,7 +93,7 @@ struct Tarjan // This should never be null; ensure you initialize it before calling // substitution methods. - const TxnLog* log; + const TxnLog* log = nullptr; std::vector edgesTy; std::vector edgesTp; diff --git a/Analysis/include/Luau/TxnLog.h b/Analysis/include/Luau/TxnLog.h index 02b87374..f238e258 100644 --- a/Analysis/include/Luau/TxnLog.h +++ b/Analysis/include/Luau/TxnLog.h @@ -307,8 +307,8 @@ private: // // We can't use a DenseHashMap here because we need a non-const iterator // over the map when we concatenate. - std::unordered_map> typeVarChanges; - std::unordered_map> typePackChanges; + std::unordered_map, DenseHashPointer> typeVarChanges; + std::unordered_map, DenseHashPointer> typePackChanges; TxnLog* parent = nullptr; diff --git a/Analysis/include/Luau/TypeInfer.h b/Analysis/include/Luau/TypeInfer.h index f61ecbf5..5592fa1f 100644 --- a/Analysis/include/Luau/TypeInfer.h +++ b/Analysis/include/Luau/TypeInfer.h @@ -103,6 +103,11 @@ struct GenericTypeDefinitions std::vector genericPacks; }; +struct HashBoolNamePair +{ + size_t operator()(const std::pair& pair) const; +}; + // All TypeVars are retained via Environment::typeVars. All TypeIds // within a program are borrowed pointers into this set. struct TypeChecker @@ -411,6 +416,12 @@ public: private: int checkRecursionCount = 0; int recursionCount = 0; + + /** + * We use this to avoid doing second-pass analysis of type aliases that are duplicates. We record a pair + * (exported, name) to properly deal with the case where the two duplicates do not have the same export status. + */ + DenseHashSet, HashBoolNamePair> duplicateTypeAliases; }; // Unit test hook diff --git a/Analysis/include/Luau/TypePack.h b/Analysis/include/Luau/TypePack.h index ca588ccb..c74bad11 100644 --- a/Analysis/include/Luau/TypePack.h +++ b/Analysis/include/Luau/TypePack.h @@ -54,9 +54,6 @@ struct TypePackVar bool persistent = false; // Pointer to the type arena that allocated this type. - // Do not depend on the value of this under any circumstances. This is for - // debugging purposes only. This is only set in debug builds; it is nullptr - // in all other environments. TypeArena* owningArena = nullptr; }; diff --git a/Analysis/include/Luau/TypeVar.h b/Analysis/include/Luau/TypeVar.h index 11dc9377..8d1a9fa6 100644 --- a/Analysis/include/Luau/TypeVar.h +++ b/Analysis/include/Luau/TypeVar.h @@ -449,9 +449,6 @@ struct TypeVar final std::optional documentationSymbol; // Pointer to the type arena that allocated this type. - // Do not depend on the value of this under any circumstances. This is for - // debugging purposes only. This is only set in debug builds; it is nullptr - // in all other environments. TypeArena* owningArena = nullptr; bool operator==(const TypeVar& rhs) const; diff --git a/Analysis/src/EmbeddedBuiltinDefinitions.cpp b/Analysis/src/EmbeddedBuiltinDefinitions.cpp index 24982506..f3ef88fc 100644 --- a/Analysis/src/EmbeddedBuiltinDefinitions.cpp +++ b/Analysis/src/EmbeddedBuiltinDefinitions.cpp @@ -1,8 +1,6 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/BuiltinDefinitions.h" -LUAU_FASTFLAGVARIABLE(LuauFixTonumberReturnType, false) - namespace Luau { @@ -115,6 +113,7 @@ declare function gcinfo(): number declare function error(message: T, level: number?) declare function tostring(value: T): string + declare function tonumber(value: T, radix: number?): number? declare function rawequal(a: T1, b: T2): boolean declare function rawget(tab: {[K]: V}, k: K): V @@ -200,14 +199,7 @@ declare function gcinfo(): number std::string getBuiltinDefinitionSource() { - std::string result = kBuiltinDefinitionLuaSrc; - - if (FFlag::LuauFixTonumberReturnType) - result += "declare function tonumber(value: T, radix: number?): number?\n"; - else - result += "declare function tonumber(value: T, radix: number?): number\n"; - - return result; + return kBuiltinDefinitionLuaSrc; } } // namespace Luau diff --git a/Analysis/src/Linter.cpp b/Analysis/src/Linter.cpp index 57a33e93..2ba6a0fc 100644 --- a/Analysis/src/Linter.cpp +++ b/Analysis/src/Linter.cpp @@ -43,6 +43,7 @@ static const char* kWarningNames[] = { "DeprecatedApi", "TableOperations", "DuplicateCondition", + "MisleadingAndOr", }; // clang-format on @@ -2040,18 +2041,28 @@ private: const Property* prop = lookupClassProp(cty, node->index.value); if (prop && prop->deprecated) - { - if (!prop->deprecatedSuggestion.empty()) - emitWarning(*context, LintWarning::Code_DeprecatedApi, node->location, "Member '%s.%s' is deprecated, use '%s' instead", - cty->name.c_str(), node->index.value, prop->deprecatedSuggestion.c_str()); - else - emitWarning(*context, LintWarning::Code_DeprecatedApi, node->location, "Member '%s.%s' is deprecated", cty->name.c_str(), - node->index.value); - } + report(node->location, *prop, cty->name.c_str(), node->index.value); + } + else if (const TableTypeVar* tty = get(follow(*ty))) + { + auto prop = tty->props.find(node->index.value); + + if (prop != tty->props.end() && prop->second.deprecated) + report(node->location, prop->second, tty->name ? tty->name->c_str() : nullptr, node->index.value); } return true; } + + void report(const Location& location, const Property& prop, const char* container, const char* field) + { + std::string suggestion = prop.deprecatedSuggestion.empty() ? "" : format(", use '%s' instead", prop.deprecatedSuggestion.c_str()); + + if (container) + emitWarning(*context, LintWarning::Code_DeprecatedApi, location, "Member '%s.%s' is deprecated%s", container, field, suggestion.c_str()); + else + emitWarning(*context, LintWarning::Code_DeprecatedApi, location, "Member '%s' is deprecated%s", field, suggestion.c_str()); + } }; class LintTableOperations : AstVisitor @@ -2257,6 +2268,39 @@ private: return false; } + bool visit(AstExprIfElse* expr) override + { + if (!expr->falseExpr->is()) + return true; + + // if..elseif chain detected, we need to unroll it + std::vector conditions; + conditions.reserve(2); + + AstExprIfElse* head = expr; + while (head) + { + head->condition->visit(this); + head->trueExpr->visit(this); + + conditions.push_back(head->condition); + + if (head->falseExpr->is()) + { + head = head->falseExpr->as(); + continue; + } + + head->falseExpr->visit(this); + break; + } + + detectDuplicates(conditions); + + // block recursive visits so that we only analyze each chain once + return false; + } + bool visit(AstExprBinary* expr) override { if (expr->op != AstExprBinary::And && expr->op != AstExprBinary::Or) @@ -2418,6 +2462,46 @@ private: } }; +class LintMisleadingAndOr : AstVisitor +{ +public: + LUAU_NOINLINE static void process(LintContext& context) + { + LintMisleadingAndOr pass; + pass.context = &context; + + context.root->visit(&pass); + } + +private: + LintContext* context; + + bool visit(AstExprBinary* node) override + { + if (node->op != AstExprBinary::Or) + return true; + + AstExprBinary* and_ = node->left->as(); + if (!and_ || and_->op != AstExprBinary::And) + return true; + + const char* alt = nullptr; + + if (and_->right->is()) + alt = "nil"; + else if (AstExprConstantBool* c = and_->right->as(); c && c->value == false) + alt = "false"; + + if (alt) + emitWarning(*context, LintWarning::Code_MisleadingAndOr, node->location, + "The and-or expression always evaluates to the second alternative because the first alternative is %s; consider using if-then-else " + "expression instead", + alt); + + return true; + } +}; + static void fillBuiltinGlobals(LintContext& context, const AstNameTable& names, const ScopePtr& env) { ScopePtr current = env; @@ -2522,6 +2606,9 @@ std::vector lint(AstStat* root, const AstNameTable& names, const Sc if (context.warningEnabled(LintWarning::Code_DuplicateLocal)) LintDuplicateLocal::process(context); + if (context.warningEnabled(LintWarning::Code_MisleadingAndOr)) + LintMisleadingAndOr::process(context); + std::sort(context.result.begin(), context.result.end(), WarningComparator()); return context.result; diff --git a/Analysis/src/Module.cpp b/Analysis/src/Module.cpp index 4fdff8f7..817a33e9 100644 --- a/Analysis/src/Module.cpp +++ b/Analysis/src/Module.cpp @@ -12,10 +12,10 @@ #include LUAU_FASTFLAGVARIABLE(DebugLuauFreezeArena, false) -LUAU_FASTFLAGVARIABLE(DebugLuauTrackOwningArena, false) +LUAU_FASTFLAGVARIABLE(DebugLuauTrackOwningArena, false) // Remove with FFlagLuauImmutableTypes LUAU_FASTINTVARIABLE(LuauTypeCloneRecursionLimit, 300) LUAU_FASTFLAG(LuauTypeAliasDefaults) - +LUAU_FASTFLAG(LuauImmutableTypes) LUAU_FASTFLAGVARIABLE(LuauPrepopulateUnionOptionsBeforeAllocation, false) namespace Luau @@ -66,7 +66,7 @@ TypeId TypeArena::addTV(TypeVar&& tv) { TypeId allocated = typeVars.allocate(std::move(tv)); - if (FFlag::DebugLuauTrackOwningArena) + if (FFlag::DebugLuauTrackOwningArena || FFlag::LuauImmutableTypes) asMutable(allocated)->owningArena = this; return allocated; @@ -76,7 +76,7 @@ TypeId TypeArena::freshType(TypeLevel level) { TypeId allocated = typeVars.allocate(FreeTypeVar{level}); - if (FFlag::DebugLuauTrackOwningArena) + if (FFlag::DebugLuauTrackOwningArena || FFlag::LuauImmutableTypes) asMutable(allocated)->owningArena = this; return allocated; @@ -86,7 +86,7 @@ TypePackId TypeArena::addTypePack(std::initializer_list types) { TypePackId allocated = typePacks.allocate(TypePack{std::move(types)}); - if (FFlag::DebugLuauTrackOwningArena) + if (FFlag::DebugLuauTrackOwningArena || FFlag::LuauImmutableTypes) asMutable(allocated)->owningArena = this; return allocated; @@ -96,7 +96,7 @@ TypePackId TypeArena::addTypePack(std::vector types) { TypePackId allocated = typePacks.allocate(TypePack{std::move(types)}); - if (FFlag::DebugLuauTrackOwningArena) + if (FFlag::DebugLuauTrackOwningArena || FFlag::LuauImmutableTypes) asMutable(allocated)->owningArena = this; return allocated; @@ -106,7 +106,7 @@ TypePackId TypeArena::addTypePack(TypePack tp) { TypePackId allocated = typePacks.allocate(std::move(tp)); - if (FFlag::DebugLuauTrackOwningArena) + if (FFlag::DebugLuauTrackOwningArena || FFlag::LuauImmutableTypes) asMutable(allocated)->owningArena = this; return allocated; @@ -116,7 +116,7 @@ TypePackId TypeArena::addTypePack(TypePackVar tp) { TypePackId allocated = typePacks.allocate(std::move(tp)); - if (FFlag::DebugLuauTrackOwningArena) + if (FFlag::DebugLuauTrackOwningArena || FFlag::LuauImmutableTypes) asMutable(allocated)->owningArena = this; return allocated; @@ -454,8 +454,16 @@ TypeId clone(TypeId typeId, TypeArena& dest, SeenTypes& seenTypes, SeenTypePacks TypeCloner cloner{dest, typeId, seenTypes, seenTypePacks, cloneState}; Luau::visit(cloner, typeId->ty); // Mutates the storage that 'res' points into. - // TODO: Make this work when the arena of 'res' might be frozen - asMutable(res)->documentationSymbol = typeId->documentationSymbol; + if (FFlag::LuauImmutableTypes) + { + // Persistent types are not being cloned and we get the original type back which might be read-only + if (!res->persistent) + asMutable(res)->documentationSymbol = typeId->documentationSymbol; + } + else + { + asMutable(res)->documentationSymbol = typeId->documentationSymbol; + } } return res; diff --git a/Analysis/src/Scope.cpp b/Analysis/src/Scope.cpp index c30db9c2..0a362a5e 100644 --- a/Analysis/src/Scope.cpp +++ b/Analysis/src/Scope.cpp @@ -2,6 +2,8 @@ #include "Luau/Scope.h" +LUAU_FASTFLAG(LuauTwoPassAliasDefinitionFix); + namespace Luau { @@ -17,6 +19,8 @@ Scope::Scope(const ScopePtr& parent, int subLevel) , returnType(parent->returnType) , level(parent->level.incr()) { + if (FFlag::LuauTwoPassAliasDefinitionFix) + level = level.incr(); level.subLevel = subLevel; } diff --git a/Analysis/src/TxnLog.cpp b/Analysis/src/TxnLog.cpp index 0968a4c1..00067bdd 100644 --- a/Analysis/src/TxnLog.cpp +++ b/Analysis/src/TxnLog.cpp @@ -250,6 +250,10 @@ PendingTypePack* TxnLog::queue(TypePackId tp) PendingType* TxnLog::pending(TypeId ty) const { + // This function will technically work if `this` is nullptr, but this + // indicates a bug, so we explicitly assert. + LUAU_ASSERT(static_cast(this) != nullptr); + for (const TxnLog* current = this; current; current = current->parent) { if (auto it = current->typeVarChanges.find(ty); it != current->typeVarChanges.end()) @@ -261,6 +265,10 @@ PendingType* TxnLog::pending(TypeId ty) const PendingTypePack* TxnLog::pending(TypePackId tp) const { + // This function will technically work if `this` is nullptr, but this + // indicates a bug, so we explicitly assert. + LUAU_ASSERT(static_cast(this) != nullptr); + for (const TxnLog* current = this; current; current = current->parent) { if (auto it = current->typePackChanges.find(tp); it != current->typePackChanges.end()) diff --git a/Analysis/src/TypeInfer.cpp b/Analysis/src/TypeInfer.cpp index 99398f74..f1c314cd 100644 --- a/Analysis/src/TypeInfer.cpp +++ b/Analysis/src/TypeInfer.cpp @@ -32,12 +32,13 @@ LUAU_FASTFLAGVARIABLE(LuauRecursiveTypeParameterRestriction, false) LUAU_FASTFLAGVARIABLE(LuauGenericFunctionsDontCacheTypeParams, false) LUAU_FASTFLAGVARIABLE(LuauIfElseBranchTypeUnion, false) LUAU_FASTFLAGVARIABLE(LuauIfElseExpectedType2, false) +LUAU_FASTFLAGVARIABLE(LuauImmutableTypes, false) LUAU_FASTFLAGVARIABLE(LuauLengthOnCompositeType, false) LUAU_FASTFLAGVARIABLE(LuauNoSealedTypeMod, false) LUAU_FASTFLAGVARIABLE(LuauQuantifyInPlace2, false) LUAU_FASTFLAGVARIABLE(LuauSealExports, false) LUAU_FASTFLAGVARIABLE(LuauSingletonTypes, false) -LUAU_FASTFLAGVARIABLE(LuauDiscriminableUnions, false) +LUAU_FASTFLAGVARIABLE(LuauDiscriminableUnions2, false) LUAU_FASTFLAGVARIABLE(LuauTypeAliasDefaults, false) LUAU_FASTFLAGVARIABLE(LuauExpectedTypesOfProperties, false) LUAU_FASTFLAGVARIABLE(LuauErrorRecoveryType, false) @@ -47,7 +48,10 @@ LUAU_FASTFLAGVARIABLE(LuauProperTypeLevels, false) LUAU_FASTFLAGVARIABLE(LuauAscribeCorrectLevelToInferredProperitesOfFreeTables, false) LUAU_FASTFLAG(LuauUnionTagMatchFix) LUAU_FASTFLAGVARIABLE(LuauUnsealedTableLiteral, false) +LUAU_FASTFLAGVARIABLE(LuauTwoPassAliasDefinitionFix, false) LUAU_FASTFLAGVARIABLE(LuauAssertStripsFalsyTypes, false) +LUAU_FASTFLAGVARIABLE(LuauReturnAnyInsteadOfICE, false) // Eventually removed as false. +LUAU_FASTFLAGVARIABLE(LuauAnotherTypeLevelFix, false) namespace Luau { @@ -213,6 +217,11 @@ static bool isMetamethod(const Name& name) name == "__metatable" || name == "__eq" || name == "__lt" || name == "__le" || name == "__mode"; } +size_t HashBoolNamePair::operator()(const std::pair& pair) const +{ + return std::hash()(pair.first) ^ std::hash()(pair.second); +} + TypeChecker::TypeChecker(ModuleResolver* resolver, InternalErrorReporter* iceHandler) : resolver(resolver) , iceHandler(iceHandler) @@ -225,6 +234,7 @@ TypeChecker::TypeChecker(ModuleResolver* resolver, InternalErrorReporter* iceHan , anyType(getSingletonTypes().anyType) , optionalNumberType(getSingletonTypes().optionalNumberType) , anyTypePack(getSingletonTypes().anyTypePack) + , duplicateTypeAliases{{false, {}}} { globalScope = std::make_shared(globalTypes.addTypePack(TypePackVar{FreeTypePack{TypeLevel{}}})); @@ -291,6 +301,9 @@ ModulePtr TypeChecker::check(const SourceModule& module, Mode mode, std::optiona unifierState.skipCacheForType.clear(); } + if (FFlag::LuauTwoPassAliasDefinitionFix) + duplicateTypeAliases.clear(); + return std::move(currentModule); } @@ -496,6 +509,9 @@ LUAU_NOINLINE void TypeChecker::checkBlockTypeAliases(const ScopePtr& scope, std { if (const auto& typealias = stat->as()) { + if (FFlag::LuauTwoPassAliasDefinitionFix && typealias->name == Parser::errorName) + continue; + auto& bindings = typealias->exported ? scope->exportedTypeBindings : scope->privateTypeBindings; Name name = typealias->name.value; @@ -1176,6 +1192,10 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias // Once with forwardDeclare, and once without. Name name = typealias.name.value; + // If the alias is missing a name, we can't do anything with it. Ignore it. + if (FFlag::LuauTwoPassAliasDefinitionFix && name == Parser::errorName) + return; + std::optional binding; if (auto it = scope->exportedTypeBindings.find(name); it != scope->exportedTypeBindings.end()) binding = it->second; @@ -1192,6 +1212,8 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias reportError(TypeError{typealias.location, DuplicateTypeDefinition{name, location}}); bindingsMap[name] = TypeFun{binding->typeParams, binding->typePackParams, errorRecoveryType(anyType)}; + if (FFlag::LuauTwoPassAliasDefinitionFix) + duplicateTypeAliases.insert({typealias.exported, name}); } else { @@ -1211,6 +1233,11 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias } else { + // If the first pass failed (this should mean a duplicate definition), the second pass isn't going to be + // interesting. + if (FFlag::LuauTwoPassAliasDefinitionFix && duplicateTypeAliases.find({typealias.exported, name})) + return; + if (!binding) ice("Not predeclared"); @@ -1235,7 +1262,8 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias if (auto ttv = getMutable(follow(ty))) { // If the table is already named and we want to rename the type function, we have to bind new alias to a copy - if (ttv->name) + // Additionally, we can't modify types that come from other modules + if (ttv->name || (FFlag::LuauImmutableTypes && follow(ty)->owningArena != ¤tModule->internalTypes)) { bool sameTys = std::equal(ttv->instantiatedTypeParams.begin(), ttv->instantiatedTypeParams.end(), binding->typeParams.begin(), binding->typeParams.end(), [](auto&& itp, auto&& tp) { @@ -1247,7 +1275,7 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias }); // Copy can be skipped if this is an identical alias - if (ttv->name != name || !sameTys || !sameTps) + if ((FFlag::LuauImmutableTypes && !ttv->name) || ttv->name != name || !sameTys || !sameTps) { // This is a shallow clone, original recursive links to self are not updated TableTypeVar clone = TableTypeVar{ttv->props, ttv->indexer, ttv->level, ttv->state}; @@ -1279,9 +1307,17 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias } } else if (auto mtv = getMutable(follow(ty))) - mtv->syntheticName = name; + { + // We can't modify types that come from other modules + if (!FFlag::LuauImmutableTypes || follow(ty)->owningArena == ¤tModule->internalTypes) + mtv->syntheticName = name; + } - unify(ty, bindingsMap[name].type, typealias.location); + TypeId& bindingType = bindingsMap[name].type; + bool ok = unify(ty, bindingType, typealias.location); + + if (FFlag::LuauTwoPassAliasDefinitionFix && ok) + bindingType = ty; } } @@ -1564,7 +1600,12 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprCa else if (auto vtp = get(retPack)) return {vtp->ty, std::move(result.predicates)}; else if (get(retPack)) - ice("Unexpected abstract type pack!", expr.location); + { + if (FFlag::LuauReturnAnyInsteadOfICE) + return {anyType, std::move(result.predicates)}; + else + ice("Unexpected abstract type pack!", expr.location); + } else ice("Unknown TypePack type!", expr.location); } @@ -1614,11 +1655,23 @@ std::optional TypeChecker::getIndexTypeFromType( tablify(type); - const PrimitiveTypeVar* primitiveType = get(type); - if (primitiveType && primitiveType->type == PrimitiveTypeVar::String) + if (FFlag::LuauDiscriminableUnions2) { - if (std::optional mtIndex = findMetatableEntry(type, "__index", location)) + if (isString(type)) + { + std::optional mtIndex = findMetatableEntry(stringType, "__index", location); + LUAU_ASSERT(mtIndex); type = *mtIndex; + } + } + else + { + const PrimitiveTypeVar* primitiveType = get(type); + if (primitiveType && primitiveType->type == PrimitiveTypeVar::String) + { + if (std::optional mtIndex = findMetatableEntry(type, "__index", location)) + type = *mtIndex; + } } if (TableTypeVar* tableType = getMutableTableType(type)) @@ -2476,7 +2529,7 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprBi auto [rhsTy, rhsPredicates] = checkExpr(innerScope, *expr.right); - return {checkBinaryOperation(FFlag::LuauDiscriminableUnions ? scope : innerScope, expr, lhsTy, rhsTy), + return {checkBinaryOperation(FFlag::LuauDiscriminableUnions2 ? scope : innerScope, expr, lhsTy, rhsTy), {AndPredicate{std::move(lhsPredicates), std::move(rhsPredicates)}}}; } else if (expr.op == AstExprBinary::Or) @@ -2489,7 +2542,7 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprBi auto [rhsTy, rhsPredicates] = checkExpr(innerScope, *expr.right); // Because of C++, I'm not sure if lhsPredicates was not moved out by the time we call checkBinaryOperation. - TypeId result = checkBinaryOperation(FFlag::LuauDiscriminableUnions ? scope : innerScope, expr, lhsTy, rhsTy, lhsPredicates); + TypeId result = checkBinaryOperation(FFlag::LuauDiscriminableUnions2 ? scope : innerScope, expr, lhsTy, rhsTy, lhsPredicates); return {result, {OrPredicate{std::move(lhsPredicates), std::move(rhsPredicates)}}}; } else if (expr.op == AstExprBinary::CompareEq || expr.op == AstExprBinary::CompareNe) @@ -2497,8 +2550,8 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprBi if (auto predicate = tryGetTypeGuardPredicate(expr)) return {booleanType, {std::move(*predicate)}}; - ExprResult lhs = checkExpr(scope, *expr.left, std::nullopt, /*forceSingleton=*/FFlag::LuauDiscriminableUnions); - ExprResult rhs = checkExpr(scope, *expr.right, std::nullopt, /*forceSingleton=*/FFlag::LuauDiscriminableUnions); + ExprResult lhs = checkExpr(scope, *expr.left, std::nullopt, /*forceSingleton=*/FFlag::LuauDiscriminableUnions2); + ExprResult rhs = checkExpr(scope, *expr.right, std::nullopt, /*forceSingleton=*/FFlag::LuauDiscriminableUnions2); PredicateVec predicates; @@ -2785,12 +2838,16 @@ TypeId TypeChecker::checkLValueBinding(const ScopePtr& scope, const AstExprIndex } else if (exprTable->state == TableState::Unsealed || exprTable->state == TableState::Free) { - TypeId resultType = freshType(scope); + TypeId resultType = freshType(FFlag::LuauAnotherTypeLevelFix ? exprTable->level : scope->level); exprTable->indexer = TableIndexer{anyIfNonstrict(indexType), anyIfNonstrict(resultType)}; return resultType; } else { + /* + * If we use [] indexing to fetch a property from a sealed table that has no indexer, we have no idea if it will + * work, so we just mint a fresh type, return that, and hope for the best. + */ TypeId resultType = freshType(scope); return resultType; } @@ -4195,6 +4252,9 @@ TypeId TypeChecker::checkRequire(const ScopePtr& scope, const ModuleInfo& module return errorRecoveryType(scope); } + if (FFlag::LuauImmutableTypes) + return *moduleType; + SeenTypes seenTypes; SeenTypePacks seenTypePacks; CloneState cloneState; @@ -5446,7 +5506,7 @@ GenericTypeDefinitions TypeChecker::createGenericTypes(const ScopePtr& scope, st void TypeChecker::refineLValue(const LValue& lvalue, RefinementMap& refis, const ScopePtr& scope, TypeIdPredicate predicate) { - LUAU_ASSERT(FFlag::LuauDiscriminableUnions); + LUAU_ASSERT(FFlag::LuauDiscriminableUnions2); const LValue* target = &lvalue; std::optional key; // If set, we know we took the base of the lvalue path and should be walking down each option of the base's type. @@ -5659,7 +5719,7 @@ void TypeChecker::resolve(const TruthyPredicate& truthyP, ErrorVec& errVec, Refi return std::nullopt; }; - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) { std::optional ty = resolveLValue(refis, scope, truthyP.lvalue); if (ty && fromOr) @@ -5772,7 +5832,7 @@ void TypeChecker::resolve(const IsAPredicate& isaP, ErrorVec& errVec, Refinement return res; }; - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) { refineLValue(isaP.lvalue, refis, scope, predicate); } @@ -5847,7 +5907,7 @@ void TypeChecker::resolve(const TypeGuardPredicate& typeguardP, ErrorVec& errVec if (auto it = primitives.find(typeguardP.kind); it != primitives.end()) { - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) { refineLValue(typeguardP.lvalue, refis, scope, it->second(sense)); return; @@ -5869,7 +5929,7 @@ void TypeChecker::resolve(const TypeGuardPredicate& typeguardP, ErrorVec& errVec } auto fail = [&](const TypeErrorData& err) { - if (!FFlag::LuauDiscriminableUnions) + if (!FFlag::LuauDiscriminableUnions2) errVec.push_back(TypeError{typeguardP.location, err}); addRefinement(refis, typeguardP.lvalue, errorRecoveryType(scope)); }; @@ -5901,7 +5961,7 @@ void TypeChecker::resolve(const EqPredicate& eqP, ErrorVec& errVec, RefinementMa return {ty}; }; - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) { std::vector rhs = options(eqP.type); diff --git a/Analysis/src/TypeVar.cpp b/Analysis/src/TypeVar.cpp index 2321eafd..7e438e31 100644 --- a/Analysis/src/TypeVar.cpp +++ b/Analysis/src/TypeVar.cpp @@ -28,6 +28,7 @@ LUAU_FASTFLAGVARIABLE(LuauMetatableAreEqualRecursion, false) LUAU_FASTFLAGVARIABLE(LuauRefactorTypeVarQuestions, false) LUAU_FASTFLAG(LuauErrorRecoveryType) LUAU_FASTFLAG(LuauUnionTagMatchFix) +LUAU_FASTFLAG(LuauDiscriminableUnions2) namespace Luau { @@ -393,7 +394,8 @@ bool hasLength(TypeId ty, DenseHashSet& seen, int* recursionCount) if (seen.contains(ty)) return true; - if (isPrim(ty, PrimitiveTypeVar::String) || get(ty) || get(ty) || get(ty)) + bool isStr = FFlag::LuauDiscriminableUnions2 ? isString(ty) : isPrim(ty, PrimitiveTypeVar::String); + if (isStr || get(ty) || get(ty) || get(ty)) return true; if (auto uty = get(ty)) diff --git a/Analysis/src/Unifier.cpp b/Analysis/src/Unifier.cpp index 89e4ae23..a8ad5159 100644 --- a/Analysis/src/Unifier.cpp +++ b/Analysis/src/Unifier.cpp @@ -15,6 +15,7 @@ LUAU_FASTINT(LuauTypeInferRecursionLimit); LUAU_FASTINT(LuauTypeInferTypePackLoopLimit); LUAU_FASTFLAGVARIABLE(LuauCommittingTxnLogFreeTpPromote, false) +LUAU_FASTFLAG(LuauImmutableTypes) LUAU_FASTFLAG(LuauUseCommittingTxnLog) LUAU_FASTINTVARIABLE(LuauTypeInferIterationLimit, 2000); LUAU_FASTFLAGVARIABLE(LuauTableSubtypingVariance2, false); @@ -24,6 +25,7 @@ LUAU_FASTFLAG(LuauErrorRecoveryType); LUAU_FASTFLAG(LuauProperTypeLevels); LUAU_FASTFLAGVARIABLE(LuauUnifyPackTails, false) LUAU_FASTFLAGVARIABLE(LuauUnionTagMatchFix, false) +LUAU_FASTFLAGVARIABLE(LuauFollowWithCommittingTxnLogInAnyUnification, false) namespace Luau { @@ -32,11 +34,13 @@ struct PromoteTypeLevels { DEPRECATED_TxnLog& DEPRECATED_log; TxnLog& log; + const TypeArena* typeArena = nullptr; TypeLevel minLevel; - explicit PromoteTypeLevels(DEPRECATED_TxnLog& DEPRECATED_log, TxnLog& log, TypeLevel minLevel) + explicit PromoteTypeLevels(DEPRECATED_TxnLog& DEPRECATED_log, TxnLog& log, const TypeArena* typeArena, TypeLevel minLevel) : DEPRECATED_log(DEPRECATED_log) , log(log) + , typeArena(typeArena) , minLevel(minLevel) { } @@ -65,8 +69,12 @@ struct PromoteTypeLevels } template - bool operator()(TID, const T&) + bool operator()(TID ty, const T&) { + // Type levels of types from other modules are already global, so we don't need to promote anything inside + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + return false; + return true; } @@ -83,12 +91,20 @@ struct PromoteTypeLevels bool operator()(TypeId ty, const FunctionTypeVar&) { + // Type levels of types from other modules are already global, so we don't need to promote anything inside + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + return false; + promote(ty, FFlag::LuauUseCommittingTxnLog ? log.getMutable(ty) : getMutable(ty)); return true; } bool operator()(TypeId ty, const TableTypeVar& ttv) { + // Type levels of types from other modules are already global, so we don't need to promote anything inside + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + return false; + if (ttv.state != TableState::Free && ttv.state != TableState::Generic) return true; @@ -108,24 +124,33 @@ struct PromoteTypeLevels } }; -void promoteTypeLevels(DEPRECATED_TxnLog& DEPRECATED_log, TxnLog& log, TypeLevel minLevel, TypeId ty) +void promoteTypeLevels(DEPRECATED_TxnLog& DEPRECATED_log, TxnLog& log, const TypeArena* typeArena, TypeLevel minLevel, TypeId ty) { - PromoteTypeLevels ptl{DEPRECATED_log, log, minLevel}; + // Type levels of types from other modules are already global, so we don't need to promote anything inside + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + return; + + PromoteTypeLevels ptl{DEPRECATED_log, log, typeArena, minLevel}; DenseHashSet seen{nullptr}; visitTypeVarOnce(ty, ptl, seen); } -void promoteTypeLevels(DEPRECATED_TxnLog& DEPRECATED_log, TxnLog& log, TypeLevel minLevel, TypePackId tp) +void promoteTypeLevels(DEPRECATED_TxnLog& DEPRECATED_log, TxnLog& log, const TypeArena* typeArena, TypeLevel minLevel, TypePackId tp) { - PromoteTypeLevels ptl{DEPRECATED_log, log, minLevel}; + // Type levels of types from other modules are already global, so we don't need to promote anything inside + if (FFlag::LuauImmutableTypes && tp->owningArena != typeArena) + return; + + PromoteTypeLevels ptl{DEPRECATED_log, log, typeArena, minLevel}; DenseHashSet seen{nullptr}; visitTypeVarOnce(tp, ptl, seen); } struct SkipCacheForType { - SkipCacheForType(const DenseHashMap& skipCacheForType) + SkipCacheForType(const DenseHashMap& skipCacheForType, const TypeArena* typeArena) : skipCacheForType(skipCacheForType) + , typeArena(typeArena) { } @@ -152,6 +177,10 @@ struct SkipCacheForType bool operator()(TypeId ty, const TableTypeVar&) { + // Types from other modules don't contain mutable elements and are ok to cache + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + return false; + TableTypeVar& ttv = *getMutable(ty); if (ttv.boundTo) @@ -172,6 +201,10 @@ struct SkipCacheForType template bool operator()(TypeId ty, const T& t) { + // Types from other modules don't contain mutable elements and are ok to cache + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + return false; + const bool* prev = skipCacheForType.find(ty); if (prev && *prev) @@ -184,8 +217,12 @@ struct SkipCacheForType } template - bool operator()(TypePackId, const T&) + bool operator()(TypePackId tp, const T&) { + // Types from other modules don't contain mutable elements and are ok to cache + if (FFlag::LuauImmutableTypes && tp->owningArena != typeArena) + return false; + return true; } @@ -208,6 +245,7 @@ struct SkipCacheForType } const DenseHashMap& skipCacheForType; + const TypeArena* typeArena = nullptr; bool result = false; }; @@ -422,13 +460,13 @@ void Unifier::tryUnify_(TypeId subTy, TypeId superTy, bool isFunctionCall, bool { if (FFlag::LuauUseCommittingTxnLog) { - promoteTypeLevels(DEPRECATED_log, log, superLevel, subTy); + promoteTypeLevels(DEPRECATED_log, log, types, superLevel, subTy); log.replace(superTy, BoundTypeVar(subTy)); } else { if (FFlag::LuauProperTypeLevels) - promoteTypeLevels(DEPRECATED_log, log, superLevel, subTy); + promoteTypeLevels(DEPRECATED_log, log, types, superLevel, subTy); else if (auto subLevel = getMutableLevel(subTy)) { if (!subLevel->subsumes(superFree->level)) @@ -466,13 +504,13 @@ void Unifier::tryUnify_(TypeId subTy, TypeId superTy, bool isFunctionCall, bool { if (FFlag::LuauUseCommittingTxnLog) { - promoteTypeLevels(DEPRECATED_log, log, subLevel, superTy); + promoteTypeLevels(DEPRECATED_log, log, types, subLevel, superTy); log.replace(subTy, BoundTypeVar(superTy)); } else { if (FFlag::LuauProperTypeLevels) - promoteTypeLevels(DEPRECATED_log, log, subLevel, superTy); + promoteTypeLevels(DEPRECATED_log, log, types, subLevel, superTy); else if (auto superLevel = getMutableLevel(superTy)) { if (!superLevel->subsumes(subFree->level)) @@ -849,7 +887,7 @@ void Unifier::cacheResult(TypeId subTy, TypeId superTy) return; auto skipCacheFor = [this](TypeId ty) { - SkipCacheForType visitor{sharedState.skipCacheForType}; + SkipCacheForType visitor{sharedState.skipCacheForType, types}; visitTypeVarOnce(ty, visitor, sharedState.seenAny); sharedState.skipCacheForType[ty] = visitor.result; @@ -1637,32 +1675,35 @@ void Unifier::tryUnifyFunctions(TypeId subTy, TypeId superTy, bool isFunctionCal tryUnify_(subFunction->retType, superFunction->retType); } - if (FFlag::LuauUseCommittingTxnLog) + if (!FFlag::LuauImmutableTypes) { - if (superFunction->definition && !subFunction->definition && !subTy->persistent) + if (FFlag::LuauUseCommittingTxnLog) { - PendingType* newSubTy = log.queue(subTy); - FunctionTypeVar* newSubFtv = getMutable(newSubTy); - LUAU_ASSERT(newSubFtv); - newSubFtv->definition = superFunction->definition; + if (superFunction->definition && !subFunction->definition && !subTy->persistent) + { + PendingType* newSubTy = log.queue(subTy); + FunctionTypeVar* newSubFtv = getMutable(newSubTy); + LUAU_ASSERT(newSubFtv); + newSubFtv->definition = superFunction->definition; + } + else if (!superFunction->definition && subFunction->definition && !superTy->persistent) + { + PendingType* newSuperTy = log.queue(superTy); + FunctionTypeVar* newSuperFtv = getMutable(newSuperTy); + LUAU_ASSERT(newSuperFtv); + newSuperFtv->definition = subFunction->definition; + } } - else if (!superFunction->definition && subFunction->definition && !superTy->persistent) + else { - PendingType* newSuperTy = log.queue(superTy); - FunctionTypeVar* newSuperFtv = getMutable(newSuperTy); - LUAU_ASSERT(newSuperFtv); - newSuperFtv->definition = subFunction->definition; - } - } - else - { - if (superFunction->definition && !subFunction->definition && !subTy->persistent) - { - subFunction->definition = superFunction->definition; - } - else if (!superFunction->definition && subFunction->definition && !superTy->persistent) - { - superFunction->definition = subFunction->definition; + if (superFunction->definition && !subFunction->definition && !subTy->persistent) + { + subFunction->definition = superFunction->definition; + } + else if (!superFunction->definition && subFunction->definition && !superTy->persistent) + { + superFunction->definition = subFunction->definition; + } } } @@ -2631,7 +2672,7 @@ static void queueTypePack(std::vector& queue, DenseHashSet& { while (true) { - a = follow(a); + a = FFlag::LuauFollowWithCommittingTxnLogInAnyUnification ? state.log.follow(a) : follow(a); if (seenTypePacks.find(a)) break; @@ -2738,7 +2779,7 @@ void Unifier::tryUnifyVariadics(TypePackId subTp, TypePackId superTp, bool rever } static void tryUnifyWithAny(std::vector& queue, Unifier& state, DenseHashSet& seen, DenseHashSet& seenTypePacks, - TypeId anyType, TypePackId anyTypePack) + const TypeArena* typeArena, TypeId anyType, TypePackId anyTypePack) { while (!queue.empty()) { @@ -2746,8 +2787,14 @@ static void tryUnifyWithAny(std::vector& queue, Unifier& state, DenseHas { TypeId ty = state.log.follow(queue.back()); queue.pop_back(); + + // Types from other modules don't have free types + if (FFlag::LuauImmutableTypes && ty->owningArena != typeArena) + continue; + if (seen.find(ty)) continue; + seen.insert(ty); if (state.log.getMutable(ty)) @@ -2853,7 +2900,7 @@ void Unifier::tryUnifyWithAny(TypeId subTy, TypeId anyTy) sharedState.tempSeenTy.clear(); sharedState.tempSeenTp.clear(); - Luau::tryUnifyWithAny(queue, *this, sharedState.tempSeenTy, sharedState.tempSeenTp, getSingletonTypes().anyType, anyTP); + Luau::tryUnifyWithAny(queue, *this, sharedState.tempSeenTy, sharedState.tempSeenTp, types, getSingletonTypes().anyType, anyTP); } void Unifier::tryUnifyWithAny(TypePackId subTy, TypePackId anyTp) @@ -2869,7 +2916,7 @@ void Unifier::tryUnifyWithAny(TypePackId subTy, TypePackId anyTp) queueTypePack(queue, sharedState.tempSeenTp, *this, subTy, anyTp); - Luau::tryUnifyWithAny(queue, *this, sharedState.tempSeenTy, sharedState.tempSeenTp, anyTy, anyTp); + Luau::tryUnifyWithAny(queue, *this, sharedState.tempSeenTy, sharedState.tempSeenTp, types, anyTy, anyTp); } std::optional Unifier::findTablePropertyRespectingMeta(TypeId lhsType, Name name) diff --git a/Ast/src/Parser.cpp b/Ast/src/Parser.cpp index f559e2e0..30b32f91 100644 --- a/Ast/src/Parser.cpp +++ b/Ast/src/Parser.cpp @@ -1133,7 +1133,7 @@ AstTypePack* Parser::parseTypeList(TempVector& result, TempVector(ic_completion_arg(cenv)); std::string_view lookup = editBuffer; char lastSep = 0; @@ -276,7 +278,7 @@ static void completeIndexer(ic_completion_env_t* cenv, const char* editBuffer) // Add an opening paren for function calls by default. completion += "("; } - ic_add_completion_ex(cenv, completion.data(), key.data(), nullptr); + addCompletionCallback(completion, std::string(key)); } } lua_pop(L, 1); @@ -295,10 +297,11 @@ static void completeIndexer(ic_completion_env_t* cenv, const char* editBuffer) { // Replace the string object with the string class to perform further lookups of string functions // Note: We retrieve the string class from _G to prevent issues if the user assigns to `string`. + lua_pop(L, 1); // Pop the string instance lua_getglobal(L, "_G"); lua_pushlstring(L, "string", 6); lua_rawget(L, -2); - lua_remove(L, -2); + lua_remove(L, -2); // Remove the global table LUAU_ASSERT(lua_istable(L, -1)); } else if (!lua_istable(L, -1)) @@ -312,6 +315,26 @@ static void completeIndexer(ic_completion_env_t* cenv, const char* editBuffer) lua_pop(L, 1); } +void getCompletions(lua_State* L, const std::string& editBuffer, const AddCompletionCallback& addCompletionCallback) +{ + // look the value up in current global table first + lua_pushvalue(L, LUA_GLOBALSINDEX); + completeIndexer(L, editBuffer, addCompletionCallback); + + // and in actual global table after that + lua_getglobal(L, "_G"); + completeIndexer(L, editBuffer, addCompletionCallback); +} + +static void icGetCompletions(ic_completion_env_t* cenv, const char* editBuffer) +{ + auto* L = reinterpret_cast(ic_completion_arg(cenv)); + + getCompletions(L, std::string(editBuffer), [cenv](const std::string& completion, const std::string& display) { + ic_add_completion_ex(cenv, completion.data(), display.data(), nullptr); + }); +} + static bool isMethodOrFunctionChar(const char* s, long len) { char c = *s; @@ -320,15 +343,7 @@ static bool isMethodOrFunctionChar(const char* s, long len) static void completeRepl(ic_completion_env_t* cenv, const char* editBuffer) { - auto* L = reinterpret_cast(ic_completion_arg(cenv)); - - // look the value up in current global table first - lua_pushvalue(L, LUA_GLOBALSINDEX); - ic_complete_word(cenv, editBuffer, completeIndexer, isMethodOrFunctionChar); - - // and in actual global table after that - lua_getglobal(L, "_G"); - ic_complete_word(cenv, editBuffer, completeIndexer, isMethodOrFunctionChar); + ic_complete_word(cenv, editBuffer, icGetCompletions, isMethodOrFunctionChar); } struct LinenoiseScopedHistory @@ -372,19 +387,20 @@ static void runReplImpl(lua_State* L) for (;;) { - const char* line = ic_readline(buffer.empty() ? "" : ">"); + const char* prompt = buffer.empty() ? "" : ">"; + std::unique_ptr line(ic_readline(prompt), free); if (!line) break; - if (buffer.empty() && runCode(L, std::string("return ") + line) == std::string()) + if (buffer.empty() && runCode(L, std::string("return ") + line.get()) == std::string()) { - ic_history_add(line); + ic_history_add(line.get()); continue; } if (!buffer.empty()) buffer += "\n"; - buffer += line; + buffer += line.get(); std::string error = runCode(L, buffer); @@ -400,7 +416,6 @@ static void runReplImpl(lua_State* L) ic_history_add(buffer.c_str()); buffer.clear(); - free((void*)line); } } @@ -504,7 +519,7 @@ static bool compileFile(const char* name, CompileFormat format) if (format == CompileFormat::Text) { - bcb.setDumpFlags(Luau::BytecodeBuilder::Dump_Code | Luau::BytecodeBuilder::Dump_Source); + bcb.setDumpFlags(Luau::BytecodeBuilder::Dump_Code | Luau::BytecodeBuilder::Dump_Source | Luau::BytecodeBuilder::Dump_Locals); bcb.setDumpSource(*source); } @@ -549,7 +564,8 @@ static void displayHelp(const char* argv0) printf(" --coverage: collect code coverage while running the code and output results to coverage.out\n"); printf(" -h, --help: Display this usage message.\n"); printf(" -i, --interactive: Run an interactive REPL after executing the last script specified.\n"); - printf(" -O: use compiler optimization level (n=0-2).\n"); + printf(" -O: compile with optimization level n (default 1, n should be between 0 and 2).\n"); + printf(" -g: compile with debug level n (default 1, n should be between 0 and 2).\n"); printf(" --profile[=N]: profile the code using N Hz sampling (default 10000) and output results to profile.out\n"); printf(" --timetrace: record compiler time tracing information into trace.json\n"); } @@ -620,6 +636,16 @@ int replMain(int argc, char** argv) } globalOptions.optimizationLevel = level; } + else if (strncmp(argv[i], "-g", 2) == 0) + { + int level = atoi(argv[i] + 2); + if (level < 0 || level > 2) + { + fprintf(stderr, "Error: Debug level must be between 0 and 2 inclusive.\n"); + return 1; + } + globalOptions.debugLevel = level; + } else if (strcmp(argv[i], "--profile") == 0) { profile = 10000; // default to 10 KHz diff --git a/CLI/Repl.h b/CLI/Repl.h index 11a077ae..cd54b7e0 100644 --- a/CLI/Repl.h +++ b/CLI/Repl.h @@ -3,10 +3,15 @@ #include "lua.h" +#include #include +using AddCompletionCallback = std::function; + // Note: These are internal functions which are being exposed in a header // so they can be included by unit tests. -int replMain(int argc, char** argv); void setupState(lua_State* L); std::string runCode(lua_State* L, const std::string& source); +void getCompletions(lua_State* L, const std::string& editBuffer, const AddCompletionCallback& addCompletionCallback); + +int replMain(int argc, char** argv); diff --git a/Compiler/src/BytecodeBuilder.cpp b/Compiler/src/BytecodeBuilder.cpp index e6d02454..09f06b68 100644 --- a/Compiler/src/BytecodeBuilder.cpp +++ b/Compiler/src/BytecodeBuilder.cpp @@ -6,8 +6,6 @@ #include #include -LUAU_FASTFLAGVARIABLE(LuauBytecodeV2Write, false) - namespace Luau { @@ -510,7 +508,7 @@ uint32_t BytecodeBuilder::getDebugPC() const void BytecodeBuilder::finalize() { LUAU_ASSERT(bytecode.empty()); - bytecode = char(FFlag::LuauBytecodeV2Write ? LBC_VERSION_FUTURE : LBC_VERSION); + bytecode = char(LBC_VERSION_FUTURE); writeStringTable(bytecode); @@ -611,9 +609,7 @@ void BytecodeBuilder::writeFunction(std::string& ss, uint32_t id) const writeVarInt(ss, child); // debug info - if (FFlag::LuauBytecodeV2Write) - writeVarInt(ss, func.debuglinedefined); - + writeVarInt(ss, func.debuglinedefined); writeVarInt(ss, func.debugname); bool hasLines = true; diff --git a/Compiler/src/Compiler.cpp b/Compiler/src/Compiler.cpp index e4253adc..656a9926 100644 --- a/Compiler/src/Compiler.cpp +++ b/Compiler/src/Compiler.cpp @@ -15,7 +15,6 @@ #include #include -LUAU_FASTFLAGVARIABLE(LuauCompileTableIndexOpt, false) LUAU_FASTFLAG(LuauCompileSelectBuiltin2) namespace Luau @@ -1182,18 +1181,9 @@ struct Compiler const AstExprTable::Item& item = expr->items.data[i]; LUAU_ASSERT(item.key); // no list portion => all items have keys - if (FFlag::LuauCompileTableIndexOpt) - { - const Constant* ckey = constants.find(item.key); + const Constant* ckey = constants.find(item.key); - indexSize += (ckey && ckey->type == Constant::Type_Number && ckey->valueNumber == double(indexSize + 1)); - } - else - { - AstExprConstantNumber* ckey = item.key->as(); - - indexSize += (ckey && ckey->value == double(indexSize + 1)); - } + indexSize += (ckey && ckey->type == Constant::Type_Number && ckey->valueNumber == double(indexSize + 1)); } // we only perform the optimization if we don't have any other []-keys @@ -1295,43 +1285,10 @@ struct Compiler { RegScope rsi(this); - if (FFlag::LuauCompileTableIndexOpt) - { - LValue lv = compileLValueIndex(reg, key, rsi); - uint8_t rv = compileExprAuto(value, rsi); + LValue lv = compileLValueIndex(reg, key, rsi); + uint8_t rv = compileExprAuto(value, rsi); - compileAssign(lv, rv); - } - else - { - // Optimization: use SETTABLEKS/SETTABLEN for literal keys, this happens often as part of usual table construction syntax - if (AstExprConstantString* ckey = key->as()) - { - BytecodeBuilder::StringRef cname = sref(ckey->value); - int32_t cid = bytecode.addConstantString(cname); - if (cid < 0) - CompileError::raise(expr->location, "Exceeded constant limit; simplify the code to compile"); - - uint8_t rv = compileExprAuto(value, rsi); - - bytecode.emitABC(LOP_SETTABLEKS, rv, reg, uint8_t(BytecodeBuilder::getStringHash(cname))); - bytecode.emitAux(cid); - } - else if (AstExprConstantNumber* ckey = key->as(); - ckey && ckey->value >= 1 && ckey->value <= 256 && double(int(ckey->value)) == ckey->value) - { - uint8_t rv = compileExprAuto(value, rsi); - - bytecode.emitABC(LOP_SETTABLEN, rv, reg, uint8_t(int(ckey->value) - 1)); - } - else - { - uint8_t rk = compileExprAuto(key, rsi); - uint8_t rv = compileExprAuto(value, rsi); - - bytecode.emitABC(LOP_SETTABLE, rv, reg, rk); - } - } + compileAssign(lv, rv); } // items without a key are set using SETLIST so that we can initialize large arrays quickly else @@ -1439,8 +1396,7 @@ struct Compiler uint8_t rt = compileExprAuto(expr->expr, rs); uint8_t i = uint8_t(int(cv->valueNumber) - 1); - if (FFlag::LuauCompileTableIndexOpt) - setDebugLine(expr->index); + setDebugLine(expr->index); bytecode.emitABC(LOP_GETTABLEN, target, rt, i); } @@ -1453,8 +1409,7 @@ struct Compiler if (cid < 0) CompileError::raise(expr->location, "Exceeded constant limit; simplify the code to compile"); - if (FFlag::LuauCompileTableIndexOpt) - setDebugLine(expr->index); + setDebugLine(expr->index); bytecode.emitABC(LOP_GETTABLEKS, target, rt, uint8_t(BytecodeBuilder::getStringHash(iname))); bytecode.emitAux(cid); @@ -1853,8 +1808,7 @@ struct Compiler void compileLValueUse(const LValue& lv, uint8_t reg, bool set) { - if (FFlag::LuauCompileTableIndexOpt) - setDebugLine(lv.location); + setDebugLine(lv.location); switch (lv.kind) { diff --git a/Sources.cmake b/Sources.cmake index 4ab1d98e..773f6f35 100644 --- a/Sources.cmake +++ b/Sources.cmake @@ -193,11 +193,12 @@ if(TARGET Luau.Analyze.CLI) CLI/Analyze.cpp) endif() -if (TARGET Luau.Ast.CLI) +if(TARGET Luau.Ast.CLI) target_sources(Luau.Ast.CLI PRIVATE + CLI/Ast.cpp CLI/FileUtils.h CLI/FileUtils.cpp - CLI/Ast.cpp) + ) endif() if(TARGET Luau.UnitTest) diff --git a/VM/src/lbuiltins.cpp b/VM/src/lbuiltins.cpp index ecc14e87..718d387d 100644 --- a/VM/src/lbuiltins.cpp +++ b/VM/src/lbuiltins.cpp @@ -1098,7 +1098,7 @@ static int luauF_select(lua_State* L, StkId res, TValue* arg0, int nresults, Stk int i = int(nvalue(arg0)); // i >= 1 && i <= n - if (unsigned(i - 1) <= unsigned(n)) + if (unsigned(i - 1) < unsigned(n)) { setobj2s(L, res, L->base - n + (i - 1)); return 1; diff --git a/VM/src/lgcdebug.cpp b/VM/src/lgcdebug.cpp index 906fb0d0..ce196520 100644 --- a/VM/src/lgcdebug.cpp +++ b/VM/src/lgcdebug.cpp @@ -250,6 +250,8 @@ void luaC_validate(lua_State* L) if (FFlag::LuauGcPagedSweep) { + validategco(L, NULL, obj2gco(g->mainthread)); + luaM_visitgco(L, L, validategco); } else @@ -565,6 +567,8 @@ void luaC_dump(lua_State* L, void* file, const char* (*categoryName)(lua_State* if (FFlag::LuauGcPagedSweep) { + dumpgco(f, NULL, obj2gco(g->mainthread)); + luaM_visitgco(L, f, dumpgco); } else diff --git a/VM/src/lmem.cpp b/VM/src/lmem.cpp index de85cf59..19617b8c 100644 --- a/VM/src/lmem.cpp +++ b/VM/src/lmem.cpp @@ -8,6 +8,76 @@ #include +/* + * Luau heap uses a size-segregated page structure, with individual pages and large allocations + * allocated using system heap (via frealloc callback). + * + * frealloc callback serves as a general, if slow, allocation callback that can allocate, free or + * resize allocations: + * + * void* frealloc(void* ud, void* ptr, size_t oldsize, size_t newsize); + * + * frealloc(ud, NULL, 0, x) creates a new block of size x + * frealloc(ud, p, x, 0) frees the block p (must return NULL) + * frealloc(ud, NULL, 0, 0) does nothing, equivalent to free(NULL) + * + * frealloc returns NULL if it cannot create or reallocate the area + * (any reallocation to an equal or smaller size cannot fail!) + * + * On top of this, Luau implements heap storage which is split into two types of allocations: + * + * - GCO, short for "garbage collected objects" + * - other objects (for example, arrays stored inside table objects) + * + * The heap layout for these two allocation types is a bit different. + * + * All GCO are allocated in pages, which is a block of memory of ~16K in size that has a page header + * (lua_Page). Each page contains 1..N blocks of the same size, where N is selected to fill the page + * completely. This amortizes the allocation cost and increases locality. Each GCO block starts with + * the GC header (GCheader) which contains the object type, mark bits and other GC metadata. If the + * GCO block is free (not used), then it must have the type set to TNIL; in this case the block can + * be part of the per-page free list, the link for that list is stored after the header (freegcolink). + * + * Importantly, the GCO block doesn't have any back references to the page it's allocated in, so it's + * impossible to free it in isolation - GCO blocks are freed by sweeping the pages they belong to, + * using luaM_freegco which must specify the page; this is called by page sweeper that traverses the + * entire page's worth of objects. For this reason it's also important that freed GCO blocks keep the + * GC header intact and accessible (with type = NIL) so that the sweeper can access it. + * + * Some GCOs are too large to fit in a 16K page without excessive fragmentation (the size threshold is + * currently 512 bytes); in this case, we allocate a dedicated small page with just a single block's worth + * storage space, but that requires allocating an extra page header. In effect large GCOs are a little bit + * less memory efficient, but this allows us to uniformly sweep small and large GCOs using page lists. + * + * All GCO pages are linked in a large intrusive linked list (global_State::allgcopages). Additionally, + * for each block size there's a page free list that contains pages that have at least one free block + * (global_State::freegcopages). This free list is used to make sure object allocation is O(1). + * + * Compared to GCOs, regular allocations have two important differences: they can be freed in isolation, + * and they don't start with a GC header. Because of this, each allocation is prefixed with block metadata, + * which contains the pointer to the page for allocated blocks, and the pointer to the next free block + * inside the page for freed blocks. + * For regular allocations that are too large to fit in a page (using the same threshold of 512 bytes), + * we don't allocate a separate page, instead simply using frealloc to allocate a vanilla block of memory. + * + * Just like GCO pages, we store a page free list (global_State::freepages) that allows O(1) allocation; + * there is no global list for non-GCO pages since we never need to traverse them directly. + * + * In both cases, we pick the page by computing the size class from the block size which rounds the block + * size up to reduce the chance that we'll allocate pages that have very few allocated blocks. The size + * class strategy is determined by SizeClassConfig constructor. + * + * Note that when the last block in a page is freed, we immediately free the page with frealloc - the + * memory manager doesn't currently attempt to keep unused memory around. This can result in excessive + * allocation traffic and can be mitigated by adding a page cache in the future. + * + * For both GCO and non-GCO pages, the per-page block allocation combines bump pointer style allocation + * (lua_Page::freeNext) and per-page free list (lua_Page::freeList). We use the bump allocator to allocate + * the contents of the page, and the free list for further reuse; this allows shorter page setup times + * which results in less variance between allocation cost, as well as tighter sweep bounds for newly + * allocated pages. + */ + LUAU_FASTFLAG(LuauGcPagedSweep) #ifndef __has_feature @@ -56,6 +126,7 @@ static_assert(offsetof(GCObject, ts) == 0, "TString data must be located at the const size_t kSizeClasses = LUA_SIZECLASSES; const size_t kMaxSmallSize = 512; const size_t kPageSize = 16 * 1024 - 24; // slightly under 16KB since that results in less fragmentation due to heap metadata + const size_t kBlockHeader = sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*); // suitable for aligning double & void* on all platforms const size_t kGCOLinkOffset = (sizeof(GCheader) + sizeof(void*) - 1) & ~(sizeof(void*) - 1); // GCO pages contain freelist links after the GC header @@ -107,24 +178,6 @@ const SizeClassConfig kSizeClassConfig; #define metadata(block) (*(void**)(block)) #define freegcolink(block) (*(void**)((char*)block + kGCOLinkOffset)) -/* -** About the realloc function: -** void * frealloc (void *ud, void *ptr, size_t osize, size_t nsize); -** (`osize' is the old size, `nsize' is the new size) -** -** Lua ensures that (ptr == NULL) iff (osize == 0). -** -** * frealloc(ud, NULL, 0, x) creates a new block of size `x' -** -** * frealloc(ud, p, x, 0) frees the block `p' -** (in this specific case, frealloc must return NULL). -** particularly, frealloc(ud, NULL, 0, 0) does nothing -** (which is equivalent to free(NULL) in ANSI C) -** -** frealloc returns NULL if it cannot create or reallocate the area -** (any reallocation to an equal or smaller size cannot fail!) -*/ - struct lua_Page { // list of pages with free blocks @@ -135,13 +188,12 @@ struct lua_Page lua_Page* gcolistprev; lua_Page* gcolistnext; - int busyBlocks; - int blockSize; + int pageSize; // page size in bytes, including page header + int blockSize; // block size in bytes, including block header (for non-GCO) - void* freeList; - int freeNext; - - int pageSize; + void* freeList; // next free block in this page; linked with metadata()/freegcolink() + int freeNext; // next free block offset in this page, in bytes; when negative, freeList is used instead + int busyBlocks; // number of blocks allocated out of this page union { @@ -177,7 +229,7 @@ static lua_Page* newpageold(lua_State* L, uint8_t sizeClass) page->gcolistprev = NULL; page->gcolistnext = NULL; - page->busyBlocks = 0; + page->pageSize = kPageSize; page->blockSize = blockSize; // note: we start with the last block in the page and move downward @@ -185,6 +237,7 @@ static lua_Page* newpageold(lua_State* L, uint8_t sizeClass) // additionally, GC stores objects in singly linked lists, and this way the GC lists end up in increasing pointer order page->freeList = NULL; page->freeNext = (blockCount - 1) * blockSize; + page->busyBlocks = 0; // prepend a page to page freelist (which is empty because we only ever allocate a new page when it is!) LUAU_ASSERT(!g->freepages[sizeClass]); @@ -214,7 +267,7 @@ static lua_Page* newpage(lua_State* L, lua_Page** gcopageset, int pageSize, int page->gcolistprev = NULL; page->gcolistnext = NULL; - page->busyBlocks = 0; + page->pageSize = pageSize; page->blockSize = blockSize; // note: we start with the last block in the page and move downward @@ -222,8 +275,7 @@ static lua_Page* newpage(lua_State* L, lua_Page** gcopageset, int pageSize, int // additionally, GC stores objects in singly linked lists, and this way the GC lists end up in increasing pointer order page->freeList = NULL; page->freeNext = (blockCount - 1) * blockSize; - - page->pageSize = pageSize; + page->busyBlocks = 0; if (gcopageset) { @@ -406,8 +458,7 @@ static void* newgcoblock(lua_State* L, int sizeClass) page->next = NULL; } - // the user data is right after the metadata - return (char*)block; + return block; } static void freeblock(lua_State* L, int sizeClass, void* block) @@ -421,6 +472,7 @@ static void freeblock(lua_State* L, int sizeClass, void* block) lua_Page* page = (lua_Page*)metadata(block); LUAU_ASSERT(page && page->busyBlocks > 0); LUAU_ASSERT(size_t(page->blockSize) == kSizeClassConfig.sizeOfClass[sizeClass] + kBlockHeader); + LUAU_ASSERT(block >= page->data && block < (char*)page + page->pageSize); // if the page wasn't in the page free list, it should be now since it got a block! if (!page->freeList && page->freeNext < 0) @@ -455,6 +507,9 @@ static void freeblock(lua_State* L, int sizeClass, void* block) static void freegcoblock(lua_State* L, int sizeClass, void* block, lua_Page* page) { LUAU_ASSERT(FFlag::LuauGcPagedSweep); + LUAU_ASSERT(page && page->busyBlocks > 0); + LUAU_ASSERT(page->blockSize == kSizeClassConfig.sizeOfClass[sizeClass]); + LUAU_ASSERT(block >= page->data && block < (char*)page + page->pageSize); global_State* g = L->global; @@ -575,6 +630,8 @@ void luaM_freegco_(lua_State* L, GCObject* block, size_t osize, uint8_t memcat, else { LUAU_ASSERT(page->busyBlocks == 1); + LUAU_ASSERT(size_t(page->blockSize) == osize); + LUAU_ASSERT((void*)block == page->data); freepage(L, &g->allgcopages, page); } @@ -626,8 +683,12 @@ void luaM_getpagewalkinfo(lua_Page* page, char** start, char** end, int* busyBlo int blockCount = (page->pageSize - offsetof(lua_Page, data)) / page->blockSize; - *start = page->data + page->freeNext + page->blockSize; - *end = page->data + blockCount * page->blockSize; + LUAU_ASSERT(page->freeNext >= -page->blockSize && page->freeNext <= (blockCount - 1) * page->blockSize); + + char* data = page->data; // silences ubsan when indexing page->data + + *start = data + page->freeNext + page->blockSize; + *end = data + blockCount * page->blockSize; *busyBlocks = page->busyBlocks; *blockSize = page->blockSize; } @@ -675,7 +736,7 @@ void luaM_visitgco(lua_State* L, void* context, bool (*visitor)(void* context, l for (lua_Page* curr = g->allgcopages; curr;) { - lua_Page* next = curr->gcolistnext; // page blockvisit might destroy the page + lua_Page* next = curr->gcolistnext; // block visit might destroy the page luaM_visitpage(curr, context, visitor); diff --git a/VM/src/lobject.cpp b/VM/src/lobject.cpp index 370c7b28..d5bd76a8 100644 --- a/VM/src/lobject.cpp +++ b/VM/src/lobject.cpp @@ -131,7 +131,7 @@ void luaO_chunkid(char* out, const char* source, size_t bufflen) { size_t l; source++; /* skip the `@' */ - bufflen -= sizeof(" '...' "); + bufflen -= sizeof("..."); l = strlen(source); strcpy(out, ""); if (l > bufflen) @@ -144,7 +144,7 @@ void luaO_chunkid(char* out, const char* source, size_t bufflen) else { /* out = [string "string"] */ size_t len = strcspn(source, "\n\r"); /* stop at first newline */ - bufflen -= sizeof(" [string \"...\"] "); + bufflen -= sizeof("[string \"...\"]"); if (len > bufflen) len = bufflen; strcpy(out, "[string \""); diff --git a/VM/src/lvmexecute.cpp b/VM/src/lvmexecute.cpp index cba3670a..c3b662a2 100644 --- a/VM/src/lvmexecute.cpp +++ b/VM/src/lvmexecute.cpp @@ -609,7 +609,8 @@ static void luau_execute(lua_State* L) if (unsigned(ic) < LUA_VECTOR_SIZE && name[1] == '\0') { - setnvalue(ra, rb->value.v[ic]); + const float* v = rb->value.v; // silences ubsan when indexing v[] + setnvalue(ra, v[ic]); VM_NEXT(); } diff --git a/tests/Compiler.test.cpp b/tests/Compiler.test.cpp index d8af94db..cd7a21d8 100644 --- a/tests/Compiler.test.cpp +++ b/tests/Compiler.test.cpp @@ -605,8 +605,6 @@ RETURN R0 1 TEST_CASE("TableLiteralsIndexConstant") { - ScopedFastFlag sff("LuauCompileTableIndexOpt", true); - // validate that we use SETTTABLEKS for constant variable keys CHECK_EQ("\n" + compileFunction0(R"( local a, b = "key", "value" @@ -2483,8 +2481,6 @@ return TEST_CASE("DebugLineInfoAssignment") { - ScopedFastFlag sff("LuauCompileTableIndexOpt", true); - Luau::BytecodeBuilder bcb; bcb.setDumpFlags(Luau::BytecodeBuilder::Dump_Code | Luau::BytecodeBuilder::Dump_Lines); Luau::compileOrThrow(bcb, R"( diff --git a/tests/Conformance.test.cpp b/tests/Conformance.test.cpp index e580949f..8b58d2ce 100644 --- a/tests/Conformance.test.cpp +++ b/tests/Conformance.test.cpp @@ -492,8 +492,6 @@ TEST_CASE("DateTime") TEST_CASE("Debug") { - ScopedFastFlag sffw("LuauBytecodeV2Write", true); - runConformance("debug.lua"); } diff --git a/tests/Linter.test.cpp b/tests/Linter.test.cpp index d1cc49b2..577415fc 100644 --- a/tests/Linter.test.cpp +++ b/tests/Linter.test.cpp @@ -1392,19 +1392,31 @@ TEST_CASE_FIXTURE(Fixture, "DeprecatedApi") {"DataCost", {typeChecker.numberType, /* deprecated= */ true}}, {"Wait", {typeChecker.anyType, /* deprecated= */ true}}, }; + + TypeId colorType = typeChecker.globalTypes.addType(TableTypeVar{{}, std::nullopt, typeChecker.globalScope->level, Luau::TableState::Sealed}); + + getMutable(colorType)->props = { + {"toHSV", {typeChecker.anyType, /* deprecated= */ true, "Color3:ToHSV"} } + }; + + addGlobalBinding(typeChecker, "Color3", Binding{colorType, {}}); + freeze(typeChecker.globalTypes); LintResult result = lintTyped(R"( return function (i: Instance) i:Wait(1.0) print(i.Name) + print(Color3.toHSV()) + print(Color3.doesntexist, i.doesntexist) -- type error, but this verifies we correctly handle non-existent members return i.DataCost end )"); - REQUIRE_EQ(result.warnings.size(), 2); + REQUIRE_EQ(result.warnings.size(), 3); CHECK_EQ(result.warnings[0].text, "Member 'Instance.Wait' is deprecated"); - CHECK_EQ(result.warnings[1].text, "Member 'Instance.DataCost' is deprecated"); + CHECK_EQ(result.warnings[1].text, "Member 'toHSV' is deprecated, use 'Color3:ToHSV' instead"); + CHECK_EQ(result.warnings[2].text, "Member 'Instance.DataCost' is deprecated"); } TEST_CASE_FIXTURE(Fixture, "TableOperations") @@ -1475,9 +1487,11 @@ _ = (true and true) or true _ = (true and false) and (42 and false) _ = true and true or false -- no warning since this is is a common pattern used as a ternary replacement + +_ = if true then 1 elseif true then 2 else 3 )"); - REQUIRE_EQ(result.warnings.size(), 7); + REQUIRE_EQ(result.warnings.size(), 8); CHECK_EQ(result.warnings[0].text, "Condition has already been checked on line 2"); CHECK_EQ(result.warnings[0].location.begin.line + 1, 4); CHECK_EQ(result.warnings[1].text, "Condition has already been checked on column 5"); @@ -1487,6 +1501,7 @@ _ = true and true or false -- no warning since this is is a common pattern used CHECK_EQ(result.warnings[5].text, "Condition has already been checked on column 6"); CHECK_EQ(result.warnings[6].text, "Condition has already been checked on column 15"); CHECK_EQ(result.warnings[6].location.begin.line + 1, 19); + CHECK_EQ(result.warnings[7].text, "Condition has already been checked on column 8"); } TEST_CASE_FIXTURE(Fixture, "DuplicateConditionsExpr") @@ -1528,4 +1543,19 @@ return foo, moo, a1, a2 CHECK_EQ(result.warnings[3].text, "Function parameter 'self' already defined implicitly"); } +TEST_CASE_FIXTURE(Fixture, "MisleadingAndOr") +{ + LintResult result = lint(R"( +_ = math.random() < 0.5 and true or 42 +_ = math.random() < 0.5 and false or 42 -- misleading +_ = math.random() < 0.5 and nil or 42 -- misleading +_ = math.random() < 0.5 and 0 or 42 +_ = (math.random() < 0.5 and false) or 42 -- currently ignored +)"); + + REQUIRE_EQ(result.warnings.size(), 2); + CHECK_EQ(result.warnings[0].text, "The and-or expression always evaluates to the second alternative because the first alternative is false; consider using if-then-else expression instead"); + CHECK_EQ(result.warnings[1].text, "The and-or expression always evaluates to the second alternative because the first alternative is nil; consider using if-then-else expression instead"); +} + TEST_SUITE_END(); diff --git a/tests/Repl.test.cpp b/tests/Repl.test.cpp index f660bcd3..1f9c9739 100644 --- a/tests/Repl.test.cpp +++ b/tests/Repl.test.cpp @@ -8,9 +8,22 @@ #include #include +#include #include #include +struct Completion +{ + std::string completion; + std::string display; + + bool operator<(Completion const& other) const + { + return std::tie(completion, display) < std::tie(other.completion, other.display); + } +}; + +using CompletionSet = std::set; class ReplFixture { @@ -34,6 +47,27 @@ public: lua_pop(L, 1); return result; } + + CompletionSet getCompletionSet(const char* inputPrefix) + { + CompletionSet result; + int top = lua_gettop(L); + getCompletions(L, inputPrefix, [&result](const std::string& completion, const std::string& display) { + result.insert(Completion{completion, display}); + }); + // Ensure that generating completions doesn't change the position of luau's stack top. + CHECK(top == lua_gettop(L)); + + return result; + } + + bool checkCompletion(const CompletionSet& completions, const std::string& prefix, const std::string& expected) + { + std::string expectedDisplay(expected.substr(0, expected.find_first_of('('))); + Completion expectedCompletion{prefix + expected, expectedDisplay}; + return completions.count(expectedCompletion) == 1; + } + lua_State* L; private: @@ -115,3 +149,61 @@ TEST_CASE_FIXTURE(ReplFixture, "MultipleArguments") } TEST_SUITE_END(); + +TEST_SUITE_BEGIN("ReplCodeCompletion"); + +TEST_CASE_FIXTURE(ReplFixture, "CompleteGlobalVariables") +{ + runCode(L, R"( + myvariable1 = 5 + myvariable2 = 5 +)"); + CompletionSet completions = getCompletionSet("myvar"); + + std::string prefix = ""; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "myvariable1")); + CHECK(checkCompletion(completions, prefix, "myvariable2")); +} + +TEST_CASE_FIXTURE(ReplFixture, "CompleteTableKeys") +{ + runCode(L, R"( + t = { color = "red", size = 1, shape = "circle" } +)"); + { + CompletionSet completions = getCompletionSet("t."); + + std::string prefix = "t."; + CHECK(completions.size() == 3); + CHECK(checkCompletion(completions, prefix, "color")); + CHECK(checkCompletion(completions, prefix, "size")); + CHECK(checkCompletion(completions, prefix, "shape")); + } + + { + CompletionSet completions = getCompletionSet("t.s"); + + std::string prefix = "t."; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "size")); + CHECK(checkCompletion(completions, prefix, "shape")); + } +} + +TEST_CASE_FIXTURE(ReplFixture, "StringMethods") +{ + runCode(L, R"( + s = "" +)"); + { + CompletionSet completions = getCompletionSet("s:l"); + + std::string prefix = "s:"; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "len(")); + CHECK(checkCompletion(completions, prefix, "lower(")); + } +} + +TEST_SUITE_END(); diff --git a/tests/TypeInfer.aliases.test.cpp b/tests/TypeInfer.aliases.test.cpp index 76ab23b3..a8729268 100644 --- a/tests/TypeInfer.aliases.test.cpp +++ b/tests/TypeInfer.aliases.test.cpp @@ -595,4 +595,65 @@ TEST_CASE_FIXTURE(Fixture, "generic_typevars_are_not_considered_to_escape_their_ LUAU_REQUIRE_NO_ERRORS(result); } +/* + * The two-pass alias definition system starts by ascribing a free TypeVar to each alias. It then + * circles back to fill in the actual type later on. + * + * If this free type is unified with something degenerate like `any`, we need to take extra care + * to ensure that the alias actually binds to the type that the user expected. + */ +TEST_CASE_FIXTURE(Fixture, "forward_declared_alias_is_not_clobbered_by_prior_unification_with_any") +{ + ScopedFastFlag sff[] = { + {"LuauTwoPassAliasDefinitionFix", true} + }; + + CheckResult result = check(R"( + local function x() + local y: FutureType = {}::any + return 1 + end + type FutureType = { foo: typeof(x()) } + local d: FutureType = { smth = true } -- missing error, 'd' is resolved to 'any' + )"); + + CHECK_EQ("{| foo: number |}", toString(requireType("d"), {true})); + + LUAU_REQUIRE_ERROR_COUNT(1, result); +} + +TEST_CASE_FIXTURE(Fixture, "forward_declared_alias_is_not_clobbered_by_prior_unification_with_any_2") +{ + ScopedFastFlag sff[] = { + {"LuauTwoPassAliasDefinitionFix", true}, + + // We also force these two flags because this surfaced an unfortunate interaction. + {"LuauErrorRecoveryType", true}, + {"LuauQuantifyInPlace2", true}, + }; + + CheckResult result = check(R"( + local B = {} + B.bar = 4 + + function B:smth1() + local self: FutureIntersection = self + self.foo = 4 + return 4 + end + + function B:smth2() + local self: FutureIntersection = self + self.bar = 5 -- error, even though we should have B part with bar + end + + type A = { foo: typeof(B.smth1({foo=3})) } -- trick toposort into sorting functions before types + type B = typeof(B) + + type FutureIntersection = A & B + )"); + + LUAU_REQUIRE_NO_ERRORS(result); +} + TEST_SUITE_END(); diff --git a/tests/TypeInfer.builtins.test.cpp b/tests/TypeInfer.builtins.test.cpp index 6730bedb..df06884d 100644 --- a/tests/TypeInfer.builtins.test.cpp +++ b/tests/TypeInfer.builtins.test.cpp @@ -7,8 +7,6 @@ #include "doctest.h" -LUAU_FASTFLAG(LuauFixTonumberReturnType) - using namespace Luau; LUAU_FASTFLAG(LuauUseCommittingTxnLog) @@ -850,11 +848,8 @@ TEST_CASE_FIXTURE(Fixture, "tonumber_returns_optional_number_type") local b: number = tonumber('asdf') )"); - if (FFlag::LuauFixTonumberReturnType) - { - LUAU_REQUIRE_ERROR_COUNT(1, result); - CHECK_EQ("Type 'number?' could not be converted into 'number'", toString(result.errors[0])); - } + LUAU_REQUIRE_ERROR_COUNT(1, result); + CHECK_EQ("Type 'number?' could not be converted into 'number'", toString(result.errors[0])); } TEST_CASE_FIXTURE(Fixture, "tonumber_returns_optional_number_type2") @@ -893,7 +888,7 @@ TEST_CASE_FIXTURE(Fixture, "assert_removes_falsy_types") { ScopedFastFlag sff[]{ {"LuauAssertStripsFalsyTypes", true}, - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, }; CheckResult result = check(R"( @@ -910,7 +905,7 @@ TEST_CASE_FIXTURE(Fixture, "assert_removes_falsy_types_even_from_type_pack_tail_ { ScopedFastFlag sff[]{ {"LuauAssertStripsFalsyTypes", true}, - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, }; CheckResult result = check(R"( @@ -927,7 +922,7 @@ TEST_CASE_FIXTURE(Fixture, "assert_returns_false_and_string_iff_it_knows_the_fir { ScopedFastFlag sff[]{ {"LuauAssertStripsFalsyTypes", true}, - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, }; CheckResult result = check(R"( diff --git a/tests/TypeInfer.provisional.test.cpp b/tests/TypeInfer.provisional.test.cpp index e5eb0dca..2bcd840c 100644 --- a/tests/TypeInfer.provisional.test.cpp +++ b/tests/TypeInfer.provisional.test.cpp @@ -262,7 +262,7 @@ TEST_CASE_FIXTURE(Fixture, "lvalue_equals_another_lvalue_with_no_overlap") // Just needs to fully support equality refinement. Which is annoying without type states. TEST_CASE_FIXTURE(Fixture, "discriminate_from_x_not_equal_to_nil") { - ScopedFastFlag sff{"LuauDiscriminableUnions", true}; + ScopedFastFlag sff{"LuauDiscriminableUnions2", true}; CheckResult result = check(R"( type T = {x: string, y: number} | {x: nil, y: nil} @@ -616,4 +616,76 @@ local a: Self CHECK_EQ(toString(requireType("a")), "Table
"); } +TEST_CASE_FIXTURE(Fixture, "do_not_ice_when_trying_to_pick_first_of_generic_type_pack") +{ + ScopedFastFlag sff[]{ + {"LuauQuantifyInPlace2", true}, + {"LuauReturnAnyInsteadOfICE", true}, + }; + + // In-place quantification causes these types to have the wrong types but only because of nasty interaction with prototyping. + // The type of f is initially () -> free1... + // Then the prototype iterator advances, and checks the function expression assigned to g, which has the type () -> free2... + // In the body it calls f and returns what f() returns. This binds free2... with free1..., causing f and g to have same types. + // We then quantify g, leaving it with the final type () -> a... + // Because free1... and free2... were bound, in combination with in-place quantification, f's return type was also turned into a... + // Then the check iterator catches up, and checks the body of f, and attempts to quantify it too. + // Alas, one of the requirements for quantification is that a type must contain free types. () -> a... has no free types. + // Thus the quantification for f was no-op, which explains why f does not have any type parameters. + // Calling f() will attempt to instantiate the function type, which turns generics in type binders into to free types. + // However, instantiations only converts generics contained within the type binders of a function, so instantiation was also no-op. + // Which means that calling f() simply returned a... rather than an instantiation of it. And since the call site was not in tail position, + // picking first element in a... triggers an ICE because calls returning generic packs are unexpected. + CheckResult result = check(R"( + local function f() end + + local g = function() return f() end + + local x = (f()) -- should error: no return values to assign from the call to f + )"); + + LUAU_REQUIRE_NO_ERRORS(result); + + // f and g should have the type () -> () + CHECK_EQ("() -> (a...)", toString(requireType("f"))); + CHECK_EQ("() -> (a...)", toString(requireType("g"))); + CHECK_EQ("any", toString(requireType("x"))); // any is returned instead of ICE for now +} + +TEST_CASE_FIXTURE(Fixture, "specialization_binds_with_prototypes_too_early") +{ + CheckResult result = check(R"( + local function id(x) return x end + local n2n: (number) -> number = id + local s2s: (string) -> string = id + )"); + + LUAU_REQUIRE_ERRORS(result); // Should not have any errors. +} + +TEST_CASE_FIXTURE(Fixture, "weird_fail_to_unify_type_pack") +{ + ScopedFastFlag sff{"LuauQuantifyInPlace2", true}; + + CheckResult result = check(R"( + local function f() return end + local g = function() return f() end + )"); + + LUAU_REQUIRE_ERRORS(result); // Should not have any errors. +} + +TEST_CASE_FIXTURE(Fixture, "weird_fail_to_unify_variadic_pack") +{ + ScopedFastFlag sff{"LuauQuantifyInPlace2", true}; + + CheckResult result = check(R"( + --!strict + local function f(...) return ... end + local g = function(...) return f(...) end + )"); + + LUAU_REQUIRE_ERRORS(result); // Should not have any errors. +} + TEST_SUITE_END(); diff --git a/tests/TypeInfer.refinements.test.cpp b/tests/TypeInfer.refinements.test.cpp index 3a610c3a..48e6be6a 100644 --- a/tests/TypeInfer.refinements.test.cpp +++ b/tests/TypeInfer.refinements.test.cpp @@ -6,7 +6,7 @@ #include "doctest.h" -LUAU_FASTFLAG(LuauDiscriminableUnions) +LUAU_FASTFLAG(LuauDiscriminableUnions2) LUAU_FASTFLAG(LuauWeakEqConstraint) LUAU_FASTFLAG(LuauQuantifyInPlace2) @@ -262,7 +262,7 @@ TEST_CASE_FIXTURE(Fixture, "typeguard_only_look_up_types_from_global_scope") end )"); - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) { LUAU_REQUIRE_NO_ERRORS(result); @@ -435,7 +435,7 @@ TEST_CASE_FIXTURE(Fixture, "lvalue_is_equal_to_a_term") TEST_CASE_FIXTURE(Fixture, "term_is_equal_to_an_lvalue") { ScopedFastFlag sff[] = { - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, {"LuauSingletonTypes", true}, }; @@ -485,7 +485,7 @@ TEST_CASE_FIXTURE(Fixture, "lvalue_is_not_nil") TEST_CASE_FIXTURE(Fixture, "free_type_is_equal_to_an_lvalue") { - ScopedFastFlag sff{"LuauDiscriminableUnions", true}; + ScopedFastFlag sff{"LuauDiscriminableUnions2", true}; ScopedFastFlag sff2{"LuauWeakEqConstraint", true}; CheckResult result = check(R"( @@ -589,7 +589,7 @@ TEST_CASE_FIXTURE(Fixture, "type_narrow_to_vector") end )"); - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) { LUAU_REQUIRE_NO_ERRORS(result); } @@ -1002,7 +1002,7 @@ TEST_CASE_FIXTURE(Fixture, "apply_refinements_on_astexprindexexpr_whose_subscrip TEST_CASE_FIXTURE(Fixture, "discriminate_from_truthiness_of_x") { ScopedFastFlag sff[] = { - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, {"LuauParseSingletonTypes", true}, {"LuauSingletonTypes", true}, }; @@ -1028,7 +1028,7 @@ TEST_CASE_FIXTURE(Fixture, "discriminate_from_truthiness_of_x") TEST_CASE_FIXTURE(Fixture, "discriminate_tag") { ScopedFastFlag sff[] = { - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, {"LuauParseSingletonTypes", true}, {"LuauSingletonTypes", true}, }; @@ -1069,7 +1069,7 @@ TEST_CASE_FIXTURE(Fixture, "narrow_boolean_to_true_or_false") ScopedFastFlag sff[]{ {"LuauParseSingletonTypes", true}, {"LuauSingletonTypes", true}, - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, {"LuauAssertStripsFalsyTypes", true}, }; @@ -1094,7 +1094,7 @@ TEST_CASE_FIXTURE(Fixture, "discriminate_on_properties_of_disjoint_tables_where_ ScopedFastFlag sff[]{ {"LuauParseSingletonTypes", true}, {"LuauSingletonTypes", true}, - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, {"LuauAssertStripsFalsyTypes", true}, }; @@ -1118,7 +1118,7 @@ TEST_CASE_FIXTURE(Fixture, "discriminate_on_properties_of_disjoint_tables_where_ TEST_CASE_FIXTURE(RefinementClassFixture, "discriminate_from_isa_of_x") { ScopedFastFlag sff[] = { - {"LuauDiscriminableUnions", true}, + {"LuauDiscriminableUnions2", true}, {"LuauParseSingletonTypes", true}, {"LuauSingletonTypes", true}, }; @@ -1157,7 +1157,7 @@ TEST_CASE_FIXTURE(RefinementClassFixture, "typeguard_cast_free_table_to_vector") end )"); - if (FFlag::LuauDiscriminableUnions) + if (FFlag::LuauDiscriminableUnions2) LUAU_REQUIRE_NO_ERRORS(result); else { diff --git a/tests/TypeInfer.test.cpp b/tests/TypeInfer.test.cpp index ead3d762..531a382f 100644 --- a/tests/TypeInfer.test.cpp +++ b/tests/TypeInfer.test.cpp @@ -5164,4 +5164,151 @@ function x:Destroy(): () end LUAU_REQUIRE_ERRORS(result); } +TEST_CASE_FIXTURE(Fixture, "do_not_modify_imported_types_2") +{ + ScopedFastFlag immutableTypes{"LuauImmutableTypes", true}; + + fileResolver.source["game/A"] = R"( +export type Type = { x: { a: number } } +return {} + )"; + + fileResolver.source["game/B"] = R"( +local types = require(game.A) +type Type = types.Type +local x: Type = { x = { a = 2 } } +type Rename = typeof(x.x) + )"; + + CheckResult result = frontend.check("game/B"); + LUAU_REQUIRE_NO_ERRORS(result); +} + +TEST_CASE_FIXTURE(Fixture, "do_not_modify_imported_types_3") +{ + ScopedFastFlag immutableTypes{"LuauImmutableTypes", true}; + + fileResolver.source["game/A"] = R"( +local y = setmetatable({}, {}) +export type Type = { x: typeof(y) } +return { x = y } + )"; + + fileResolver.source["game/B"] = R"( +local types = require(game.A) +type Type = types.Type +local x: Type = types +type Rename = typeof(x.x) + )"; + + CheckResult result = frontend.check("game/B"); + LUAU_REQUIRE_NO_ERRORS(result); +} + +TEST_CASE_FIXTURE(Fixture, "indexing_on_string_singletons") +{ + ScopedFastFlag sff[]{ + {"LuauDiscriminableUnions2", true}, + {"LuauRefactorTypeVarQuestions", true}, + {"LuauSingletonTypes", true}, + }; + + CheckResult result = check(R"( + local a: string = "hi" + if a == "hi" then + local x = a:byte() + end + )"); + + LUAU_REQUIRE_NO_ERRORS(result); + + CHECK_EQ(R"("hi")", toString(requireTypeAtPosition({3, 22}))); +} + +TEST_CASE_FIXTURE(Fixture, "indexing_on_union_of_string_singletons") +{ + ScopedFastFlag sff[]{ + {"LuauDiscriminableUnions2", true}, + {"LuauRefactorTypeVarQuestions", true}, + {"LuauSingletonTypes", true}, + }; + + CheckResult result = check(R"( + local a: string = "hi" + if a == "hi" or a == "bye" then + local x = a:byte() + end + )"); + + LUAU_REQUIRE_NO_ERRORS(result); + + CHECK_EQ(R"("bye" | "hi")", toString(requireTypeAtPosition({3, 22}))); +} + +TEST_CASE_FIXTURE(Fixture, "taking_the_length_of_string_singleton") +{ + ScopedFastFlag sff[]{ + {"LuauDiscriminableUnions2", true}, + {"LuauRefactorTypeVarQuestions", true}, + {"LuauSingletonTypes", true}, + {"LuauLengthOnCompositeType", true}, + }; + + CheckResult result = check(R"( + local a: string = "hi" + if a == "hi" then + local x = #a + end + )"); + + LUAU_REQUIRE_NO_ERRORS(result); + + CHECK_EQ(R"("hi")", toString(requireTypeAtPosition({3, 23}))); +} + +TEST_CASE_FIXTURE(Fixture, "taking_the_length_of_union_of_string_singleton") +{ + ScopedFastFlag sff[]{ + {"LuauDiscriminableUnions2", true}, + {"LuauRefactorTypeVarQuestions", true}, + {"LuauSingletonTypes", true}, + {"LuauLengthOnCompositeType", true}, + }; + + CheckResult result = check(R"( + local a: string = "hi" + if a == "hi" or a == "bye" then + local x = #a + end + )"); + + LUAU_REQUIRE_NO_ERRORS(result); + + CHECK_EQ(R"("bye" | "hi")", toString(requireTypeAtPosition({3, 23}))); +} + +/* + * When we add new properties to an unsealed table, we should do a level check and promote the property type to be at + * the level of the table. + */ +TEST_CASE_FIXTURE(Fixture, "inferred_properties_of_a_table_should_start_with_the_same_TypeLevel_of_that_table") +{ + CheckResult result = check(R"( + --!strict + local T = {} + + local function f(prop) + T[1] = { + prop = prop, + } + end + + local function g() + local l = T[1].prop + end + )"); + + LUAU_REQUIRE_NO_ERRORS(result); +} + TEST_SUITE_END(); diff --git a/tests/TypeInfer.tryUnify.test.cpp b/tests/TypeInfer.tryUnify.test.cpp index 0aeca096..8c7fb79a 100644 --- a/tests/TypeInfer.tryUnify.test.cpp +++ b/tests/TypeInfer.tryUnify.test.cpp @@ -273,4 +273,21 @@ TEST_CASE_FIXTURE(TryUnifyFixture, "recursive_metatable_getmatchtag") state.tryUnify(&metatable, &variant); } +TEST_CASE_FIXTURE(TryUnifyFixture, "cli_50320_follow_in_any_unification") +{ + ScopedFastFlag sffs[] = { + {"LuauUseCommittingTxnLog", true}, + {"LuauFollowWithCommittingTxnLogInAnyUnification", true}, + }; + + TypePackVar free{FreeTypePack{TypeLevel{}}}; + TypePackVar target{TypePack{}}; + + TypeVar func{FunctionTypeVar{&free, &free}}; + + state.tryUnify(&free, &target); + // Shouldn't assert or error. + state.tryUnify(&func, typeChecker.anyType); +} + TEST_SUITE_END(); diff --git a/tests/conformance/basic.lua b/tests/conformance/basic.lua index de091632..78d90077 100644 --- a/tests/conformance/basic.lua +++ b/tests/conformance/basic.lua @@ -118,9 +118,7 @@ assert((function() return #_G end)() == 0) assert((function() return #{1,2} end)() == 2) assert((function() return #'g' end)() == 1) -local ud = newproxy(true) -getmetatable(ud).__len = function() return 42 end -assert((function() return #ud end)() == 42) +assert((function() local ud = newproxy(true) getmetatable(ud).__len = function() return 42 end return #ud end)() == 42) assert((function() local a = 1 a = -a return a end)() == -1) @@ -325,6 +323,10 @@ assert((function() local t = {6, 9, 7} t[4.5] = 10 return t[4.5] end)() == 10) assert((function() local t = {6, 9, 7} t['a'] = 11 return t['a'] end)() == 11) assert((function() local t = {6, 9, 7} setmetatable(t, { __newindex = function(t,i,v) rawset(t, i * 10, v) end }) t[1] = 17 t[5] = 1 return concat(t[1],t[5],t[50]) end)() == "17,nil,1") +-- userdata access +assert((function() local ud = newproxy(true) getmetatable(ud).__index = function(ud,i) return i * 10 end return ud[2] end)() == 20) +assert((function() local ud = newproxy(true) getmetatable(ud).__index = function() return function(self, i) return i * 10 end end return ud:meow(2) end)() == 20) + -- and/or -- rhs is a constant assert((function() local a = 1 a = a and 2 return a end)() == 2) @@ -462,7 +464,7 @@ assert((function() a = {} b = {} mt = { __eq = function(l, r) return #l == #r en -- metatable ops local function vec3t(x, y, z) - return setmetatable({ x=x, y=y, z=z}, { + return setmetatable({x=x, y=y, z=z}, { __add = function(l, r) return vec3t(l.x + r.x, l.y + r.y, l.z + r.z) end, __sub = function(l, r) return vec3t(l.x - r.x, l.y - r.y, l.z - r.z) end, __mul = function(l, r) return type(r) == "number" and vec3t(l.x * r, l.y * r, l.z * r) or vec3t(l.x * r.x, l.y * r.y, l.z * r.z) end, diff --git a/tests/conformance/debug.lua b/tests/conformance/debug.lua index 8c96ab33..0e410000 100644 --- a/tests/conformance/debug.lua +++ b/tests/conformance/debug.lua @@ -37,6 +37,7 @@ coroutine.resume(co2, 0 / 0, 42) assert(debug.traceback(co2) == "debug.lua:31 function halp\n") assert(debug.info(co2, 0, "l") == 31) +assert(debug.info(co2, 0, "f") == halp) -- info errors function qux(...) diff --git a/tests/conformance/errors.lua b/tests/conformance/errors.lua index d5ff215b..751188be 100644 --- a/tests/conformance/errors.lua +++ b/tests/conformance/errors.lua @@ -260,8 +260,7 @@ local a,b = loadstring(s) assert(not a) --assert(string.find(b, "line 2")) --- Test for CLI-28786 --- The xpcall is intentially going to cause an exception +-- The xpcall is intentionally going to cause an exception -- followed by a forced exception in the error handler. -- If the secondary handler isn't trapped, it will cause -- the unit test to fail. If the xpcall captures the @@ -281,6 +280,19 @@ coroutine.wrap(function() assert(not pcall(debug.getinfo, coroutine.running(), 0, ">")) end)() +-- loadstring chunk truncation +local a,b = loadstring("nope", "@short") +assert(not a and b:match('[^ ]+') == "short:1:") + +local a,b = loadstring("nope", "@" .. string.rep("thisisaverylongstringitssolongthatitwontfitintotheinternalbufferprovidedtovariousdebugfacilities", 10)) +assert(not a and b:match('[^ ]+') == "...wontfitintotheinternalbufferprovidedtovariousdebugfacilitiesthisisaverylongstringitssolongthatitwontfitintotheinternalbufferprovidedtovariousdebugfacilitiesthisisaverylongstringitssolongthatitwontfitintotheinternalbufferprovidedtovariousdebugfacilities:1:") + +local a,b = loadstring("nope", "=short") +assert(not a and b:match('[^ ]+') == "short:1:") + +local a,b = loadstring("nope", "=" .. string.rep("thisisaverylongstringitssolongthatitwontfitintotheinternalbufferprovidedtovariousdebugfacilities", 10)) +assert(not a and b:match('[^ ]+') == "thisisaverylongstringitssolongthatitwontfitintotheinternalbufferprovidedtovariousdebugfacilitiesthisisaverylongstringitssolongthatitwontfitintotheinternalbufferprovidedtovariousdebugfacilitiesthisisaverylongstringitssolongthatitwontfitintotheinternalbuffe:1:") + -- arith errors function ecall(fn, ...) local ok, err = pcall(fn, ...) diff --git a/tests/conformance/gc.lua b/tests/conformance/gc.lua index 409cd224..5804ea7f 100644 --- a/tests/conformance/gc.lua +++ b/tests/conformance/gc.lua @@ -180,6 +180,11 @@ x,y,z=nil collectgarbage() assert(next(a) == string.rep('$', 11)) +-- shrinking tables reduce their capacity; confirming the shrinking is difficult but we can at least test the surface level behavior +a = {}; setmetatable(a, {__mode = 'ks'}) +for i=1,lim do a[{}] = i end +collectgarbage() +assert(next(a) == nil) -- testing userdata collectgarbage("stop") -- stop collection @@ -315,8 +320,6 @@ do end collectgarbage() - end - return('OK') diff --git a/tests/conformance/math.lua b/tests/conformance/math.lua index bfea0e1f..79ea0fb6 100644 --- a/tests/conformance/math.lua +++ b/tests/conformance/math.lua @@ -289,6 +289,7 @@ assert(math.sqrt("4") == 2) assert(math.tanh("0") == 0) assert(math.tan("0") == 0) assert(math.clamp("0", 2, 3) == 2) +assert(math.clamp("4", 2, 3) == 3) assert(math.sign("2") == 1) assert(math.sign("-2") == -1) assert(math.sign("0") == 0) diff --git a/tests/conformance/vararg.lua b/tests/conformance/vararg.lua index d05f9577..178c56b8 100644 --- a/tests/conformance/vararg.lua +++ b/tests/conformance/vararg.lua @@ -139,6 +139,12 @@ assert(selectmany(1, 10, 20, 30) == "10,20,30") assert(selectone(2, 10, 20, 30) == 20) assert(selectmany(2, 10, 20, 30) == "20,30") +assert(selectone(3, 10, 20, 30) == 30) +assert(selectmany(3, 10, 20, 30) == "30") + +assert(selectone(4, 10, 20, 30) == nil) +assert(selectmany(4, 10, 20, 30) == "") + assert(selectone(-2, 10, 20, 30) == 20) assert(selectmany(-2, 10, 20, 30) == "20,30") diff --git a/tests/conformance/vector.lua b/tests/conformance/vector.lua index 7d18bda3..22d6adfc 100644 --- a/tests/conformance/vector.lua +++ b/tests/conformance/vector.lua @@ -87,9 +87,18 @@ assert(pcall(function() local t = {} rawset(t, vector(0/0, 2, 3), 1) end) == fal -- make sure we cover both builtin and C impl assert(vector(1, 2, 4) == vector("1", "2", "4")) +-- validate component access (both cases) +assert(vector(1, 2, 3).x == 1) +assert(vector(1, 2, 3).X == 1) +assert(vector(1, 2, 3).y == 2) +assert(vector(1, 2, 3).Y == 2) +assert(vector(1, 2, 3).z == 3) +assert(vector(1, 2, 3).Z == 3) + -- additional checks for 4-component vectors if vector_size == 4 then assert(vector(1, 2, 3, 4).w == 4) + assert(vector(1, 2, 3, 4).W == 4) end return 'OK' diff --git a/tools/heapgraph.py b/tools/heapgraph.py index 106db549..d4d29af1 100644 --- a/tools/heapgraph.py +++ b/tools/heapgraph.py @@ -7,10 +7,20 @@ # The result of analysis is a .svg file which can be viewed in a browser # To generate these dumps, use luaC_dump, ideally preceded by luaC_fullgc +import argparse import json import sys import svg +argumentParser = argparse.ArgumentParser(description='Luau heap snapshot analyzer') + +argumentParser.add_argument('--split', dest = 'split', type = str, default = 'none', help = 'Perform additional root split using memory categories', choices = ['none', 'custom', 'all']) + +argumentParser.add_argument('snapshot') +argumentParser.add_argument('snapshotnew', nargs='?') + +arguments = argumentParser.parse_args() + class Node(svg.Node): def __init__(self): svg.Node.__init__(self) @@ -30,14 +40,14 @@ class Node(svg.Node): return "{} ({:,} bytes, {:.1%}); self: {:,} bytes in {:,} objects".format(self.name, self.width, self.width / root.width, self.size, self.count) # load files -if len(sys.argv) == 2: +if arguments.snapshotnew == None: dumpold = None - with open(sys.argv[1]) as f: + with open(arguments.snapshot) as f: dump = json.load(f) else: - with open(sys.argv[1]) as f: + with open(arguments.snapshot) as f: dumpold = json.load(f) - with open(sys.argv[2]) as f: + with open(arguments.snapshotnew) as f: dump = json.load(f) # reachability analysis: how much of the heap is reachable from roots? @@ -111,12 +121,15 @@ while offset < len(queue): if "object" in obj: queue.append((obj["object"], node)) -def annotateContainedCategories(node): +def annotateContainedCategories(node, start): for obj in node.objects: + if obj["cat"] < start: + obj["cat"] = 0 + node.categories.add(obj["cat"]) for child in node.children.values(): - annotateContainedCategories(child) + annotateContainedCategories(child, start) for cat in child.categories: node.categories.add(cat) @@ -172,9 +185,11 @@ def splitIntoCategories(root): return result -# temporarily disabled because it makes FG harder to read, maybe this should be a separate command line option? -if dump["stats"].get("categories") and False: - annotateContainedCategories(root) +if dump["stats"].get("categories") and arguments.split != 'none': + if arguments.split == 'custom': + annotateContainedCategories(root, 128) + else: + annotateContainedCategories(root, 0) root = splitIntoCategories(root) diff --git a/tools/svg.py b/tools/svg.py index 99853fb6..21200eeb 100644 --- a/tools/svg.py +++ b/tools/svg.py @@ -452,7 +452,7 @@ def display(root, title, colors, flip = False): .replace("$gradient-start", gradient_start) .replace("$gradient-end", gradient_end) .replace("$height", str(svgheight)) - .replace("$status", str(svgheight - 16 + 3)) + .replace("$status", str((svgheight - 16 + 3 if flip else 3 * 16 - 3))) .replace("$flip", str(int(flip))) ) From db90c7da48f30a833802f23e0f41daab1ab6c316 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey <403333+asajeffrey@users.noreply.github.com> Date: Fri, 11 Feb 2022 14:38:35 -0600 Subject: [PATCH 06/25] Add a typeToString function to the prototype (#354) * Added Luau.Type.ToString --- .github/workflows/prototyping.yml | 1 + prototyping/Examples.agda | 2 +- prototyping/Examples/Type.agda | 27 +++++++++++++++++++++++ prototyping/Luau/Type.agda | 33 +++++++++++++++++++++++++++++ prototyping/Luau/Type/ToString.agda | 26 +++++++++++++++++++++++ prototyping/Properties.agda | 2 ++ 6 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 prototyping/Examples/Type.agda create mode 100644 prototyping/Luau/Type/ToString.agda diff --git a/.github/workflows/prototyping.yml b/.github/workflows/prototyping.yml index 02e021ee..76c1e3c1 100644 --- a/.github/workflows/prototyping.yml +++ b/.github/workflows/prototyping.yml @@ -4,6 +4,7 @@ on: push: branches: - 'master' + - 'prototyping-*' paths: - '.github/workflows/**' - 'prototyping/**' diff --git a/prototyping/Examples.agda b/prototyping/Examples.agda index fe396eff..212067b7 100644 --- a/prototyping/Examples.agda +++ b/prototyping/Examples.agda @@ -4,4 +4,4 @@ module Examples where import Examples.Syntax import Examples.OpSem import Examples.Run - +import Examples.Type diff --git a/prototyping/Examples/Type.agda b/prototyping/Examples/Type.agda new file mode 100644 index 00000000..859227ee --- /dev/null +++ b/prototyping/Examples/Type.agda @@ -0,0 +1,27 @@ +module Examples.Type where + +open import Agda.Builtin.Equality using (_≡_; refl) +open import FFI.Data.String using (_++_) +open import Luau.Type using (nil; _∪_; _∩_; _⇒_) +open import Luau.Type.ToString using (typeToString) + +ex1 : typeToString(nil) ≡ "nil" +ex1 = refl + +ex2 : typeToString(nil ⇒ nil) ≡ "(nil) -> nil" +ex2 = refl + +ex3 : typeToString(nil ⇒ (nil ⇒ nil)) ≡ "(nil) -> (nil) -> nil" +ex3 = refl + +ex4 : typeToString(nil ∪ (nil ⇒ (nil ⇒ nil))) ≡ "((nil) -> (nil) -> nil)?" +ex4 = refl + +ex5 : typeToString(nil ⇒ ((nil ⇒ nil) ∪ nil)) ≡ "(nil) -> ((nil) -> nil)?" +ex5 = refl + +ex6 : typeToString((nil ⇒ nil) ∪ (nil ⇒ (nil ⇒ nil))) ≡ "((nil) -> nil | (nil) -> (nil) -> nil)" +ex6 = refl + +ex7 : typeToString((nil ⇒ nil) ∪ ((nil ⇒ (nil ⇒ nil)) ∪ nil)) ≡ "((nil) -> nil | (nil) -> (nil) -> nil)?" +ex7 = refl diff --git a/prototyping/Luau/Type.agda b/prototyping/Luau/Type.agda index 8da3a985..6e384c3b 100644 --- a/prototyping/Luau/Type.agda +++ b/prototyping/Luau/Type.agda @@ -1,5 +1,7 @@ module Luau.Type where +open import FFI.Data.Maybe using (Maybe; just; nothing) + data Type : Set where nil : Type _⇒_ : Type → Type → Type @@ -8,3 +10,34 @@ data Type : Set where _∪_ : Type → Type → Type _∩_ : Type → Type → Type +src : Type → Type +src nil = none +src (S ⇒ T) = S +src none = none +src any = any +src (S ∪ T) = (src S) ∪ (src T) +src (S ∩ T) = (src S) ∩ (src T) + +tgt : Type → Type +tgt nil = none +tgt (S ⇒ T) = T +tgt none = none +tgt any = any +tgt (S ∪ T) = (tgt S) ∪ (tgt T) +tgt (S ∩ T) = (tgt S) ∩ (tgt T) + +optional : Type → Type +optional nil = nil +optional (T ∪ nil) = (T ∪ nil) +optional T = (T ∪ nil) + +normalizeOptional : Type → Type +normalizeOptional (S ∪ T) with normalizeOptional S | normalizeOptional T +normalizeOptional (S ∪ T) | (S′ ∪ nil) | (T′ ∪ nil) = (S′ ∪ T′) ∪ nil +normalizeOptional (S ∪ T) | S′ | (T′ ∪ nil) = (S′ ∪ T′) ∪ nil +normalizeOptional (S ∪ T) | (S′ ∪ nil) | T′ = (S′ ∪ T′) ∪ nil +normalizeOptional (S ∪ T) | S′ | nil = optional S′ +normalizeOptional (S ∪ T) | nil | T′ = optional T′ +normalizeOptional (S ∪ T) | S′ | T′ = S′ ∪ T′ +normalizeOptional T = T + diff --git a/prototyping/Luau/Type/ToString.agda b/prototyping/Luau/Type/ToString.agda new file mode 100644 index 00000000..698d6e8e --- /dev/null +++ b/prototyping/Luau/Type/ToString.agda @@ -0,0 +1,26 @@ +module Luau.Type.ToString where + +open import FFI.Data.String using (String; _++_) +open import Luau.Type using (Type; nil; _⇒_; none; any; _∪_; _∩_; normalizeOptional) + +{-# TERMINATING #-} +typeToString : Type → String +typeToStringᵁ : Type → String +typeToStringᴵ : Type → String + +typeToString nil = "nil" +typeToString (S ⇒ T) = "(" ++ (typeToString S) ++ ") -> " ++ (typeToString T) +typeToString none = "none" +typeToString any = "any" +typeToString (S ∪ T) with normalizeOptional(S ∪ T) +typeToString (S ∪ T) | ((S′ ⇒ T′) ∪ nil) = "(" ++ typeToString (S′ ⇒ T′) ++ ")?" +typeToString (S ∪ T) | (S′ ∪ nil) = typeToString S′ ++ "?" +typeToString (S ∪ T) | (S′ ∪ T′) = "(" ++ typeToStringᵁ (S ∪ T) ++ ")" +typeToString (S ∪ T) | T′ = typeToString T′ +typeToString (S ∩ T) = "(" ++ typeToStringᴵ (S ∩ T) ++ ")" + +typeToStringᵁ (S ∪ T) = (typeToStringᵁ S) ++ " | " ++ (typeToStringᵁ T) +typeToStringᵁ T = typeToString T + +typeToStringᴵ (S ∩ T) = (typeToStringᴵ S) ++ " & " ++ (typeToStringᴵ T) +typeToStringᴵ T = typeToString T diff --git a/prototyping/Properties.agda b/prototyping/Properties.agda index a3c685d1..b08a3a81 100644 --- a/prototyping/Properties.agda +++ b/prototyping/Properties.agda @@ -1,3 +1,5 @@ module Properties where import Properties.Dec +import Properties.Step +import Properties.Remember From e0a9bc191aef922dbc019ff08c4a3125ce3ac749 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey <403333+asajeffrey@users.noreply.github.com> Date: Fri, 11 Feb 2022 19:03:26 -0600 Subject: [PATCH 07/25] Prototype: added syntax for optional type annotations (#358) --- prototyping/Examples/OpSem.agda | 8 +-- prototyping/Examples/Run.agda | 9 +-- prototyping/Examples/Syntax.agda | 18 +++--- prototyping/Interpreter.agda | 2 +- prototyping/Luau/Heap.agda | 31 ++++++----- prototyping/Luau/OpSem.agda | 32 +++++------ prototyping/Luau/Run.agda | 10 ++-- prototyping/Luau/RuntimeError.agda | 10 ++-- prototyping/Luau/RuntimeError/ToString.agda | 7 ++- prototyping/Luau/Substitution.agda | 16 +++--- prototyping/Luau/Syntax.agda | 62 +++++++++++++++------ prototyping/Luau/Syntax/FromJSON.agda | 20 +++---- prototyping/Luau/Syntax/ToString.agda | 37 +++++++----- prototyping/Luau/Value.agda | 4 +- prototyping/PrettyPrinter.agda | 2 +- prototyping/Properties/Step.agda | 26 ++++----- 16 files changed, 165 insertions(+), 129 deletions(-) diff --git a/prototyping/Examples/OpSem.agda b/prototyping/Examples/OpSem.agda index 4043869e..ec8bce7b 100644 --- a/prototyping/Examples/OpSem.agda +++ b/prototyping/Examples/OpSem.agda @@ -1,11 +1,9 @@ module Examples.OpSem where open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; subst) -open import Luau.Syntax using (var; nil; local_←_; _∙_; done; return) -open import Luau.Heap using (emp) +open import Luau.Syntax using (Block; var; nil; local_←_; _∙_; done; return; block_is_end) +open import Luau.Heap using (∅) -x = var "x" - -ex1 : emp ⊢ (local "x" ← nil ∙ return x ∙ done) ⟶ᴮ (return nil ∙ done) ⊣ emp +ex1 : ∅ ⊢ (local (var "x") ← nil ∙ return (var "x") ∙ done) ⟶ᴮ (return nil ∙ done) ⊣ ∅ ex1 = subst diff --git a/prototyping/Examples/Run.agda b/prototyping/Examples/Run.agda index bfa79839..88ca39b6 100644 --- a/prototyping/Examples/Run.agda +++ b/prototyping/Examples/Run.agda @@ -3,16 +3,13 @@ module Examples.Run where open import Agda.Builtin.Equality using (_≡_; refl) -open import Luau.Syntax using (nil; var; _$_; function_⟨_⟩_end; return; _∙_; done) +open import Luau.Syntax using (nil; var; _$_; function_is_end; return; _∙_; done; _⟨_⟩) open import Luau.Value using (nil) open import Luau.Run using (run; return) -open import Luau.Heap using (emp; lookup-next; next-emp; lookup-next-emp) +open import Luau.Heap using (lookup-next; next-emp; lookup-next-emp) import Agda.Builtin.Equality.Rewrite {-# REWRITE lookup-next next-emp lookup-next-emp #-} -x = var "x" -id = var "id" - -ex1 : (run (function "id" ⟨ "x" ⟩ return x ∙ done end ∙ return (id $ nil) ∙ done) ≡ return nil _) +ex1 : (run (function "id" ⟨ var "x" ⟩ is return (var "x") ∙ done end ∙ return (var "id" $ nil) ∙ done) ≡ return nil _) ex1 = refl diff --git a/prototyping/Examples/Syntax.agda b/prototyping/Examples/Syntax.agda index 8af9bca8..4ffcf4c5 100644 --- a/prototyping/Examples/Syntax.agda +++ b/prototyping/Examples/Syntax.agda @@ -2,21 +2,21 @@ module Examples.Syntax where open import Agda.Builtin.Equality using (_≡_; refl) open import FFI.Data.String using (_++_) -open import Luau.Syntax using (var; _$_; return; nil; function_⟨_⟩_end; done; _∙_) +open import Luau.Syntax using (var; _$_; return; nil; function_is_end; local_←_; done; _∙_; _⟨_⟩) open import Luau.Syntax.ToString using (exprToString; blockToString) -f = var "f" -x = var "x" - -ex1 : exprToString(f $ x) ≡ - "f(x)" +ex1 : exprToString(function "" ⟨ var "x" ⟩ is return (var "f" $ var "x") ∙ done end) ≡ + "function(x)\n" ++ + " return f(x)\n" ++ + "end" ex1 = refl -ex2 : blockToString(return nil ∙ done) ≡ - "return nil" +ex2 : blockToString(local var "x" ← nil ∙ return (var "x") ∙ done) ≡ + "local x = nil\n" ++ + "return x" ex2 = refl -ex3 : blockToString(function "f" ⟨ "x" ⟩ return x ∙ done end ∙ return f ∙ done) ≡ +ex3 : blockToString(function "f" ⟨ var "x" ⟩ is return (var "x") ∙ done end ∙ return (var "f") ∙ done) ≡ "local function f(x)\n" ++ " return x\n" ++ "end\n" ++ diff --git a/prototyping/Interpreter.agda b/prototyping/Interpreter.agda index 9184235e..3ec5b8d1 100644 --- a/prototyping/Interpreter.agda +++ b/prototyping/Interpreter.agda @@ -18,7 +18,7 @@ open import Luau.Run using (run; return; done; error) open import Luau.RuntimeError.ToString using (errToStringᴮ) open import Luau.Value.ToString using (valueToString) -runBlock : Block → IO ⊤ +runBlock : ∀ {a} → Block a → IO ⊤ runBlock block with run block runBlock block | return V D = putStrLn (valueToString V) runBlock block | done D = putStrLn "nil" diff --git a/prototyping/Luau/Heap.agda b/prototyping/Luau/Heap.agda index 1a0416e4..12f8dab0 100644 --- a/prototyping/Luau/Heap.agda +++ b/prototyping/Luau/Heap.agda @@ -5,39 +5,40 @@ open import FFI.Data.Maybe using (Maybe; just) open import FFI.Data.Vector using (Vector; length; snoc; empty) open import Luau.Addr using (Addr) open import Luau.Var using (Var) -open import Luau.Syntax using (Block; Expr; nil; addr; function⟨_⟩_end) +open import Luau.Syntax using (Block; Expr; Annotated; FunDec; nil; addr; function_is_end) -data HeapValue : Set where - function_⟨_⟩_end : Var → Var → Block → HeapValue +data HeapValue (a : Annotated) : Set where + function_is_end : FunDec a → Block a → HeapValue a -Heap = Vector HeapValue +Heap : Annotated → Set +Heap a = Vector (HeapValue a) -data _≡_⊕_↦_ : Heap → Heap → Addr → HeapValue → Set where +data _≡_⊕_↦_ {a} : Heap a → Heap a → Addr → HeapValue a → Set where defn : ∀ {H val} → ----------------------------------- (snoc H val) ≡ H ⊕ (length H) ↦ val -lookup : Heap → Addr → Maybe HeapValue -lookup = FFI.Data.Vector.lookup +_[_] : ∀ {a} → Heap a → Addr → Maybe (HeapValue a) +_[_] = FFI.Data.Vector.lookup -emp : Heap -emp = empty +∅ : ∀ {a} → Heap a +∅ = empty -data AllocResult (H : Heap) (V : HeapValue) : Set where - ok : ∀ a H′ → (H′ ≡ H ⊕ a ↦ V) → AllocResult H V +data AllocResult a (H : Heap a) (V : HeapValue a) : Set where + ok : ∀ b H′ → (H′ ≡ H ⊕ b ↦ V) → AllocResult a H V -alloc : ∀ H V → AllocResult H V +alloc : ∀ {a} H V → AllocResult a H V alloc H V = ok (length H) (snoc H V) defn -next : Heap → Addr +next : ∀ {a} → Heap a → Addr next = length -allocated : Heap → HeapValue → Heap +allocated : ∀ {a} → Heap a → HeapValue a → Heap a allocated = snoc --- next-emp : (length empty ≡ 0) +-- next-emp : (length ∅ ≡ 0) next-emp = FFI.Data.Vector.length-empty -- lookup-next : ∀ V H → (lookup (allocated H V) (next H) ≡ just V) diff --git a/prototyping/Luau/OpSem.agda b/prototyping/Luau/OpSem.agda index dcd474bf..878b8bd0 100644 --- a/prototyping/Luau/OpSem.agda +++ b/prototyping/Luau/OpSem.agda @@ -2,13 +2,13 @@ module Luau.OpSem where open import Agda.Builtin.Equality using (_≡_) open import FFI.Data.Maybe using (just) -open import Luau.Heap using (Heap; _≡_⊕_↦_; lookup; function_⟨_⟩_end) +open import Luau.Heap using (Heap; _≡_⊕_↦_; _[_]; function_is_end) open import Luau.Substitution using (_[_/_]ᴮ) -open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function⟨_⟩_end; _$_; block_is_end; local_←_; _∙_; done; function_⟨_⟩_end; return) +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; name; fun; arg) open import Luau.Value using (addr; val) -data _⊢_⟶ᴮ_⊣_ : Heap → Block → Block → Heap → Set -data _⊢_⟶ᴱ_⊣_ : Heap → Expr → Expr → Heap → Set +data _⊢_⟶ᴮ_⊣_ {a} : Heap a → Block a → Block a → Heap a → Set +data _⊢_⟶ᴱ_⊣_ {a} : Heap a → Expr a → Expr a → Heap a → Set data _⊢_⟶ᴱ_⊣_ where @@ -17,11 +17,11 @@ data _⊢_⟶ᴱ_⊣_ where ------------------- H ⊢ nil ⟶ᴱ nil ⊣ H - function : ∀ {H H′ a x B} → + function : ∀ {H H′ a F B} → - H′ ≡ H ⊕ a ↦ (function "anon" ⟨ x ⟩ B end) → + H′ ≡ H ⊕ a ↦ (function F is B end) → ------------------------------------------- - H ⊢ (function⟨ x ⟩ B end) ⟶ᴱ (addr a) ⊣ H′ + H ⊢ (function F is B end) ⟶ᴱ (addr a) ⊣ H′ app : ∀ {H H′ M M′ N} → @@ -29,11 +29,11 @@ data _⊢_⟶ᴱ_⊣_ where ----------------------------- H ⊢ (M $ N) ⟶ᴱ (M′ $ N) ⊣ H′ - beta : ∀ {H M a f x B} → + beta : ∀ {H M a F B} → - (lookup H a) ≡ just(function f ⟨ x ⟩ B end) → + H [ a ] ≡ just(function F is B end) → ----------------------------------------------------- - H ⊢ (addr a $ M) ⟶ᴱ (block f is local x ← M ∙ B end) ⊣ H + H ⊢ (addr a $ M) ⟶ᴱ (block (fun F) is local (arg F) ← M ∙ B end) ⊣ H block : ∀ {H H′ B B′ b} → @@ -61,14 +61,14 @@ data _⊢_⟶ᴮ_⊣_ where subst : ∀ {H x v B} → - ------------------------------------------------- - H ⊢ (local x ← val v ∙ B) ⟶ᴮ (B [ v / x ]ᴮ) ⊣ H + ------------------------------------------------------ + H ⊢ (local x ← val v ∙ B) ⟶ᴮ (B [ v / name x ]ᴮ) ⊣ H - function : ∀ {H H′ a f x B C} → + function : ∀ {H H′ a F B C} → - H′ ≡ H ⊕ a ↦ (function f ⟨ x ⟩ C end) → + H′ ≡ H ⊕ a ↦ (function F is C end) → -------------------------------------------------------------- - H ⊢ (function f ⟨ x ⟩ C end ∙ B) ⟶ᴮ (B [ addr a / f ]ᴮ) ⊣ H′ + H ⊢ (function F is C end ∙ B) ⟶ᴮ (B [ addr a / fun F ]ᴮ) ⊣ H′ return : ∀ {H H′ M M′ B} → @@ -76,7 +76,7 @@ data _⊢_⟶ᴮ_⊣_ where -------------------------------------------- H ⊢ (return M ∙ B) ⟶ᴮ (return M′ ∙ B) ⊣ H′ -data _⊢_⟶*_⊣_ : Heap → Block → Block → Heap → Set where +data _⊢_⟶*_⊣_ {a} : Heap a → Block a → Block a → Heap a → Set where refl : ∀ {H B} → diff --git a/prototyping/Luau/Run.agda b/prototyping/Luau/Run.agda index 29709ceb..d84f75de 100644 --- a/prototyping/Luau/Run.agda +++ b/prototyping/Luau/Run.agda @@ -1,20 +1,20 @@ module Luau.Run where open import Agda.Builtin.Equality using (_≡_; refl) -open import Luau.Heap using (Heap; emp) +open import Luau.Heap using (Heap; ∅) open import Luau.Syntax using (Block; return; _∙_; done) open import Luau.OpSem using (_⊢_⟶*_⊣_; refl; step) open import Luau.Value using (val) open import Properties.Step using (stepᴮ; step; return; done; error) open import Luau.RuntimeError using (RuntimeErrorᴮ) -data RunResult (H : Heap) (B : Block) : Set where +data RunResult {a} (H : Heap a) (B : Block a) : Set where return : ∀ V {B′ H′} → (H ⊢ B ⟶* (return (val V) ∙ B′) ⊣ H′) → RunResult H B done : ∀ {H′} → (H ⊢ B ⟶* done ⊣ H′) → RunResult H B error : ∀ {B′ H′} → (RuntimeErrorᴮ H′ B′) → (H ⊢ B ⟶* B′ ⊣ H′) → RunResult H B {-# TERMINATING #-} -run′ : ∀ H B → RunResult H B +run′ : ∀ {a} H B → RunResult {a} H B run′ H B with stepᴮ H B run′ H B | step H′ B′ D with run′ H′ B′ run′ H B | step H′ B′ D | return V D′ = return V (step D D′) @@ -24,5 +24,5 @@ run′ H _ | return V refl = return V refl run′ H _ | done refl = done refl run′ H B | error E = error E refl -run : ∀ B → RunResult emp B -run = run′ emp +run : ∀ {a} B → RunResult {a} ∅ B +run = run′ ∅ diff --git a/prototyping/Luau/RuntimeError.agda b/prototyping/Luau/RuntimeError.agda index 86e7cf21..e514dc9d 100644 --- a/prototyping/Luau/RuntimeError.agda +++ b/prototyping/Luau/RuntimeError.agda @@ -1,17 +1,17 @@ module Luau.RuntimeError where open import Agda.Builtin.Equality using (_≡_) -open import Luau.Heap using (Heap; lookup) +open import Luau.Heap using (Heap; _[_]) open import FFI.Data.Maybe using (just; nothing) -open import Luau.Syntax using (Block; Expr; nil; var; addr; function⟨_⟩_end; block_is_end; _$_; local_←_; function_⟨_⟩_end; return; done; _∙_) +open import Luau.Syntax using (Block; Expr; nil; var; addr; block_is_end; _$_; local_←_; return; done; _∙_) -data RuntimeErrorᴮ (H : Heap) : Block → Set -data RuntimeErrorᴱ (H : Heap) : Expr → Set +data RuntimeErrorᴮ {a} (H : Heap a) : Block a → Set +data RuntimeErrorᴱ {a} (H : Heap a) : Expr a → Set data RuntimeErrorᴱ H where NilIsNotAFunction : ∀ {M} → RuntimeErrorᴱ H (nil $ M) UnboundVariable : ∀ x → RuntimeErrorᴱ H (var x) - SEGV : ∀ a → (lookup H a ≡ nothing) → RuntimeErrorᴱ H (addr a) + SEGV : ∀ a → (H [ a ] ≡ nothing) → RuntimeErrorᴱ H (addr a) app : ∀ {M N} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (M $ N) block : ∀ b {B} → RuntimeErrorᴮ H B → RuntimeErrorᴱ H (block b is B end) diff --git a/prototyping/Luau/RuntimeError/ToString.agda b/prototyping/Luau/RuntimeError/ToString.agda index f11756f7..e8287157 100644 --- a/prototyping/Luau/RuntimeError/ToString.agda +++ b/prototyping/Luau/RuntimeError/ToString.agda @@ -4,9 +4,10 @@ open import FFI.Data.String using (String; _++_) open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; UnboundVariable; SEGV; app; block) open import Luau.Addr.ToString using (addrToString) open import Luau.Var.ToString using (varToString) +open import Luau.Syntax using (name) -errToStringᴱ : ∀ {H B} → RuntimeErrorᴱ H B → String -errToStringᴮ : ∀ {H B} → RuntimeErrorᴮ H B → String +errToStringᴱ : ∀ {a H B} → RuntimeErrorᴱ {a} H B → String +errToStringᴮ : ∀ {a H B} → RuntimeErrorᴮ {a} H B → String errToStringᴱ NilIsNotAFunction = "nil is not a function" errToStringᴱ (UnboundVariable x) = "variable " ++ varToString x ++ " is unbound" @@ -14,5 +15,5 @@ errToStringᴱ (SEGV a x) = "address " ++ addrToString a ++ " is unallocated" errToStringᴱ (app E) = errToStringᴱ E errToStringᴱ (block b E) = errToStringᴮ E ++ "\n in call of function " ++ varToString b -errToStringᴮ (local x E) = errToStringᴱ E ++ "\n in definition of " ++ varToString x +errToStringᴮ (local x E) = errToStringᴱ E ++ "\n in definition of " ++ varToString (name x) errToStringᴮ (return E) = errToStringᴱ E ++ "\n in return statement" diff --git a/prototyping/Luau/Substitution.agda b/prototyping/Luau/Substitution.agda index 1a361174..b956aeae 100644 --- a/prototyping/Luau/Substitution.agda +++ b/prototyping/Luau/Substitution.agda @@ -1,24 +1,24 @@ module Luau.Substitution where -open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function⟨_⟩_end; _$_; block_is_end; local_←_; _∙_; done; function_⟨_⟩_end; return) +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; _⟨_⟩ ; name; fun; arg) open import Luau.Value using (Value; val) open import Luau.Var using (Var; _≡ⱽ_) open import Properties.Dec using (Dec; yes; no) -_[_/_]ᴱ : Expr → Value → Var → Expr -_[_/_]ᴮ : Block → Value → Var → Block -var_[_/_]ᴱwhenever_ : ∀ {P} → Var → Value → Var → (Dec P) → Expr -_[_/_]ᴮunless_ : ∀ {P} → Block → Value → Var → (Dec P) → Block +_[_/_]ᴱ : ∀ {a} → Expr a → Value → Var → Expr a +_[_/_]ᴮ : ∀ {a} → Block a → Value → Var → Block a +var_[_/_]ᴱwhenever_ : ∀ {a P} → Var → Value → Var → (Dec P) → Expr a +_[_/_]ᴮunless_ : ∀ {a P} → Block a → Value → Var → (Dec P) → Block a nil [ v / x ]ᴱ = nil var y [ v / x ]ᴱ = var y [ v / x ]ᴱwhenever (x ≡ⱽ y) addr a [ v / x ]ᴱ = addr a (M $ N) [ v / x ]ᴱ = (M [ v / x ]ᴱ) $ (N [ v / x ]ᴱ) -function⟨ y ⟩ C end [ v / x ]ᴱ = function⟨ y ⟩ C [ v / x ]ᴮunless (x ≡ⱽ y) end +function F is C end [ v / x ]ᴱ = function F is C [ v / x ]ᴮunless (x ≡ⱽ name(arg F)) end block b is C end [ v / x ]ᴱ = block b is C [ v / x ]ᴮ end -(function f ⟨ y ⟩ C end ∙ B) [ v / x ]ᴮ = function f ⟨ y ⟩ (C [ v / x ]ᴮunless (x ≡ⱽ y)) end ∙ (B [ v / x ]ᴮunless (x ≡ⱽ f)) -(local y ← M ∙ B) [ v / x ]ᴮ = local y ← (M [ v / x ]ᴱ) ∙ (B [ v / x ]ᴮunless (x ≡ⱽ y)) +(function F is C end ∙ B) [ v / x ]ᴮ = function F is (C [ v / x ]ᴮunless (x ≡ⱽ name(arg F))) end ∙ (B [ v / x ]ᴮunless (x ≡ⱽ fun F)) +(local y ← M ∙ B) [ v / x ]ᴮ = local y ← (M [ v / x ]ᴱ) ∙ (B [ v / x ]ᴮunless (x ≡ⱽ name y)) (return M ∙ B) [ v / x ]ᴮ = return (M [ v / x ]ᴱ) ∙ (B [ v / x ]ᴮ) done [ v / x ]ᴮ = done diff --git a/prototyping/Luau/Syntax.agda b/prototyping/Luau/Syntax.agda index 3fe05bc8..04970b62 100644 --- a/prototyping/Luau/Syntax.agda +++ b/prototyping/Luau/Syntax.agda @@ -1,27 +1,55 @@ module Luau.Syntax where +open import Agda.Builtin.Equality using (_≡_) +open import Properties.Dec using (⊥) open import Luau.Var using (Var) open import Luau.Addr using (Addr) +open import Luau.Type using (Type) infixr 5 _∙_ -data Block : Set -data Stat : Set -data Expr : Set +data Annotated : Set where + maybe : Annotated + yes : Annotated -data Block where - _∙_ : Stat → Block → Block - done : Block +data VarDec : Annotated → Set where + var : Var → VarDec maybe + var_∈_ : ∀ {a} → Var → Type → VarDec a -data Stat where - function_⟨_⟩_end : Var → Var → Block → Stat - local_←_ : Var → Expr → Stat - return : Expr → Stat +name : ∀ {a} → VarDec a → Var +name (var x) = x +name (var x ∈ T) = x + +data FunDec : Annotated → Set where + _⟨_⟩∈_ : ∀ {a} → Var → VarDec a → Type → FunDec a + _⟨_⟩ : Var → VarDec maybe → FunDec maybe + +fun : ∀ {a} → FunDec a → Var +fun (f ⟨ x ⟩∈ T) = f +fun (f ⟨ x ⟩) = f + +arg : ∀ {a} → FunDec a → VarDec a +arg (f ⟨ x ⟩∈ T) = x +arg (f ⟨ x ⟩) = x + +data Block (a : Annotated) : Set +data Stat (a : Annotated) : Set +data Expr (a : Annotated) : Set + +data Block a where + _∙_ : Stat a → Block a → Block a + done : Block a + +data Stat a where + function_is_end : FunDec a → Block a → Stat a + local_←_ : VarDec a → Expr a → Stat a + return : Expr a → Stat a + +data Expr a where + nil : Expr a + var : Var → Expr a + addr : Addr → Expr a + _$_ : Expr a → Expr a → Expr a + function_is_end : FunDec a → Block a → Expr a + block_is_end : Var → Block a → Expr a -data Expr where - nil : Expr - var : Var → Expr - addr : Addr → Expr - _$_ : Expr → Expr → Expr - function⟨_⟩_end : Var → Block → Expr - block_is_end : Var → Block → Expr diff --git a/prototyping/Luau/Syntax/FromJSON.agda b/prototyping/Luau/Syntax/FromJSON.agda index 0a4164fb..b081d156 100644 --- a/prototyping/Luau/Syntax/FromJSON.agda +++ b/prototyping/Luau/Syntax/FromJSON.agda @@ -1,6 +1,6 @@ module Luau.Syntax.FromJSON where -open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; function⟨_⟩_end; local_←_; function_⟨_⟩_end; return; done; _∙_) +open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe) open import Agda.Builtin.List using (List; _∷_; []) @@ -31,12 +31,12 @@ lookupIn (key ∷ keys) obj with lookup (fromString key) obj lookupIn (key ∷ keys) obj | nothing = lookupIn keys obj lookupIn (key ∷ keys) obj | just value = (key , value) -exprFromJSON : Value → Either String Expr -exprFromObject : Object → Either String Expr -statFromJSON : Value → Either String Stat -statFromObject : Object → Either String Stat -blockFromJSON : Value → Either String Block -blockFromArray : Array → Either String Block +exprFromJSON : Value → Either String (Expr maybe) +exprFromObject : Object → Either String (Expr maybe) +statFromJSON : Value → Either String (Stat maybe) +statFromObject : Object → Either String (Stat maybe) +blockFromJSON : Value → Either String (Block maybe) +blockFromArray : Array → Either String (Block maybe) exprFromJSON (object obj) = exprFromObject obj exprFromJSON val = Left "AstExpr not an object" @@ -55,7 +55,7 @@ exprFromObject obj | just (string "AstExprCall") | _ | nothing = Left ("AstExpr exprFromObject obj | just (string "AstExprConstantNil") = Right nil exprFromObject obj | just (string "AstExprFunction") with lookup args obj | lookup body obj exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value with head arr | blockFromJSON value -exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | just (string x) | Right B = Right (function⟨ x ⟩ B end) +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | just (string x) | Right B = Right (function "" ⟨ var x ⟩ is B end) exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | just _ | Right B = Left "AstExprFunction args not a string array" exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | nothing | Right B = Left "Unsupported AstExprFunction empty args" exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | _ | Left err = Left err @@ -78,7 +78,7 @@ statFromObject obj with lookup type obj statFromObject obj | just(string "AstStatLocal") with lookup vars obj | lookup values obj statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) with head(arr1) | head(arr2) statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) with exprFromJSON(value) -statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) | Right M = Right (local x ← M) +statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) | Right M = Right (local (var x) ← M) statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) | Left err = Left err statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | nothing = Left "AstStatLocal empty values" statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(_) | _ = Left "AstStatLocal vars not a string array" @@ -89,7 +89,7 @@ statFromObject obj | just(string "AstStatLocal") | just(_) | nothing = Left "Ast statFromObject obj | just(string "AstStatLocal") | nothing | _ = Left "AstStatLocal missing vars" statFromObject obj | just(string "AstStatLocalFunction") with lookup name obj | lookup func obj statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value with exprFromJSON value -statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value | Right (function⟨ x ⟩ B end) = Right (function f ⟨ x ⟩ B end) +statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value | Right (function "" ⟨ x ⟩ is B end) = Right (function f ⟨ x ⟩ is B end) statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value | Left err = Left err statFromObject obj | just(string "AstStatLocalFunction") | just _ | just _ | Right _ = Left "AstStatLocalFunction func is not an AstExprFunction" statFromObject obj | just(string "AstStatLocalFunction") | just _ | just _ = Left "AstStatLocalFunction name is not a string" diff --git a/prototyping/Luau/Syntax/ToString.agda b/prototyping/Luau/Syntax/ToString.agda index afec0935..18d36175 100644 --- a/prototyping/Luau/Syntax/ToString.agda +++ b/prototyping/Luau/Syntax/ToString.agda @@ -1,13 +1,24 @@ module Luau.Syntax.ToString where -open import Luau.Syntax using (Block; Stat; Expr; nil; var; addr; _$_; function⟨_⟩_end; return; function_⟨_⟩_end ;local_←_; _∙_; done; block_is_end) +open import Luau.Syntax using (Block; Stat; Expr; VarDec; FunDec; nil; var; var_∈_; addr; _$_; function_is_end; return; local_←_; _∙_; done; block_is_end; _⟨_⟩; _⟨_⟩∈_) open import FFI.Data.String using (String; _++_) open import Luau.Addr.ToString using (addrToString) +open import Luau.Type.ToString using (typeToString) open import Luau.Var.ToString using (varToString) -exprToString′ : String → Expr → String -statToString′ : String → Stat → String -blockToString′ : String → Block → String +varDecToString : ∀ {a} → VarDec a → String +varDecToString (var x) = varToString x +varDecToString (var x ∈ T) = varToString x ++ " : " ++ typeToString T + +funDecToString : ∀ {a} → FunDec a → String +funDecToString ("" ⟨ x ⟩∈ T) = "function(" ++ varDecToString x ++ "): " ++ typeToString T +funDecToString ("" ⟨ x ⟩) = "function(" ++ varDecToString x ++ ")" +funDecToString (f ⟨ x ⟩∈ T) = "function " ++ varToString f ++ "(" ++ varDecToString x ++ "): " ++ typeToString T +funDecToString (f ⟨ x ⟩) = "function " ++ varToString f ++ "(" ++ varDecToString x ++ ")" + +exprToString′ : ∀ {a} → String → Expr a → String +statToString′ : ∀ {a} → String → Stat a → String +blockToString′ : ∀ {a} → String → Block a → String exprToString′ lb nil = "nil" @@ -17,21 +28,21 @@ exprToString′ lb (var x) = varToString(x) exprToString′ lb (M $ N) = (exprToString′ lb M) ++ "(" ++ (exprToString′ lb N) ++ ")" -exprToString′ lb (function⟨ x ⟩ B end) = - "function(" ++ x ++ ")" ++ lb ++ +exprToString′ lb (function F is B end) = + funDecToString F ++ lb ++ " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ "end" exprToString′ lb (block b is B end) = - "(function " ++ b ++ "()" ++ lb ++ + "(" ++ b ++ "()" ++ lb ++ " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ "end)()" -statToString′ lb (function f ⟨ x ⟩ B end) = - "local function " ++ f ++ "(" ++ x ++ ")" ++ lb ++ +statToString′ lb (function F is B end) = + "local " ++ funDecToString F ++ lb ++ " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ "end" statToString′ lb (local x ← M) = - "local " ++ x ++ " = " ++ (exprToString′ lb M) + "local " ++ varDecToString x ++ " = " ++ (exprToString′ lb M) statToString′ lb (return M) = "return " ++ (exprToString′ lb M) @@ -39,11 +50,11 @@ blockToString′ lb (S ∙ done) = statToString′ lb S blockToString′ lb (S ∙ B) = statToString′ lb S ++ lb ++ blockToString′ lb B blockToString′ lb (done) = "" -exprToString : Expr → String +exprToString : ∀ {a} → Expr a → String exprToString = exprToString′ "\n" -statToString : Stat → String +statToString : ∀ {a} → Stat a → String statToString = statToString′ "\n" -blockToString : Block → String +blockToString : ∀ {a} → Block a → String blockToString = blockToString′ "\n" diff --git a/prototyping/Luau/Value.agda b/prototyping/Luau/Value.agda index 855e663e..4768f859 100644 --- a/prototyping/Luau/Value.agda +++ b/prototyping/Luau/Value.agda @@ -1,14 +1,14 @@ module Luau.Value where open import Luau.Addr using (Addr) -open import Luau.Syntax using (Block; Expr; nil; addr; function⟨_⟩_end) +open import Luau.Syntax using (Block; Expr; nil; addr) open import Luau.Var using (Var) data Value : Set where nil : Value addr : Addr → Value -val : Value → Expr +val : ∀ {a} → Value → Expr a val nil = nil val (addr a) = addr a diff --git a/prototyping/PrettyPrinter.agda b/prototyping/PrettyPrinter.agda index dfb2c053..4a6c7dd6 100644 --- a/prototyping/PrettyPrinter.agda +++ b/prototyping/PrettyPrinter.agda @@ -15,7 +15,7 @@ open import Luau.Syntax using (Block) open import Luau.Syntax.FromJSON using (blockFromJSON) open import Luau.Syntax.ToString using (blockToString) -runBlock : Block → IO ⊤ +runBlock : ∀ {a} → Block a → IO ⊤ runBlock block = putStrLn (blockToString block) runJSON : Value → IO ⊤ diff --git a/prototyping/Properties/Step.agda b/prototyping/Properties/Step.agda index 0eddd9fe..eeda4ef2 100644 --- a/prototyping/Properties/Step.agda +++ b/prototyping/Properties/Step.agda @@ -2,16 +2,16 @@ module Properties.Step where open import Agda.Builtin.Equality using (_≡_; refl) open import FFI.Data.Maybe using (just; nothing) -open import Luau.Heap using (Heap; lookup; alloc; ok; function_⟨_⟩_end) -open import Luau.Syntax using (Block; Expr; nil; var; addr; function⟨_⟩_end; block_is_end; _$_; local_←_; function_⟨_⟩_end; return; done; _∙_) +open import Luau.Heap using (Heap; _[_]; alloc; ok; function_is_end) +open import Luau.Syntax using (Block; Expr; nil; var; addr; function_is_end; block_is_end; _$_; local_←_; return; done; _∙_; name; fun; arg) open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app ; beta; function; block; return; done; local; subst) open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; UnboundVariable; SEGV; app; block; local; return) open import Luau.Substitution using (_[_/_]ᴮ) open import Luau.Value using (nil; addr; val) open import Properties.Remember using (remember; _,_) -data StepResultᴮ (H : Heap) (B : Block) : Set -data StepResultᴱ (H : Heap) (M : Expr) : Set +data StepResultᴮ {a} (H : Heap a) (B : Block a) : Set +data StepResultᴱ {a} (H : Heap a) (M : Expr a) : Set data StepResultᴮ H B where step : ∀ H′ B′ → (H ⊢ B ⟶ᴮ B′ ⊣ H′) → StepResultᴮ H B @@ -24,8 +24,8 @@ data StepResultᴱ H M where value : ∀ V → (M ≡ val V) → StepResultᴱ H M error : (RuntimeErrorᴱ H M) → StepResultᴱ H M -stepᴱ : ∀ H M → StepResultᴱ H M -stepᴮ : ∀ H B → StepResultᴮ H B +stepᴱ : ∀ {a} H M → StepResultᴱ {a} H M +stepᴮ : ∀ {a} H B → StepResultᴮ {a} H B stepᴱ H nil = value nil refl stepᴱ H (var x) = error (UnboundVariable x) @@ -33,23 +33,23 @@ stepᴱ H (addr a) = value (addr a) refl stepᴱ H (M $ N) with stepᴱ H M stepᴱ H (M $ N) | step H′ M′ D = step H′ (M′ $ N) (app D) stepᴱ H (nil $ N) | value nil refl = error NilIsNotAFunction -stepᴱ H (addr a $ N) | value (addr a) refl with remember (lookup H a) +stepᴱ H (addr a $ N) | value (addr a) refl with remember (H [ a ]) stepᴱ H (addr a $ N) | value (addr a) refl | (nothing , p) = error (app (SEGV a p)) -stepᴱ H (addr a $ N) | value (addr a) refl | (just(function f ⟨ x ⟩ B end) , p) = step H (block f is local x ← N ∙ B end) (beta p) +stepᴱ H (addr a $ N) | value (addr a) refl | (just(function F is B end) , p) = step H (block fun F is (local arg F ← N) ∙ B end) (beta p) stepᴱ H (M $ N) | error E = error (app E) -stepᴱ H (function⟨ x ⟩ B end) with alloc H (function "anon" ⟨ x ⟩ B end) -stepᴱ H (function⟨ x ⟩ B end) | ok a H′ p = step H′ (addr a) (function p) stepᴱ H (block b is B end) with stepᴮ H B stepᴱ H (block b is B end) | step H′ B′ D = step H′ (block b is B′ end) (block D) stepᴱ H (block b is (return _ ∙ B′) end) | return V refl = step H (val V) return stepᴱ H (block b is done end) | done refl = step H nil done stepᴱ H (block b is B end) | error E = error (block b E) +stepᴱ H (function F is C end) with alloc H (function F is C end) +stepᴱ H function F is C end | ok a H′ p = step H′ (addr a) (function p) -stepᴮ H (function f ⟨ x ⟩ C end ∙ B) with alloc H (function f ⟨ x ⟩ C end) -stepᴮ H (function f ⟨ x ⟩ C end ∙ B) | ok a H′ p = step H′ (B [ addr a / f ]ᴮ) (function p) +stepᴮ H (function F is C end ∙ B) with alloc H (function F is C end) +stepᴮ H (function F is C end ∙ B) | ok a H′ p = step H′ (B [ addr a / fun F ]ᴮ) (function p) stepᴮ H (local x ← M ∙ B) with stepᴱ H M stepᴮ H (local x ← M ∙ B) | step H′ M′ D = step H′ (local x ← M′ ∙ B) (local D) -stepᴮ H (local x ← _ ∙ B) | value V refl = step H (B [ V / x ]ᴮ) subst +stepᴮ H (local x ← _ ∙ B) | value V refl = step H (B [ V / name x ]ᴮ) subst stepᴮ H (local x ← M ∙ B) | error E = error (local x E) stepᴮ H (return M ∙ B) with stepᴱ H M stepᴮ H (return M ∙ B) | step H′ M′ D = step H′ (return M′ ∙ B) (return D) From 7d679b317fea4db3bd979c682c1a3be1f1c092a5 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Mon, 14 Feb 2022 10:04:07 -0800 Subject: [PATCH 08/25] RFC: Generalized iteration (#335) Co-authored-by: dcope-rbx <91100513+dcope-rbx@users.noreply.github.com> Co-authored-by: vegorov-rbx <75688451+vegorov-rbx@users.noreply.github.com> --- rfcs/generalized-iteration.md | 122 ++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 rfcs/generalized-iteration.md diff --git a/rfcs/generalized-iteration.md b/rfcs/generalized-iteration.md new file mode 100644 index 00000000..72bdd69e --- /dev/null +++ b/rfcs/generalized-iteration.md @@ -0,0 +1,122 @@ +# Generalized iteration + +## Summary + +Introduce support for iterating over tables without using `pairs`/`ipairs` as well as a generic customization point for iteration via `__iter` metamethod. + +## Motivation + +Today there are many different ways to iterate through various containers that are syntactically incompatible. + +To iterate over arrays, you need to use `ipairs`: `for i, v in ipairs(t) do`. The traversal goes over a sequence `1..k` of numeric keys until `t[k] == nil`, preserving order. + +To iterate over dictionaries, you need to use `pairs`: `for k, v in pairs(t) do`. The traversal goes over all keys, numeric and otherwise, but doesn't guarantee an order; when iterating over arrays this may happen to work but is not guaranteed to work, as it depends on how keys are distributed between array and hash portion. + +To iterate over custom objects, whether they are represented as tables (user-specified) or userdata (host-specified), you need to expose special iteration methods, for example `for k, v in obj:Iterator() do`. + +All of these rely on the standard Lua iteration protocol, but it's impossible to trigger them in a generic fashion. Additionally, you *must* use one of `pairs`/`ipairs`/`next` to iterate over tables, which is easy to forget - a naive `for k, v in tab do` doesn't work and produces a hard-to-understand error `attempt to call a table value`. + +This proposal solves all of these by providing a way to implement uniform iteration with self-iterating objects by allowing to iterate over objects and tables directly via convenient `for k, v in obj do` syntax, and specifies the default iteration behavior for tables, thus mostly rendering `pairs`/`ipairs` obsolete - making Luau easier to use and teach. + +## Design + +In Lua, `for vars in iter do` has the following semantics (otherwise known as the iteration protocol): `iter` is expanded into three variables, `gen`, `state` and `index` (using `nil` if `iter` evaluates to fewer than 3 results); after this the loop is converted to the following pseudocode: + +```lua +while true do + vars... = gen(state, index) + index = vars... -- copy the first variable into the index + if index == nil then break end + + -- loop body goes here +end +``` + +This is a general mechanism that can support iteration through many containers, especially if `gen` is allowed to mutate state. Importantly, the *first* returned variable (which is exposed to the user) is used to continue the process on the next iteration - this can be limiting because it may require `gen` or `state` to carry extra internal iteration data for efficiency. To work around this for table iteration to avoid repeated calls to `next`, Luau compiler produces a special instruction sequence that recognizes `pairs`/`ipairs` iterators and stores the iteration index separately. + +Thus, today the loop `for k, v in tab do` effectively executes `k, v = tab()` on the first iteration, which is why it yields `attempt to call a table value`. If the object defines `__call` metamethod then it can act as a self-iterating method, but this is not idiomatic, not efficient and not pure/clean. + +This proposal comes in two parts: general support for `__iter` metamethod and default implementation for tables without one. With both of these in place, there's going to be a single, idiomatic, general and performant way to iterate through the object of any type: + +```lua +for k, v in obj do +... +end +``` + +### __iter + +To support self-iterating objects, we modify the iteration protocol as follows: instead of simply expanding the result of expression `iter` into three variables (`gen`, `state` and `index`), we check if the first result has an `__iter` metamethod (which can be the case if it's a table, userdata or another composite object (e.g. a record in the future). If it does, the metamethod is called with `gen` as the first argument, and the returned three values replace `gen`/`state`/`index`. This happens *before* the loop: + +```lua +if getmetatable(gen) and getmetatable(gen).__iter then + gen, state, index = getmetatable(gen).__iter(gen) +end +``` + +This check is comparatively trivial: usually `gen` is a function, and functions don't have metatables; as such we can simply check the type of `gen` and if it's a table/userdata, we can check if it has a metamethod `__iter`. Due to tag-method cache, this check is also very cheap if the metamethod is absent. + +This allows objects to provide a custom function that guides the iteration. Since the function is called once, it is easy to reuse other functions in the implementation, for example here's a node object that exposes iteration through its children: + +```lua +local Node = {} +Node.__index = Node + +function Node.new(children) + return setmetatable({ children = children }, Node) +end + +function Node:__iter() + return next, self.children +end +``` + +Luau compiler already emits a bytecode instruction, FORGPREP*, to perform initial loop setup - this is where we can evaluate `__iter` as well. + +Naturally, this means that if the table has `__iter` metamethod and you need to iterate through the table fields instead of using the provided metamethod, you can't rely on the general iteration scheme and need to use `pairs`. This is similar to other parts of the language, like `t[k]` vs `rawget(t, 'k')`, where the default behavior is overrideable but a library function can help peek behind the curtain. + +### Default table iteration + +If the argument is a table and it does not implement `__iter` metamethod, we treat this as an attempt to iterate through the table using the builtin iteration order. + +> Note: we also check if the table implements `__call`; if it does, we fall back to the default handling. We may be able to remove this check in the future, but we will need this initially to preserve backwards compatibility with custom table-driven iterator objects that implement `__call`. In either case, we will be able to collect detailed analytics about the use of `__call` in iteration, and if neither is present we can emit a specialized error message such as `object X is not iteratable`. + +To have a single, unified, iteration scheme over tables regardless of whether they are arrays or dictionaries, we establish the following semantics: + +- First, the traversal goes over numeric keys in range `1..k` up until reaching the first `k` such that `t[k] == nil` +- Then, the traversal goes over the remaining keys (with non-nil values), numeric and otherwise, in unspecified order. + +For arrays with gaps, this iterates until the first gap in order, and the remaining order is not specified. + +> Note: This behavior is similar to what `pairs` happens to provide today, but `pairs` doesn't give any guarantees, and it doesn't always provide this behavior in practice. + +To ensure that this traversal is performant, the actual implementation of the traversal involves going over the array part (in index order) and then over the hash part (in hash order). For that implementation to satisfy the criteria above, we need to make two additional changes to table insertion/rehash: + +- When inserting key `k` in the table when `k == t->sizearray + 1`, we force the table to rehash (resize its array portion). Today this is only performed if the hash portion is full, as such sometimes numeric keys can end up in the hash part. +- When rehashing the table, we ensure that the hash part doesn't contain the key `newsizearray + 1`. This requires checking if the table has this key, which may require an additional hash lookup but we only need to do this in rare cases based on the analysis of power-of-two key buckets that we already collect during rehash. + +These changes guarantee that the order observed via standard traversal with `next`/`pairs` matches the guarantee above, which is nice because it means we can minimize the complexity cost of this change by reusing the traversal code, including VM optimizations. They also mean that the array boundary (aka `#t`) can *always* be computed from just the array portion, which simplifies the table length computation and may slightly speed it up. + +## Drawbacks + +This makes `for` desugaring and implementation a little more complicated; it's not a large complexity factor in Luau because we already have special handling for `for` loops in the VM, but it's something to keep in mind. + +While the proposed iteration scheme should be a superset to both `pairs` and `ipairs` for tables, for arrays `ipairs` may in some cases be faster because it stops at the first `nil`, whereas the proposed new scheme (like `pairs`) needs to iterate through the rest of the table's array storage. This may be fixable in the future, if we replace our cached table length (`aboundary`) with Lua 5.4's `alimit`, which maintains the invariant that all values after `alimit` in the array are `nil`. This would make default table iteration maximally performant as well as help us accelerate GC in some cases, but will require extra checks during table assignments which is a cost we may not be willing to pay. Thus it is theoretically possible that we will end up with `ipairs` being a slightly faster equivalent for array iteration forever. + +The resulting iteration behavior, while powerful, increases the divergence between Luau and Lua, making more programs that are written for Luau not runnable in Lua. Luau language in general does not consider this type of compatibility essential, but this is noted for posterity. + +The changes in insertion behavior that facilitate single iteration order may have a small cost; that said, they are currently understood to belong to paths that are already slow and the added cost is minimal. + +The extra semantics will make inferring the types of the variables in a for loop more difficult - if we know the type of the expression that is being iterated through it probably is not a problem though. + +## Alternatives + +Other major designs have been considered. + +A minor variation of the proposal involves having `__iter` be called on every iteration instead of at loop startup, effectively having `__iter` work as an alternative to `__call`. The issue with this variant is that while it's a little simpler to specify and implement, it restricts the options when implementing custom iteratable objects, because it would be difficult for iteratable objects to store custom iteration state elsewhere since `__iter` method would effectively need to be pure, as it can't modify the object itself as more than one concurrent iteration needs to be supported. + +A major variation of the proposal involves instead supporting `__pairs` from Lua 5.2. The issue with this variant is that it still requires the use of a library method, `pairs`, to work, which doesn't make the language simpler as far as table iteration, which is the 95% case, is concerned. Additionally, with some rare exceptions metamethods today extend the *language* behavior, not the *library* behavior, and extending extra library functions with metamethods does not seem true to the core of the language. Finally, this only works if the user uses `pairs` to iterate and doesn't work with `ipairs`/`next`. + +Another variation involves using a new pseudo-keyword, `foreach`, instead of overloading existing `for`, and only using the new `__iter` semantics there. This can more cleanly separate behavior, requiring the object to have an `__iter` metamethod (or be a table) in `foreach` - which also avoids having to deal with `__call` - but it also requires teaching the users a new keyword which fragments the iteration space a little bit more. Compared to that, the main proposal doesn't introduce new divergent syntax, and merely tweaks existing behavior to be more general, thus making an existing construct easier to use. + +Finally, the author also considered and rejected extending the iteration protocol as part of this change. One problem with the current protocol is that the iterator requires an allocation (per loop execution) to keep extra state that isn't exposed to the user. The builtin iterators like `pairs`/`ipairs` work around this by feeding the user-visible index back to the search function, but that's not always practical. That said, having a different iteration protocol in effect only when `__iter` is used makes the language more complicated for unclear efficiency gains, thus this design doesn't suggest a new core protocol to favor simplicity. From 4b7e06e14f9968a26e105590fafddeebea04e109 Mon Sep 17 00:00:00 2001 From: Lily Brown Date: Mon, 14 Feb 2022 14:14:11 -0800 Subject: [PATCH 09/25] Emit more information for AstLocals in JSON encoder (#364) We don't emit type annotations right now for `AstLocal`s in the JSON encoder. This makes it really hard to surface annotations in the Agda implementation. This PR changes it to emit location and type annotations, if present. --- Analysis/src/JsonEncoder.cpp | 15 ++++++++- prototyping/Luau/Syntax/FromJSON.agda | 48 +++++++++++++++++---------- tests/JsonEncoder.test.cpp | 2 +- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/Analysis/src/JsonEncoder.cpp b/Analysis/src/JsonEncoder.cpp index 8dd597e1..0d48f2ca 100644 --- a/Analysis/src/JsonEncoder.cpp +++ b/Analysis/src/JsonEncoder.cpp @@ -150,6 +150,10 @@ struct AstJsonEncoder : public AstVisitor { writeRaw(std::to_string(i)); } + void write(std::nullptr_t) + { + writeRaw("null"); + } void write(std::string_view str) { writeString(str); @@ -177,7 +181,16 @@ struct AstJsonEncoder : public AstVisitor void write(AstLocal* local) { - write(local->name); + writeRaw("{"); + bool c = pushComma(); + if (local->annotation != nullptr) + write("type", local->annotation); + else + write("type", nullptr); + write("name", local->name); + write("location", local->location); + popComma(c); + writeRaw("}"); } void writeNode(AstNode* node) diff --git a/prototyping/Luau/Syntax/FromJSON.agda b/prototyping/Luau/Syntax/FromJSON.agda index b081d156..82f95193 100644 --- a/prototyping/Luau/Syntax/FromJSON.agda +++ b/prototyping/Luau/Syntax/FromJSON.agda @@ -1,6 +1,6 @@ module Luau.Syntax.FromJSON where -open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe) +open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec) open import Agda.Builtin.List using (List; _∷_; []) @@ -31,6 +31,8 @@ lookupIn (key ∷ keys) obj with lookup (fromString key) obj lookupIn (key ∷ keys) obj | nothing = lookupIn keys obj lookupIn (key ∷ keys) obj | just value = (key , value) +varDecFromJSON : Value → Either String (VarDec maybe) +varDecFromObject : Object → Either String (VarDec maybe) exprFromJSON : Value → Either String (Expr maybe) exprFromObject : Object → Either String (Expr maybe) statFromJSON : Value → Either String (Stat maybe) @@ -38,6 +40,14 @@ statFromObject : Object → Either String (Stat maybe) blockFromJSON : Value → Either String (Block maybe) blockFromArray : Array → Either String (Block maybe) +varDecFromJSON (object arg) = varDecFromObject arg +varDecFromJSON val = Left "VarDec not an object" + +varDecFromObject obj with lookup name obj +varDecFromObject obj | just (string name) = Right (var name) +varDecFromObject obj | just _ = Left "AstLocal name is not a string" +varDecFromObject obj | nothing = Left "AstLocal missing name" + exprFromJSON (object obj) = exprFromObject obj exprFromJSON val = Left "AstExpr not an object" @@ -54,17 +64,19 @@ exprFromObject obj | just (string "AstExprCall") | nothing | _ = Left ("AstExpr exprFromObject obj | just (string "AstExprCall") | _ | nothing = Left ("AstExprCall missing args") exprFromObject obj | just (string "AstExprConstantNil") = Right nil exprFromObject obj | just (string "AstExprFunction") with lookup args obj | lookup body obj -exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value with head arr | blockFromJSON value -exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | just (string x) | Right B = Right (function "" ⟨ var x ⟩ is B end) -exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | just _ | Right B = Left "AstExprFunction args not a string array" -exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | nothing | Right B = Left "Unsupported AstExprFunction empty args" -exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just value | _ | Left err = Left err +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just blockValue with head arr | blockFromJSON blockValue +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just blockValue | just argValue | Right B with varDecFromJSON argValue +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just blockValue | just argValue | Right B | Right arg = Right (function "" ⟨ arg ⟩ is B end) +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just blockValue | just argValue | Right B | Left err = Left err +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just blockValue | nothing | Right B = Left "Unsupported AstExprFunction empty args" +exprFromObject obj | just (string "AstExprFunction") | just (array arr) | just blockValue | _ | Left err = Left err exprFromObject obj | just (string "AstExprFunction") | just _ | just _ = Left "AstExprFunction args not an array" exprFromObject obj | just (string "AstExprFunction") | nothing | _ = Left "AstExprFunction missing args" exprFromObject obj | just (string "AstExprFunction") | _ | nothing = Left "AstExprFunction missing body" exprFromObject obj | just (string "AstExprLocal") with lookup lokal obj -exprFromObject obj | just (string "AstExprLocal") | just (string x) = Right (var x) -exprFromObject obj | just (string "AstExprLocal") | just (_) = Left "AstExprLocal local not a string" +exprFromObject obj | just (string "AstExprLocal") | just x with varDecFromJSON x +exprFromObject obj | just (string "AstExprLocal") | just x | Right x′ = Right (var (Luau.Syntax.name x′)) +exprFromObject obj | just (string "AstExprLocal") | just x | Left err = Left err exprFromObject obj | just (string "AstExprLocal") | nothing = Left "AstExprLocal missing local" exprFromObject obj | just (string ty) = Left ("TODO: Unsupported AstExpr " ++ ty) exprFromObject obj | just _ = Left "AstExpr type not a string" @@ -77,22 +89,22 @@ statFromJSON _ = Left "AstStat not an object" statFromObject obj with lookup type obj statFromObject obj | just(string "AstStatLocal") with lookup vars obj | lookup values obj statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) with head(arr1) | head(arr2) -statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) with exprFromJSON(value) -statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) | Right M = Right (local (var x) ← M) -statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | just(value) | Left err = Left err -statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(string x) | nothing = Left "AstStatLocal empty values" -statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(_) | _ = Left "AstStatLocal vars not a string array" +statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(x) | just(value) with varDecFromJSON(x) | exprFromJSON(value) +statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(x) | just(value) | Right x′ | Right M = Right (local x′ ← M) +statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(x) | just(value) | Left err | _ = Left err +statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(x) | just(value) | _ | Left err = Left err +statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | just(x) | nothing = Left "AstStatLocal empty values" statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(array arr2) | nothing | _ = Left "AstStatLocal empty vars" statFromObject obj | just(string "AstStatLocal") | just(array arr1) | just(_) = Left "AstStatLocal values not an array" statFromObject obj | just(string "AstStatLocal") | just(_) | just(_) = Left "AstStatLocal vars not an array" statFromObject obj | just(string "AstStatLocal") | just(_) | nothing = Left "AstStatLocal missing values" statFromObject obj | just(string "AstStatLocal") | nothing | _ = Left "AstStatLocal missing vars" statFromObject obj | just(string "AstStatLocalFunction") with lookup name obj | lookup func obj -statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value with exprFromJSON value -statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value | Right (function "" ⟨ x ⟩ is B end) = Right (function f ⟨ x ⟩ is B end) -statFromObject obj | just(string "AstStatLocalFunction") | just (string f) | just value | Left err = Left err -statFromObject obj | just(string "AstStatLocalFunction") | just _ | just _ | Right _ = Left "AstStatLocalFunction func is not an AstExprFunction" -statFromObject obj | just(string "AstStatLocalFunction") | just _ | just _ = Left "AstStatLocalFunction name is not a string" +statFromObject obj | just(string "AstStatLocalFunction") | just fnName | just value with varDecFromJSON fnName | exprFromJSON value +statFromObject obj | just(string "AstStatLocalFunction") | just fnName | just value | Right fnVar | Right (function "" ⟨ x ⟩ is B end) = Right (function (Luau.Syntax.name fnVar) ⟨ x ⟩ is B end) +statFromObject obj | just(string "AstStatLocalFunction") | just fnName | just value | Left err | _ = Left err +statFromObject obj | just(string "AstStatLocalFunction") | just fnName | just value | _ | Left err = Left err +statFromObject obj | just(string "AstStatLocalFunction") | just _ | just _ | Right _ | Right _ = Left "AstStatLocalFunction func is not an AstExprFunction" statFromObject obj | just(string "AstStatLocalFunction") | nothing | _ = Left "AstStatFunction missing name" statFromObject obj | just(string "AstStatLocalFunction") | _ | nothing = Left "AstStatFunction missing func" statFromObject obj | just(string "AstStatReturn") with lookup list obj diff --git a/tests/JsonEncoder.test.cpp b/tests/JsonEncoder.test.cpp index 4a717275..cb508072 100644 --- a/tests/JsonEncoder.test.cpp +++ b/tests/JsonEncoder.test.cpp @@ -46,7 +46,7 @@ TEST_CASE("encode_AstStatBlock") AstStatBlock block{Location(), bodyArray}; CHECK_EQ( - (R"({"type":"AstStatBlock","location":"0,0 - 0,0","body":[{"type":"AstStatLocal","location":"0,0 - 0,0","vars":["a_local"],"values":[]}]})"), + (R"({"type":"AstStatBlock","location":"0,0 - 0,0","body":[{"type":"AstStatLocal","location":"0,0 - 0,0","vars":[{"type":null,"name":"a_local","location":"0,0 - 0,0"}],"values":[]}]})"), toJson(&block)); } From f0c9d84461339fa8bcd2d8bf7d6f5be5be029f46 Mon Sep 17 00:00:00 2001 From: Lily Brown Date: Tue, 15 Feb 2022 14:10:43 -0800 Subject: [PATCH 10/25] Prototyping: Parse type annotations (#366) Parses type annotations from the JSON output of `luau-ast`. --- prototyping/Examples/SmokeTest.lua | 5 ++ prototyping/Examples/SmokeTestOutput.lua | 8 ++- prototyping/Luau/Syntax/FromJSON.agda | 15 ++++-- prototyping/Luau/Type/FromJSON.agda | 66 ++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 prototyping/Luau/Type/FromJSON.agda diff --git a/prototyping/Examples/SmokeTest.lua b/prototyping/Examples/SmokeTest.lua index e1e37e32..e663534a 100644 --- a/prototyping/Examples/SmokeTest.lua +++ b/prototyping/Examples/SmokeTest.lua @@ -10,4 +10,9 @@ local function comp(f) end local id2 = comp(id)(id) local nil2 = id2(nil) +local a : any = nil +local b : nil = nil +local c : (nil) -> nil = nil +local d : (any & nil) = nil +local e : any? = nil return id2(nil2) diff --git a/prototyping/Examples/SmokeTestOutput.lua b/prototyping/Examples/SmokeTestOutput.lua index b1b91e19..e663534a 100644 --- a/prototyping/Examples/SmokeTestOutput.lua +++ b/prototyping/Examples/SmokeTestOutput.lua @@ -8,5 +8,11 @@ local function comp(f) end end end -local id2 = id(id) +local id2 = comp(id)(id) local nil2 = id2(nil) +local a : any = nil +local b : nil = nil +local c : (nil) -> nil = nil +local d : (any & nil) = nil +local e : any? = nil +return id2(nil2) diff --git a/prototyping/Luau/Syntax/FromJSON.agda b/prototyping/Luau/Syntax/FromJSON.agda index 82f95193..8ae620bb 100644 --- a/prototyping/Luau/Syntax/FromJSON.agda +++ b/prototyping/Luau/Syntax/FromJSON.agda @@ -1,6 +1,7 @@ module Luau.Syntax.FromJSON where -open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec) +open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; var_∈_; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec) +open import Luau.Type.FromJSON using (typeFromJSON) open import Agda.Builtin.List using (List; _∷_; []) @@ -43,10 +44,14 @@ blockFromArray : Array → Either String (Block maybe) varDecFromJSON (object arg) = varDecFromObject arg varDecFromJSON val = Left "VarDec not an object" -varDecFromObject obj with lookup name obj -varDecFromObject obj | just (string name) = Right (var name) -varDecFromObject obj | just _ = Left "AstLocal name is not a string" -varDecFromObject obj | nothing = Left "AstLocal missing name" +varDecFromObject obj with lookup name obj | lookup type obj +varDecFromObject obj | just (string name) | nothing = Right (var name) +varDecFromObject obj | just (string name) | just Value.null = Right (var name) +varDecFromObject obj | just (string name) | just tyValue with typeFromJSON tyValue +varDecFromObject obj | just (string name) | just tyValue | Right ty = Right (var name ∈ ty) +varDecFromObject obj | just (string name) | just tyValue | Left err = Left err +varDecFromObject obj | just _ | _ = Left "AstLocal name is not a string" +varDecFromObject obj | nothing | _ = Left "AstLocal missing name" exprFromJSON (object obj) = exprFromObject obj exprFromJSON val = Left "AstExpr not an object" diff --git a/prototyping/Luau/Type/FromJSON.agda b/prototyping/Luau/Type/FromJSON.agda new file mode 100644 index 00000000..45bda5f3 --- /dev/null +++ b/prototyping/Luau/Type/FromJSON.agda @@ -0,0 +1,66 @@ +module Luau.Type.FromJSON where + +open import Luau.Type using (Type; nil; _⇒_; _∪_; _∩_; any) + +open import Agda.Builtin.List using (List; _∷_; []) + +open import FFI.Data.Aeson using (Value; Array; Object; object; array; string; fromString; lookup) +open import FFI.Data.Bool using (true; false) +open import FFI.Data.Either using (Either; Left; Right) +open import FFI.Data.Maybe using (Maybe; nothing; just) +open import FFI.Data.String using (String; _++_) +open import FFI.Data.Vector using (head; tail; null; empty) + +name = fromString "name" +type = fromString "type" +argTypes = fromString "argTypes" +returnTypes = fromString "returnTypes" +types = fromString "types" + +{-# TERMINATING #-} +typeFromJSON : Value → Either String Type +compoundFromArray : (Type → Type → Type) → Array → Either String Type + +typeFromJSON (object o) with lookup type o +typeFromJSON (object o) | just (string "AstTypeFunction") with lookup argTypes o | lookup returnTypes o +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) with lookup types argsSet | lookup types retsSet +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) with head args | head rets +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) | just argValue | just retValue with typeFromJSON argValue | typeFromJSON retValue +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) | just argValue | just retValue | Right arg | Right ret = Right (arg ⇒ ret) +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) | just argValue | just retValue | Left err | _ = Left err +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) | just argValue | just retValue | _ | Left err = Left err +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) | _ | nothing = Left "No return type" +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just (array args) | just (array rets) | nothing | _ = Left "No argument type" +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | just _ | _ = Left "argTypes.types is not an array" +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | _ | just _ = Left "retTypes.types is not an array" +typeFromJSON (object o) | just (string "AstTypeFunction") | just (object argsSet) | just (object retsSet) | nothing | _ = Left "argTypes.types does not exist" +typeFromJSON (object o) | just (string "AstTypeFunction") | _ | just _ = Left "argTypes is not an object" +typeFromJSON (object o) | just (string "AstTypeFunction") | just _ | _ = Left "returnTypes is not an object" +typeFromJSON (object o) | just (string "AstTypeFunction") | nothing | nothing = Left "Missing argTypes and returnTypes" + +typeFromJSON (object o) | just (string "AstTypeReference") with lookup name o +typeFromJSON (object o) | just (string "AstTypeReference") | just (string "nil") = Right nil +typeFromJSON (object o) | just (string "AstTypeReference") | just (string "any") = Right any +typeFromJSON (object o) | just (string "AstTypeReference") | _ = Left "Unknown referenced type" + +typeFromJSON (object o) | just (string "AstTypeUnion") with lookup types o +typeFromJSON (object o) | just (string "AstTypeUnion") | just (array types) = compoundFromArray _∪_ types +typeFromJSON (object o) | just (string "AstTypeUnion") | _ = Left "`types` field must be an array" + +typeFromJSON (object o) | just (string "AstTypeIntersection") with lookup types o +typeFromJSON (object o) | just (string "AstTypeIntersection") | just (array types) = compoundFromArray _∩_ types +typeFromJSON (object o) | just (string "AstTypeIntersection") | _ = Left "`types` field must be an array" + +typeFromJSON (object o) | just (string ty) = Left ("Unsupported type " ++ ty) +typeFromJSON (object o) | just _ = Left "`type` field must be a string" +typeFromJSON (object o) | nothing = Left "No `type` field" +typeFromJSON _ = Left "Unsupported JSON type" + +compoundFromArray ctor ts with head ts | tail ts +compoundFromArray ctor ts | just hd | tl with null tl +compoundFromArray ctor ts | just hd | tl | true = typeFromJSON hd +compoundFromArray ctor ts | just hd | tl | false with typeFromJSON hd | compoundFromArray ctor tl +compoundFromArray ctor ts | just hd | tl | false | Right hdTy | Right tlTy = Right (ctor hdTy tlTy) +compoundFromArray ctor ts | just hd | tl | false | Left err | _ = Left err +compoundFromArray ctor ts | just hd | tl | false | _ | Left Err = Left Err +compoundFromArray ctor ts | nothing | empty = Left "Empty types array?" From c8d6dc2758ef6369010eb7e23e368d478790aa94 Mon Sep 17 00:00:00 2001 From: Lily Brown Date: Tue, 15 Feb 2022 14:24:51 -0800 Subject: [PATCH 11/25] Revise GHA workflows for prototyping (#367) Changed the GHA workflows to: - Not run `build` and `release` workflows for PRs that only affect `prototyping/` - Run `prototyping` workflow when PRs affect `Analysis/**`, `Ast/**`, or the `luau-ast` source files --- .github/workflows/build.yml | 2 ++ .github/workflows/prototyping.yml | 10 ++++++++-- .github/workflows/release.yml | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b797e91b..93b92645 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,12 +9,14 @@ on: - 'papers/**' - 'rfcs/**' - '*.md' + - 'prototyping/**' pull_request: paths-ignore: - 'docs/**' - 'papers/**' - 'rfcs/**' - '*.md' + - 'prototyping/**' jobs: unix: diff --git a/.github/workflows/prototyping.yml b/.github/workflows/prototyping.yml index 76c1e3c1..4a1b8e08 100644 --- a/.github/workflows/prototyping.yml +++ b/.github/workflows/prototyping.yml @@ -8,12 +8,18 @@ on: paths: - '.github/workflows/**' - 'prototyping/**' - - 'Analysis/src/JsonEncoder.cpp' + - 'Analysis/**' + - 'Ast/**' + - 'CLI/Ast.cpp' + - 'CLI/FileUtils.*' pull_request: paths: - '.github/workflows/**' - 'prototyping/**' - - 'Analysis/src/JsonEncoder.cpp' + - 'Analysis/**' + - 'Ast/**' + - 'CLI/Ast.cpp' + - 'CLI/FileUtils.*' jobs: linux: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5cd952b..2bf796a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ on: - 'papers/**' - 'rfcs/**' - '*.md' + - 'prototyping/**' jobs: build: From e541e19f440b4eaefceb122c65b66103460973f9 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Tue, 15 Feb 2022 18:37:02 -0800 Subject: [PATCH 12/25] Create RFC status tracking document (#363) This tracks status of all unimplemented RFCs in one central place. Hopefully we won't forget to update this when new RFCs are added! --- rfcs/STATUS.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 rfcs/STATUS.md diff --git a/rfcs/STATUS.md b/rfcs/STATUS.md new file mode 100644 index 00000000..9a45afe0 --- /dev/null +++ b/rfcs/STATUS.md @@ -0,0 +1,64 @@ +This document tracks unimplemented RFCs. + +## Deprecate getfenv/setfenv + +[RFC: Deprecate getfenv/setfenv](https://github.com/Roblox/luau/blob/master/rfcs/deprecate-getfenv-setfenv.md) + +**Status**: Needs implementation. + +**Notes**: Implementing this RFC triggers warnings across the board in the apps ecosystem, in particular in testing libraries. Pending code changes / decisions. + +## Read-only and write-only properties + +[RFC: Read-only properties](https://github.com/Roblox/luau/blob/master/rfcs/property-readonly.md) | +[RFC: Write-only properties](https://github.com/Roblox/luau/blob/master/rfcs/property-writeonly.md) + +**Status**: Needs implementation + +## Sealed/unsealed typing changes + +[RFC: Sealed table subtyping](https://github.com/Roblox/luau/blob/master/rfcs/sealed-table-subtyping.md) | +[RFC: Unsealed table literals](https://github.com/Roblox/luau/blob/master/rfcs/unsealed-table-literals.md) | +[RFC: Only strip optional properties from unsealed tables during subtyping](https://github.com/Roblox/luau/blob/master/rfcs/unsealed-table-subtyping-strips-optional-properties.md) + +**Status**: Implemented but depends on new transaction log implementation that is not fully live yet. + +## Default type parameters + +[RFC: Default type alias type parameters](https://github.com/Roblox/luau/blob/master/rfcs/syntax-default-type-alias-type-parameters.md) + +**Status**: Implemented but not fully rolled out yet. + +## Singleton types + +[RFC: Singleton types](https://github.com/Roblox/luau/blob/master/rfcs/syntax-singleton-types.md) + +**Status**: Implemented but not fully rolled out yet. + +## Nil-forgiving operator + +[RFC: nil-forgiving postfix operator !](https://github.com/Roblox/luau/blob/master/rfcs/syntax-nil-forgiving-operator.md) + +**Status**: Needs implementation. + +**Notes**: Do we need to reevaluate the necessity given `assert` and improved refinements? + +## Safe navigation operator + +[RFC: Safe navigation postfix operator (?)](https://github.com/Roblox/luau/blob/master/rfcs/syntax-safe-navigation-operator.md) + +**Status**: Needs implementation. + +**Notes**: We have unresolved issues with interaction between this feature and Roblox instance hierarchy. This may affect the viability of this proposal. + +## String interpolation + +[RFC: String interpolation](https://github.com/Roblox/luau/blob/master/rfcs/syntax-string-interpolation.md) + +**Status**: Needs implementation + +## Generalized iteration + +[RFC: Generalized iteration](https://github.com/Roblox/luau/blob/master/rfcs/generalized-iteration.md) + +**Status**: Needs implementation From e49a0fd4cd785462ca5b8b2b4a29d0684f50e2ac Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Thu, 17 Feb 2022 16:14:35 -0800 Subject: [PATCH 13/25] Mark default type parameters RFC as implemented (#369) --- rfcs/STATUS.md | 6 ------ rfcs/syntax-default-type-alias-type-parameters.md | 2 ++ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/rfcs/STATUS.md b/rfcs/STATUS.md index 9a45afe0..72db2003 100644 --- a/rfcs/STATUS.md +++ b/rfcs/STATUS.md @@ -23,12 +23,6 @@ This document tracks unimplemented RFCs. **Status**: Implemented but depends on new transaction log implementation that is not fully live yet. -## Default type parameters - -[RFC: Default type alias type parameters](https://github.com/Roblox/luau/blob/master/rfcs/syntax-default-type-alias-type-parameters.md) - -**Status**: Implemented but not fully rolled out yet. - ## Singleton types [RFC: Singleton types](https://github.com/Roblox/luau/blob/master/rfcs/syntax-singleton-types.md) diff --git a/rfcs/syntax-default-type-alias-type-parameters.md b/rfcs/syntax-default-type-alias-type-parameters.md index 15d23c81..443bbac3 100644 --- a/rfcs/syntax-default-type-alias-type-parameters.md +++ b/rfcs/syntax-default-type-alias-type-parameters.md @@ -1,5 +1,7 @@ # Default type alias type parameters +**Status**: Implemented + ## Summary Introduce syntax to provide default type values inside the type alias type parameter list. From 731e197757ead8908e558cfc63f9d518f53940d5 Mon Sep 17 00:00:00 2001 From: Alexander McCord <11488393+alexmccord@users.noreply.github.com> Date: Thu, 17 Feb 2022 16:16:31 -0800 Subject: [PATCH 14/25] Mark singleton types RFC as implemented (#370) --- rfcs/STATUS.md | 6 ------ rfcs/syntax-singleton-types.md | 2 ++ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/rfcs/STATUS.md b/rfcs/STATUS.md index 72db2003..742cb501 100644 --- a/rfcs/STATUS.md +++ b/rfcs/STATUS.md @@ -23,12 +23,6 @@ This document tracks unimplemented RFCs. **Status**: Implemented but depends on new transaction log implementation that is not fully live yet. -## Singleton types - -[RFC: Singleton types](https://github.com/Roblox/luau/blob/master/rfcs/syntax-singleton-types.md) - -**Status**: Implemented but not fully rolled out yet. - ## Nil-forgiving operator [RFC: nil-forgiving postfix operator !](https://github.com/Roblox/luau/blob/master/rfcs/syntax-nil-forgiving-operator.md) diff --git a/rfcs/syntax-singleton-types.md b/rfcs/syntax-singleton-types.md index 26ea3028..2c1f5442 100644 --- a/rfcs/syntax-singleton-types.md +++ b/rfcs/syntax-singleton-types.md @@ -2,6 +2,8 @@ > Note: this RFC was adapted from an internal proposal that predates RFC process +**Status**: Implemented + ## Summary Introduce a new kind of type variable, called singleton types. They are just like normal types but has the capability to represent a constant runtime value as a type. From 6aaaafcc8d3e515514ff1a72621da500f0c83404 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Thu, 17 Feb 2022 16:58:43 -0800 Subject: [PATCH 15/25] Add documentation for upcoming CommentDirective lint (#361) --- docs/_pages/lint.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/_pages/lint.md b/docs/_pages/lint.md index de4d247e..340d35a2 100644 --- a/docs/_pages/lint.md +++ b/docs/_pages/lint.md @@ -317,3 +317,13 @@ The code above can be rewritten as follows to avoid the warning and the associat ```lua local x = if flag then false else true ``` + +## CommentDirective (26) + +Luau uses comments that start from `!` to control certain aspects of analysis, for example setting type checking mode via `--!strict` or disabling individual lints with `--!nolint`. Unknown directives are ignored, for example `--!nostrict` doesn't have any effect on the type checking process as the correct spelling is `--!nonstrict`. This warning flags comment directives that are ignored during processing: + +```lua +--!nostrict +-- Unknown comment directive 'nostrict'; did you mean 'nonstrict'?" +``` +``` From 1ac64af48416a6ae801673c315cd7bc999958308 Mon Sep 17 00:00:00 2001 From: Lily Brown Date: Thu, 17 Feb 2022 17:15:33 -0800 Subject: [PATCH 16/25] Prototyping: Revise CI (#371) Introduces a test runner with test cases. Also significantly overhauls the GHA configuration. --- .github/workflows/prototyping.yml | 40 ++-- prototyping/.gitignore | 6 + prototyping/README.md | 18 ++ .../Tests/Interpreter/return_nil/in.lua | 5 + .../Tests/Interpreter/return_nil/out.txt | 1 + .../PrettyPrinter/smoke_test/in.lua} | 12 +- .../PrettyPrinter/smoke_test/out.txt} | 0 prototyping/tests | 197 ++++++++++++++++++ 8 files changed, 260 insertions(+), 19 deletions(-) create mode 100644 prototyping/Tests/Interpreter/return_nil/in.lua create mode 100644 prototyping/Tests/Interpreter/return_nil/out.txt rename prototyping/{Examples/SmokeTestOutput.lua => Tests/PrettyPrinter/smoke_test/in.lua} (72%) rename prototyping/{Examples/SmokeTest.lua => Tests/PrettyPrinter/smoke_test/out.txt} (100%) create mode 100755 prototyping/tests diff --git a/.github/workflows/prototyping.yml b/.github/workflows/prototyping.yml index 4a1b8e08..637ed840 100644 --- a/.github/workflows/prototyping.yml +++ b/.github/workflows/prototyping.yml @@ -34,6 +34,11 @@ jobs: with: path: ~/.cabal/store key: prototyping-${{ runner.os }}-${{ matrix.agda }} + - uses: actions/cache@v2 + id: luau-ast-cache + with: + path: ./build + key: prototyping-${{ runner.os }}-${{ hashFiles('Ast/**', 'Analysis/**', 'CLI/Ast.cpp', 'CLI/FileUtils.*')}} - name: install cabal run: sudo apt-get install -y cabal-install - name: cabal update @@ -43,26 +48,35 @@ jobs: working-directory: prototyping run: | cabal install Agda-${{ matrix.agda }} - cabal install --lib scientific --package-env . - cabal install --lib vector --package-env . - cabal install --lib aeson --package-env . - - name: check examples + cabal install --lib scientific vector aeson --package-env . + - name: check targets working-directory: prototyping - run: ~/.cabal/bin/agda Examples.agda + run: | + ~/.cabal/bin/agda Examples.agda + ~/.cabal/bin/agda Properties.agda - name: build executables working-directory: prototyping run: | ~/.cabal/bin/agda --compile PrettyPrinter.agda ~/.cabal/bin/agda --compile Interpreter.agda - name: cmake configure - run: cmake . + if: steps.luau-ast-cache.outputs.cache-hit != 'true' + run: | + mkdir -p build + cd build + cmake build ../ - name: cmake build luau-ast - run: cmake --build . --target Luau.Ast.CLI - - name: run smoketest + if: steps.luau-ast-cache.outputs.cache-hit != 'true' + run: | + cmake --build ./build --target Luau.Ast.CLI + - name: run tests working-directory: prototyping run: | - ../luau-ast Examples/SmokeTest.lua | ./PrettyPrinter > Examples/SmokeTestOutput.lua - ../luau-ast Examples/SmokeTest.lua | ./Interpreter - - name: diff smoketest - working-directory: prototyping - run: diff Examples/SmokeTest.lua Examples/SmokeTestOutput.lua + mkdir test-failures + python tests -l ../build/luau-ast --write-diff-failures --diff-failure-location test-failures/ + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: test failures + path: prototyping/test-failures + retention-days: 5 diff --git a/prototyping/.gitignore b/prototyping/.gitignore index cca4f464..10edac52 100644 --- a/prototyping/.gitignore +++ b/prototyping/.gitignore @@ -2,5 +2,11 @@ *.agdai Main MAlonzo +Examples PrettyPrinter +Interpreter +Properties +!Tests/Interpreter +!Tests/PrettyPrinter .ghc.* +test-failures/ diff --git a/prototyping/README.md b/prototyping/README.md index f1761d1c..965c9c49 100644 --- a/prototyping/README.md +++ b/prototyping/README.md @@ -25,3 +25,21 @@ and run! ``` luau-ast Examples/SmokeTest.lua | ./PrettyPrinter ``` + +## Testing + +We have a series of snapshot tests in the `Tests/` directory. You interact with the tests using the `tests` Python script in the `prototyping` directory. To simply run the tests, run: + +```sh +tests --luau-cli ../build/luau-ast --build +``` + +This will build the test targets and run them. Run `tests --help` for information about all the command-line options. + +### Adding a new test + +To add a new test, add it to `Tests/{SUITE_NAME}/{CASE_NAME}`. You'll need an `in.lua` file and an `out.txt` file. The `in.lua` file is the input Luau source code, while the `out.txt` file is the expected output after running `luau-ast in.lua | test_executable`. + +### Updating a test + +If you make a change to the prototype that results in an expected change in behavior, you might want to update the test cases automatically. To do this, run `tests` with the `--accept-new-output` (`-a` for short) flag. Rather than diffing the output, this will overwrite the `out.txt` files for each test case with the actual result. Commit the resulting changes with your PR. diff --git a/prototyping/Tests/Interpreter/return_nil/in.lua b/prototyping/Tests/Interpreter/return_nil/in.lua new file mode 100644 index 00000000..4e6ce2db --- /dev/null +++ b/prototyping/Tests/Interpreter/return_nil/in.lua @@ -0,0 +1,5 @@ +local function foo(x) + return nil +end + +return foo(nil) diff --git a/prototyping/Tests/Interpreter/return_nil/out.txt b/prototyping/Tests/Interpreter/return_nil/out.txt new file mode 100644 index 00000000..607602cf --- /dev/null +++ b/prototyping/Tests/Interpreter/return_nil/out.txt @@ -0,0 +1 @@ +nil diff --git a/prototyping/Examples/SmokeTestOutput.lua b/prototyping/Tests/PrettyPrinter/smoke_test/in.lua similarity index 72% rename from prototyping/Examples/SmokeTestOutput.lua rename to prototyping/Tests/PrettyPrinter/smoke_test/in.lua index e663534a..0d23bbb2 100644 --- a/prototyping/Examples/SmokeTestOutput.lua +++ b/prototyping/Tests/PrettyPrinter/smoke_test/in.lua @@ -1,12 +1,12 @@ local function id(x) - return x + return x end local function comp(f) - return function(g) - return function(x) - return f(g(x)) - end - end + return function(g) + return function(x) + return f(g(x)) + end + end end local id2 = comp(id)(id) local nil2 = id2(nil) diff --git a/prototyping/Examples/SmokeTest.lua b/prototyping/Tests/PrettyPrinter/smoke_test/out.txt similarity index 100% rename from prototyping/Examples/SmokeTest.lua rename to prototyping/Tests/PrettyPrinter/smoke_test/out.txt diff --git a/prototyping/tests b/prototyping/tests new file mode 100755 index 00000000..89e105d0 --- /dev/null +++ b/prototyping/tests @@ -0,0 +1,197 @@ +#!/bin/python + +import argparse +import difflib +import enum +import os +import os.path +import subprocess +import sys + +SUITES = ["interpreter", "prettyprinter"] +IN_FILE_NAME = "in.lua" +OUT_FILE_NAME = "out.txt" +SUITE_EXE_NAMES = { + "interpreter": "Interpreter", + "prettyprinter": "PrettyPrinter", +} + +SUITE_ENTRY_POINTS = { + "interpreter": "Interpreter.agda", + "prettyprinter": "PrettyPrinter.agda", +} + +SUITE_ROOTS = { + "interpreter": "Tests/Interpreter", + "prettyprinter": "Tests/PrettyPrinter", +} + +class TestResultStatus(enum.Enum): + CLI_ERROR = 0 + EXE_ERROR = 1 + DIFF_ERROR = 2 + SUCCESS = 3 + WROTE_NEW = 4 + +class DiffFailure: + def __init__(self, expected, actual): + self.expected = expected + self.actual = actual + + def diff_text(self): + diff_generator = difflib.context_diff(self.expected.splitlines(), self.actual.splitlines(), fromfile="expected", tofile="actual", n=3) + return "".join(diff_generator) + + def diff_html(self): + differ = difflib.HtmlDiff(tabsize=4) + return differ.make_file(self.expected.splitlines(), self.actual.splitlines(), fromdesc="Expected", todesc="Actual", context=True, numlines=5) + +class TestCaseResult: + def __init__(self, suite, case, status, details): + self.suite = suite + self.case = case + self.status = status + self.details = details + + def did_pass(self): + return self.status == TestResultStatus.SUCCESS or self.status == TestResultStatus.WROTE_NEW + + def to_string(self): + prefix = f"[{self.suite}/{self.case}]: " + if self.status == TestResultStatus.CLI_ERROR: + return f"{prefix}CLI ERROR: {self.details}" + elif self.status == TestResultStatus.EXE_ERROR: + return f"{prefix}EXE ERROR: {self.details}" + elif self.status == TestResultStatus.DIFF_ERROR: + text_diff = self.details.diff_text() + return f"{prefix}FAILED:\n{text_diff}" + elif self.status == TestResultStatus.SUCCESS: + return f"{prefix}SUCCEEDED" + elif self.status == TestResultStatus.WROTE_NEW: + return f"{prefix}WROTE NEW RESULT" + + def write_artifact(self, artifact_root): + if self.status != TestResultStatus.DIFF_ERROR: + return + + filename = f"{self.suite}-{self.case}.out.html" + path = os.path.join(artifact_root, filename) + html = self.details.diff_html() + with open(path, "w") as file: + file.write(html) + +parser = argparse.ArgumentParser(description="Runs prototype test cases") +parser.add_argument("--luau-cli", "-l", dest="cli_location", required=True, help="The location of luau-cli") +parser.add_argument("--root", "-r", dest="prototype_root", required=False, default=os.getcwd(), help="The root of the prototype") +parser.add_argument("--build", "-b", dest="build", action="store_true", default=True, help="Whether to automatically build required test binaries") +parser.add_argument("--suite", "-s", dest="suites", action="append", default=[], choices=SUITES, help="Which test suites to run") +parser.add_argument("--case", "-c", dest="cases", action="append", default=[], help="Which test cases to run") +parser.add_argument("--accept-new-output", "-a", dest="snapshot", action="store_true", default=False, help="Whether to write the new output to files, instead of diffing against it") +parser.add_argument("--write-diff-failures", dest="write_diffs", action="store_true", default=False, help="Whether to write test failure diffs to files") +parser.add_argument("--diff-failure-location", dest="diff_location", default=None, help="Where to write diff failure files to") + +def build_suite(root, suite): + entry_point = SUITE_ENTRY_POINTS.get(suite) + if entry_point is None: + return (False, "Invalid suite") + + result = subprocess.run(["~/.cabal/bin/agda", "--compile", entry_point], shell=True, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode == 0: + return (True, None) + else: + return (False, result.stdout) + +def run_test(in_path, out_path, cli_path, exe_path, snapshot): + cli_result = subprocess.run([cli_path, in_path], capture_output=True) + if cli_result.returncode != 0: + return (TestResultStatus.CLI_ERROR, f"CLI error: {cli_result.stderr}") + + exe_result = subprocess.run(exe_path, input=cli_result.stdout, capture_output=True) + if exe_result.returncode != 0: + return (TestResultStatus.EXE_ERROR, f"Executable error; stdout:{exe_result.stdout}\n\nstderr: {exe_result.stderr}") + actual_result = exe_result.stdout.decode("utf-8") + + if snapshot: + with open(out_path, "w") as out_file: + out_file.write(actual_result) + return (TestResultStatus.WROTE_NEW, None) + else: + with open(out_path, "r") as out_file: + expected_result = out_file.read() + + if expected_result != actual_result: + return (TestResultStatus.DIFF_ERROR, DiffFailure(expected_result, actual_result)) + + return (TestResultStatus.SUCCESS, None) + +def should_run_case(case_name, filters): + if len(filters) == 0: + return True + + return any([f in case_name for f in filters]) + +def run_test_suite(args, suite, suite_root, suite_exe): + results = [] + + for entry in os.listdir(suite_root): + if not should_run_case(entry, args.cases): + continue + + case_path = os.path.join(suite_root, entry) + if os.path.isdir(case_path): + in_path = os.path.join(case_path, IN_FILE_NAME) + out_path = os.path.join(case_path, OUT_FILE_NAME) + + if not os.path.exists(in_path) or not os.path.exists(out_path): + continue + + status, details = run_test(in_path, out_path, args.cli_location, suite_exe, args.snapshot) + result = TestCaseResult(suite, entry, status, details) + results.append(result) + + return results + +def main(): + args = parser.parse_args() + + suites = args.suites if len(args.suites) > 0 else SUITES + root = os.path.abspath(args.prototype_root) + + if args.build: + for suite in suites: + success, reason = build_suite(root, suite) + + if not success: + print(f"Error building executable for test suite {suite}:\n{reason}") + sys.exit(1) + else: + print(f"Built executable for test suite {suite} successfully.") + + failed = False + for suite in suites: + suite_root = os.path.join(root, SUITE_ROOTS.get(suite)) + suite_exe = os.path.join(root, SUITE_EXE_NAMES.get(suite)) + print(f"Running test suite {suite}...") + results = run_test_suite(args, suite, suite_root, suite_exe) + + passed = 0 + total = len(results) + + for result in results: + if result.did_pass(): + passed += 1 + else: + failed = True + + print(f"Suite {suite} [{passed} / {total} passed]:") + for result in results: + print(result.to_string()) + + if args.write_diffs: + result.write_artifact(args.diff_location) + + if failed: + sys.exit(1) + +if __name__ == "__main__": + main() From 5b784650594ffbef70c731fb6c9cef136226121c Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Thu, 17 Feb 2022 17:18:01 -0800 Subject: [PATCH 17/25] Sync to upstream/release/514 (#372) --- Analysis/include/Luau/Documentation.h | 3 + Analysis/include/Luau/Frontend.h | 3 +- Analysis/include/Luau/Linter.h | 7 +- Analysis/include/Luau/Module.h | 4 +- Analysis/include/Luau/Quantify.h | 5 +- Analysis/include/Luau/Substitution.h | 20 +- Analysis/include/Luau/TypeInfer.h | 45 +++- Analysis/src/Autocomplete.cpp | 45 ++-- Analysis/src/Config.cpp | 2 +- Analysis/src/EmbeddedBuiltinDefinitions.cpp | 2 +- Analysis/src/Frontend.cpp | 35 ++- Analysis/src/JsonEncoder.cpp | 21 +- Analysis/src/Linter.cpp | 126 ++++++++- Analysis/src/Quantify.cpp | 10 +- Analysis/src/Substitution.cpp | 23 -- Analysis/src/TopoSortStatements.cpp | 3 +- Analysis/src/Transpiler.cpp | 8 +- Analysis/src/TypeAttach.cpp | 4 +- Analysis/src/TypeInfer.cpp | 142 +++-------- Analysis/src/TypeUtils.cpp | 12 + Analysis/src/TypeVar.cpp | 6 +- Analysis/src/TypedAllocator.cpp | 3 + Analysis/src/Unifier.cpp | 16 +- Ast/include/Luau/Ast.h | 17 +- Ast/include/Luau/Lexer.h | 5 + Ast/include/Luau/ParseResult.h | 69 +++++ Ast/include/Luau/Parser.h | 56 +--- Ast/src/Ast.cpp | 29 +-- Ast/src/Lexer.cpp | 12 +- Ast/src/Parser.cpp | 167 +++++++----- Ast/src/TimeTrace.cpp | 6 + CLI/FileUtils.cpp | 7 +- CLI/Repl.cpp | 171 +++++++++---- CMakeLists.txt | 14 + Sources.cmake | 1 + VM/src/lapi.cpp | 25 +- VM/src/ldebug.cpp | 2 +- VM/src/ldo.cpp | 13 +- VM/src/ldo.h | 2 +- VM/src/lgc.cpp | 4 +- VM/src/lgc.h | 2 +- VM/src/lgcdebug.cpp | 10 +- VM/src/lperf.cpp | 6 + VM/src/lstate.cpp | 35 ++- VM/src/lstate.h | 8 +- VM/src/lvmexecute.cpp | 2 +- VM/src/lvmload.cpp | 6 +- VM/src/lvmutils.cpp | 4 +- fuzz/linter.cpp | 2 +- fuzz/proto.cpp | 2 +- tests/Autocomplete.test.cpp | 22 +- tests/Compiler.test.cpp | 14 +- tests/Conformance.test.cpp | 8 + tests/Fixture.cpp | 4 +- tests/Fixture.h | 1 - tests/Frontend.test.cpp | 3 - tests/Linter.test.cpp | 48 +++- tests/NonstrictMode.test.cpp | 1 - tests/Parser.test.cpp | 51 ++-- tests/Repl.test.cpp | 209 ++++++++++++++- tests/RequireTracer.test.cpp | 2 +- tests/TypeInfer.aliases.test.cpp | 6 +- tests/TypeInfer.annotations.test.cpp | 1 - tests/TypeInfer.builtins.test.cpp | 1 - tests/TypeInfer.classes.test.cpp | 1 - tests/TypeInfer.definitions.test.cpp | 1 - tests/TypeInfer.generics.test.cpp | 1 - tests/TypeInfer.intersectionTypes.test.cpp | 1 - tests/TypeInfer.provisional.test.cpp | 1 - tests/TypeInfer.singletons.test.cpp | 2 - tests/TypeInfer.tables.test.cpp | 43 +++- tests/TypeInfer.test.cpp | 27 +- tests/TypeInfer.tryUnify.test.cpp | 1 - tests/TypeInfer.typePacks.cpp | 3 - tests/TypeInfer.unionTypes.test.cpp | 1 - tests/TypePack.test.cpp | 1 - tests/TypeVar.test.cpp | 1 - tests/conformance/coroutine.lua | 9 + tests/conformance/coverage.lua | 8 + tests/conformance/debug.lua | 3 + tests/main.cpp | 7 +- tools/natvis/Analysis.natvis | 78 ++++++ tools/natvis/Ast.natvis | 25 ++ tools/natvis/VM.natvis | 269 ++++++++++++++++++++ 84 files changed, 1497 insertions(+), 579 deletions(-) create mode 100644 Ast/include/Luau/ParseResult.h create mode 100644 tools/natvis/Analysis.natvis create mode 100644 tools/natvis/Ast.natvis create mode 100644 tools/natvis/VM.natvis diff --git a/Analysis/include/Luau/Documentation.h b/Analysis/include/Luau/Documentation.h index 68ff3a7c..7a2b56ff 100644 --- a/Analysis/include/Luau/Documentation.h +++ b/Analysis/include/Luau/Documentation.h @@ -21,6 +21,7 @@ struct BasicDocumentation { std::string documentation; std::string learnMoreLink; + std::string codeSample; }; struct FunctionParameterDocumentation @@ -37,6 +38,7 @@ struct FunctionDocumentation std::vector parameters; std::vector returns; std::string learnMoreLink; + std::string codeSample; }; struct OverloadedFunctionDocumentation @@ -52,6 +54,7 @@ struct TableDocumentation std::string documentation; Luau::DenseHashMap keys; std::string learnMoreLink; + std::string codeSample; }; using DocumentationDatabase = Luau::DenseHashMap; diff --git a/Analysis/include/Luau/Frontend.h b/Analysis/include/Luau/Frontend.h index 1f64db30..0bf8f362 100644 --- a/Analysis/include/Luau/Frontend.h +++ b/Analysis/include/Luau/Frontend.h @@ -24,6 +24,7 @@ struct TypeChecker; struct FileResolver; struct ModuleResolver; struct ParseResult; +struct HotComment; struct LoadDefinitionFileResult { @@ -35,7 +36,7 @@ struct LoadDefinitionFileResult LoadDefinitionFileResult loadDefinitionFile( TypeChecker& typeChecker, ScopePtr targetScope, std::string_view definition, const std::string& packageName); -std::optional parseMode(const std::vector& hotcomments); +std::optional parseMode(const std::vector& hotcomments); std::vector parsePathExpr(const AstExpr& pathExpr); diff --git a/Analysis/include/Luau/Linter.h b/Analysis/include/Luau/Linter.h index ec3c124d..6c7ce47f 100644 --- a/Analysis/include/Luau/Linter.h +++ b/Analysis/include/Luau/Linter.h @@ -14,6 +14,7 @@ class AstStat; class AstNameTable; struct TypeChecker; struct Module; +struct HotComment; using ScopePtr = std::shared_ptr; @@ -50,6 +51,7 @@ struct LintWarning Code_TableOperations = 23, Code_DuplicateCondition = 24, Code_MisleadingAndOr = 25, + Code_CommentDirective = 26, Code__Count }; @@ -60,7 +62,7 @@ struct LintWarning static const char* getName(Code code); static Code parseName(const char* name); - static uint64_t parseMask(const std::vector& hotcomments); + static uint64_t parseMask(const std::vector& hotcomments); }; struct LintResult @@ -90,7 +92,8 @@ struct LintOptions void setDefaults(); }; -std::vector lint(AstStat* root, const AstNameTable& names, const ScopePtr& env, const Module* module, const LintOptions& options); +std::vector lint(AstStat* root, const AstNameTable& names, const ScopePtr& env, const Module* module, + const std::vector& hotcomments, const LintOptions& options); std::vector getDeprecatedGlobals(const AstNameTable& names); diff --git a/Analysis/include/Luau/Module.h b/Analysis/include/Luau/Module.h index 1bf0473c..61200771 100644 --- a/Analysis/include/Luau/Module.h +++ b/Analysis/include/Luau/Module.h @@ -6,7 +6,7 @@ #include "Luau/TypedAllocator.h" #include "Luau/ParseOptions.h" #include "Luau/Error.h" -#include "Luau/Parser.h" +#include "Luau/ParseResult.h" #include #include @@ -37,8 +37,8 @@ struct SourceModule AstStatBlock* root = nullptr; std::optional mode; - uint64_t ignoreLints = 0; + std::vector hotcomments; std::vector commentLocations; SourceModule() diff --git a/Analysis/include/Luau/Quantify.h b/Analysis/include/Luau/Quantify.h index f46df146..e48cad40 100644 --- a/Analysis/include/Luau/Quantify.h +++ b/Analysis/include/Luau/Quantify.h @@ -6,9 +6,6 @@ namespace Luau { -struct Module; -using ModulePtr = std::shared_ptr; - -void quantify(ModulePtr module, TypeId ty, TypeLevel level); +void quantify(TypeId ty, TypeLevel level); } // namespace Luau diff --git a/Analysis/include/Luau/Substitution.h b/Analysis/include/Luau/Substitution.h index f85b4269..9662d5b3 100644 --- a/Analysis/include/Luau/Substitution.h +++ b/Analysis/include/Luau/Substitution.h @@ -101,9 +101,6 @@ struct Tarjan // This is hot code, so we optimize recursion to a stack. TarjanResult loop(); - // Clear the state - void clear(); - // Find or create the index for a vertex. // Return a boolean which is `true` if it's a freshly created index. std::pair indexify(TypeId ty); @@ -166,7 +163,17 @@ struct FindDirty : Tarjan // and replaces them with clean ones. struct Substitution : FindDirty { - ModulePtr currentModule; +protected: + Substitution(const TxnLog* log_, TypeArena* arena) + : arena(arena) + { + log = log_; + LUAU_ASSERT(log); + LUAU_ASSERT(arena); + } + +public: + TypeArena* arena; DenseHashMap newTypes{nullptr}; DenseHashMap newPacks{nullptr}; @@ -192,12 +199,13 @@ struct Substitution : FindDirty template TypeId addType(const T& tv) { - return currentModule->internalTypes.addType(tv); + return arena->addType(tv); } + template TypePackId addTypePack(const T& tp) { - return currentModule->internalTypes.addTypePack(TypePackVar{tp}); + return arena->addTypePack(TypePackVar{tp}); } }; diff --git a/Analysis/include/Luau/TypeInfer.h b/Analysis/include/Luau/TypeInfer.h index 5592fa1f..3c5ded3c 100644 --- a/Analysis/include/Luau/TypeInfer.h +++ b/Analysis/include/Luau/TypeInfer.h @@ -5,7 +5,6 @@ #include "Luau/Error.h" #include "Luau/Module.h" #include "Luau/Symbol.h" -#include "Luau/Parser.h" #include "Luau/Substitution.h" #include "Luau/TxnLog.h" #include "Luau/TypePack.h" @@ -37,6 +36,15 @@ struct Unifier; // A substitution which replaces generic types in a given set by free types. struct ReplaceGenerics : Substitution { + ReplaceGenerics( + const TxnLog* log, TypeArena* arena, TypeLevel level, const std::vector& generics, const std::vector& genericPacks) + : Substitution(log, arena) + , level(level) + , generics(generics) + , genericPacks(genericPacks) + { + } + TypeLevel level; std::vector generics; std::vector genericPacks; @@ -50,8 +58,13 @@ struct ReplaceGenerics : Substitution // A substitution which replaces generic functions by monomorphic functions struct Instantiation : Substitution { + Instantiation(const TxnLog* log, TypeArena* arena, TypeLevel level) + : Substitution(log, arena) + , level(level) + { + } + TypeLevel level; - ReplaceGenerics replaceGenerics; bool ignoreChildren(TypeId ty) override; bool isDirty(TypeId ty) override; bool isDirty(TypePackId tp) override; @@ -62,6 +75,12 @@ struct Instantiation : Substitution // A substitution which replaces free types by generic types. struct Quantification : Substitution { + Quantification(TypeArena* arena, TypeLevel level) + : Substitution(TxnLog::empty(), arena) + , level(level) + { + } + TypeLevel level; std::vector generics; std::vector genericPacks; @@ -74,6 +93,13 @@ struct Quantification : Substitution // A substitution which replaces free types by any struct Anyification : Substitution { + Anyification(TypeArena* arena, TypeId anyType, TypePackId anyTypePack) + : Substitution(TxnLog::empty(), arena) + , anyType(anyType) + , anyTypePack(anyTypePack) + { + } + TypeId anyType; TypePackId anyTypePack; bool isDirty(TypeId ty) override; @@ -85,6 +111,13 @@ struct Anyification : Substitution // A substitution which replaces the type parameters of a type function by arguments struct ApplyTypeFunction : Substitution { + ApplyTypeFunction(TypeArena* arena, TypeLevel level) + : Substitution(TxnLog::empty(), arena) + , level(level) + , encounteredForwardedType(false) + { + } + TypeLevel level; bool encounteredForwardedType; std::unordered_map typeArguments; @@ -351,8 +384,7 @@ private: // Note: `scope` must be a fresh scope. GenericTypeDefinitions createGenericTypes(const ScopePtr& scope, std::optional levelOpt, const AstNode& node, - const AstArray& genericNames, const AstArray& genericPackNames, - bool useCache = false); + const AstArray& genericNames, const AstArray& genericPackNames, bool useCache = false); public: ErrorVec resolve(const PredicateVec& predicates, const ScopePtr& scope, bool sense); @@ -392,11 +424,6 @@ public: ModulePtr currentModule; ModuleName currentModuleName; - Instantiation instantiation; - Quantification quantification; - Anyification anyification; - ApplyTypeFunction applyTypeFunction; - std::function prepareModuleScope; InternalErrorReporter* iceHandler; diff --git a/Analysis/src/Autocomplete.cpp b/Analysis/src/Autocomplete.cpp index 85099e12..5a1ae397 100644 --- a/Analysis/src/Autocomplete.cpp +++ b/Analysis/src/Autocomplete.cpp @@ -7,6 +7,7 @@ #include "Luau/ToString.h" #include "Luau/TypeInfer.h" #include "Luau/TypePack.h" +#include "Luau/Parser.h" // TODO: only needed for autocompleteSource which is deprecated #include #include @@ -14,9 +15,9 @@ LUAU_FASTFLAG(LuauUseCommittingTxnLog) LUAU_FASTFLAGVARIABLE(LuauAutocompleteAvoidMutation, false); -LUAU_FASTFLAGVARIABLE(LuauCompleteBrokenStringParams, false); LUAU_FASTFLAGVARIABLE(LuauMissingFollowACMetatables, false); LUAU_FASTFLAGVARIABLE(PreferToCallFunctionsForIntersects, false); +LUAU_FASTFLAGVARIABLE(LuauIfElseExprFixCompletionIssue, false); static const std::unordered_set kStatementStartingKeywords = { "while", "if", "local", "repeat", "function", "do", "for", "return", "break", "continue", "type", "export"}; @@ -380,7 +381,7 @@ static void autocompleteProps(const Module& module, TypeArena* typeArena, TypeId { // We are walking up the class hierarchy, so if we encounter a property that we have // already populated, it takes precedence over the property we found just now. - if (result.count(name) == 0 && name != Parser::errorName) + if (result.count(name) == 0 && name != kParseNameError) { Luau::TypeId type = Luau::follow(prop.type); TypeCorrectKind typeCorrect = indexType == PropIndexType::Key ? TypeCorrectKind::Correct @@ -948,9 +949,12 @@ AutocompleteEntryMap autocompleteTypeNames(const Module& module, Position positi } } - for (size_t i = 0; i < node->returnAnnotation.types.size; i++) + if (!node->returnAnnotation) + return result; + + for (size_t i = 0; i < node->returnAnnotation->types.size; i++) { - AstType* ret = node->returnAnnotation.types.data[i]; + AstType* ret = node->returnAnnotation->types.data[i]; if (ret->location.containsClosed(position)) { @@ -965,7 +969,7 @@ AutocompleteEntryMap autocompleteTypeNames(const Module& module, Position positi } } - if (AstTypePack* retTp = node->returnAnnotation.tailType) + if (AstTypePack* retTp = node->returnAnnotation->tailType) { if (auto variadic = retTp->as()) { @@ -1136,7 +1140,7 @@ static AutocompleteEntryMap autocompleteStatement( AstNode* parent = ancestry.rbegin()[1]; if (AstStatIf* statIf = parent->as()) { - if (!statIf->elsebody || (statIf->hasElse && statIf->elseLocation.containsClosed(position))) + if (!statIf->elsebody || (statIf->elseLocation && statIf->elseLocation->containsClosed(position))) { result.emplace("else", AutocompleteEntry{AutocompleteEntryKind::Keyword}); result.emplace("elseif", AutocompleteEntry{AutocompleteEntryKind::Keyword}); @@ -1164,8 +1168,7 @@ static AutocompleteEntryMap autocompleteStatement( return result; } -// Returns true if completions were generated (completions will be inserted into 'outResult') -// Returns false if no completions were generated +// Returns true iff `node` was handled by this function (completions, if any, are returned in `outResult`) static bool autocompleteIfElseExpression( const AstNode* node, const std::vector& ancestry, const Position& position, AutocompleteEntryMap& outResult) { @@ -1173,6 +1176,13 @@ static bool autocompleteIfElseExpression( if (!parent) return false; + if (FFlag::LuauIfElseExprFixCompletionIssue && node->is()) + { + // Don't try to complete when the current node is an if-else expression (i.e. only try to complete when the node is a child of an if-else + // expression. + return true; + } + AstExprIfElse* ifElseExpr = parent->as(); if (!ifElseExpr || ifElseExpr->condition->location.containsClosed(position)) { @@ -1310,7 +1320,7 @@ static std::optional autocompleteStringParams(const Source return std::nullopt; } - if (!nodes.back()->is() && (!FFlag::LuauCompleteBrokenStringParams || !nodes.back()->is())) + if (!nodes.back()->is() && !nodes.back()->is()) { return std::nullopt; } @@ -1408,8 +1418,8 @@ static AutocompleteResult autocomplete(const SourceModule& sourceModule, const M } else if (auto typeReference = node->as()) { - if (typeReference->hasPrefix) - return {autocompleteModuleTypes(*module, position, typeReference->prefix.value), finder.ancestry}; + if (typeReference->prefix) + return {autocompleteModuleTypes(*module, position, typeReference->prefix->value), finder.ancestry}; else return {autocompleteTypeNames(*module, position, finder.ancestry), finder.ancestry}; } @@ -1419,9 +1429,9 @@ static AutocompleteResult autocomplete(const SourceModule& sourceModule, const M } else if (AstStatLocal* statLocal = node->as()) { - if (statLocal->vars.size == 1 && (!statLocal->hasEqualsSign || position < statLocal->equalsSignLocation.begin)) + if (statLocal->vars.size == 1 && (!statLocal->equalsSignLocation || position < statLocal->equalsSignLocation->begin)) return {{{"function", AutocompleteEntry{AutocompleteEntryKind::Keyword}}}, finder.ancestry}; - else if (statLocal->hasEqualsSign && position >= statLocal->equalsSignLocation.end) + else if (statLocal->equalsSignLocation && position >= statLocal->equalsSignLocation->end) return {autocompleteExpression(sourceModule, *module, typeChecker, typeArena, finder.ancestry, position), finder.ancestry}; else return {}; @@ -1449,7 +1459,7 @@ static AutocompleteResult autocomplete(const SourceModule& sourceModule, const M if (!statForIn->hasIn || position <= statForIn->inLocation.begin) { AstLocal* lastName = statForIn->vars.data[statForIn->vars.size - 1]; - if (lastName->name == Parser::errorName || lastName->location.containsClosed(position)) + if (lastName->name == kParseNameError || lastName->location.containsClosed(position)) { // Here we are either working with a missing binding (as would be the case in a bare "for" keyword) or // the cursor is still touching a binding name. The user is still typing a new name, so we should not offer @@ -1499,7 +1509,7 @@ static AutocompleteResult autocomplete(const SourceModule& sourceModule, const M else if (AstStatWhile* statWhile = extractStat(finder.ancestry); statWhile && !statWhile->hasDo) return {{{"do", AutocompleteEntry{AutocompleteEntryKind::Keyword}}}, finder.ancestry}; - else if (AstStatIf* statIf = node->as(); statIf && !statIf->hasElse) + else if (AstStatIf* statIf = node->as(); statIf && !statIf->elseLocation.has_value()) { return {{{"else", AutocompleteEntry{AutocompleteEntryKind::Keyword}}, {"elseif", AutocompleteEntry{AutocompleteEntryKind::Keyword}}}, finder.ancestry}; @@ -1508,11 +1518,11 @@ static AutocompleteResult autocomplete(const SourceModule& sourceModule, const M { if (statIf->condition->is()) return {autocompleteExpression(sourceModule, *module, typeChecker, typeArena, finder.ancestry, position), finder.ancestry}; - else if (!statIf->hasThen || statIf->thenLocation.containsClosed(position)) + else if (!statIf->thenLocation || statIf->thenLocation->containsClosed(position)) return {{{"then", AutocompleteEntry{AutocompleteEntryKind::Keyword}}}, finder.ancestry}; } else if (AstStatIf* statIf = extractStat(finder.ancestry); - statIf && (!statIf->hasThen || statIf->thenLocation.containsClosed(position))) + statIf && (!statIf->thenLocation || statIf->thenLocation->containsClosed(position))) return {{{"then", AutocompleteEntry{AutocompleteEntryKind::Keyword}}}, finder.ancestry}; else if (AstStatRepeat* statRepeat = node->as(); statRepeat && statRepeat->condition->is()) return {autocompleteExpression(sourceModule, *module, typeChecker, typeArena, finder.ancestry, position), finder.ancestry}; @@ -1612,6 +1622,7 @@ AutocompleteResult autocomplete(Frontend& frontend, const ModuleName& moduleName OwningAutocompleteResult autocompleteSource(Frontend& frontend, std::string_view source, Position position, StringCompletionCallback callback) { + // TODO: Remove #include "Luau/Parser.h" with this function auto sourceModule = std::make_unique(); ParseOptions parseOptions; parseOptions.captureComments = true; diff --git a/Analysis/src/Config.cpp b/Analysis/src/Config.cpp index d9fc44f8..35a2259d 100644 --- a/Analysis/src/Config.cpp +++ b/Analysis/src/Config.cpp @@ -1,7 +1,7 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/Config.h" -#include "Luau/Parser.h" +#include "Luau/Lexer.h" #include "Luau/StringUtils.h" namespace diff --git a/Analysis/src/EmbeddedBuiltinDefinitions.cpp b/Analysis/src/EmbeddedBuiltinDefinitions.cpp index f3ef88fc..bf6e1193 100644 --- a/Analysis/src/EmbeddedBuiltinDefinitions.cpp +++ b/Analysis/src/EmbeddedBuiltinDefinitions.cpp @@ -167,7 +167,7 @@ declare function gcinfo(): number foreach: ({[K]: V}, (K, V) -> ()) -> (), foreachi: ({V}, (number, V) -> ()) -> (), - move: ({V}, number, number, number, {V}?) -> (), + move: ({V}, number, number, number, {V}?) -> {V}, clear: ({[K]: V}) -> (), freeze: ({[K]: V}) -> {[K]: V}, diff --git a/Analysis/src/Frontend.cpp b/Analysis/src/Frontend.cpp index 9001b19d..d8906f6e 100644 --- a/Analysis/src/Frontend.cpp +++ b/Analysis/src/Frontend.cpp @@ -4,6 +4,7 @@ #include "Luau/Common.h" #include "Luau/Config.h" #include "Luau/FileResolver.h" +#include "Luau/Parser.h" #include "Luau/Scope.h" #include "Luau/StringUtils.h" #include "Luau/TimeTrace.h" @@ -16,23 +17,25 @@ #include LUAU_FASTFLAG(LuauInferInNoCheckMode) -LUAU_FASTFLAGVARIABLE(LuauTypeCheckTwice, false) LUAU_FASTFLAGVARIABLE(LuauKnowsTheDataModel3, false) namespace Luau { -std::optional parseMode(const std::vector& hotcomments) +std::optional parseMode(const std::vector& hotcomments) { - for (const std::string& hc : hotcomments) + for (const HotComment& hc : hotcomments) { - if (hc == "nocheck") + if (!hc.header) + continue; + + if (hc.content == "nocheck") return Mode::NoCheck; - if (hc == "nonstrict") + if (hc.content == "nonstrict") return Mode::Nonstrict; - if (hc == "strict") + if (hc.content == "strict") return Mode::Strict; } @@ -607,13 +610,15 @@ std::pair Frontend::lintFragment(std::string_view sour SourceModule sourceModule = parse(ModuleName{}, source, config.parseOptions); + uint64_t ignoreLints = LintWarning::parseMask(sourceModule.hotcomments); + Luau::LintOptions lintOptions = enabledLintWarnings.value_or(config.enabledLint); - lintOptions.warningMask &= sourceModule.ignoreLints; + lintOptions.warningMask &= ~ignoreLints; double timestamp = getTimestamp(); - std::vector warnings = - Luau::lint(sourceModule.root, *sourceModule.names.get(), typeChecker.globalScope, nullptr, enabledLintWarnings.value_or(config.enabledLint)); + std::vector warnings = Luau::lint(sourceModule.root, *sourceModule.names.get(), typeChecker.globalScope, nullptr, + sourceModule.hotcomments, enabledLintWarnings.value_or(config.enabledLint)); stats.timeLint += getTimestamp() - timestamp; @@ -651,8 +656,10 @@ LintResult Frontend::lint(const SourceModule& module, std::optionalgetConfig(module.name); + uint64_t ignoreLints = LintWarning::parseMask(module.hotcomments); + LintOptions options = enabledLintWarnings.value_or(config.enabledLint); - options.warningMask &= ~module.ignoreLints; + options.warningMask &= ~ignoreLints; Mode mode = module.mode.value_or(config.mode); if (mode != Mode::NoCheck) @@ -671,7 +678,7 @@ LintResult Frontend::lint(const SourceModule& module, std::optional warnings = Luau::lint(module.root, *module.names, environmentScope, modulePtr.get(), options); + std::vector warnings = Luau::lint(module.root, *module.names, environmentScope, modulePtr.get(), module.hotcomments, options); stats.timeLint += getTimestamp() - timestamp; @@ -839,7 +846,6 @@ SourceModule Frontend::parse(const ModuleName& name, std::string_view src, const { sourceModule.root = parseResult.root; sourceModule.mode = parseMode(parseResult.hotcomments); - sourceModule.ignoreLints = LintWarning::parseMask(parseResult.hotcomments); } else { @@ -848,8 +854,13 @@ SourceModule Frontend::parse(const ModuleName& name, std::string_view src, const } sourceModule.name = name; + if (parseOptions.captureComments) + { sourceModule.commentLocations = std::move(parseResult.commentLocations); + sourceModule.hotcomments = std::move(parseResult.hotcomments); + } + return sourceModule; } diff --git a/Analysis/src/JsonEncoder.cpp b/Analysis/src/JsonEncoder.cpp index 0d48f2ca..ec399158 100644 --- a/Analysis/src/JsonEncoder.cpp +++ b/Analysis/src/JsonEncoder.cpp @@ -158,6 +158,13 @@ struct AstJsonEncoder : public AstVisitor { writeString(str); } + void write(std::optional name) + { + if (name) + write(*name); + else + writeRaw("null"); + } void write(AstName name) { writeString(name.value ? name.value : ""); @@ -327,7 +334,7 @@ struct AstJsonEncoder : public AstVisitor if (node->self) PROP(self); PROP(args); - if (node->hasReturnAnnotation) + if (node->returnAnnotation) PROP(returnAnnotation); PROP(vararg); PROP(varargLocation); @@ -341,6 +348,14 @@ struct AstJsonEncoder : public AstVisitor }); } + void write(const std::optional& typeList) + { + if (typeList) + write(*typeList); + else + writeRaw("null"); + } + void write(const AstTypeList& typeList) { writeRaw("{"); @@ -544,7 +559,7 @@ struct AstJsonEncoder : public AstVisitor PROP(thenbody); if (node->elsebody) PROP(elsebody); - PROP(hasThen); + write("hasThen", node->thenLocation.has_value()); PROP(hasEnd); }); } @@ -728,7 +743,7 @@ struct AstJsonEncoder : public AstVisitor void write(class AstTypeReference* node) { writeNode(node, "AstTypeReference", [&]() { - if (node->hasPrefix) + if (node->prefix) PROP(prefix); PROP(name); PROP(parameters); diff --git a/Analysis/src/Linter.cpp b/Analysis/src/Linter.cpp index 2ba6a0fc..8d7d2d97 100644 --- a/Analysis/src/Linter.cpp +++ b/Analysis/src/Linter.cpp @@ -12,6 +12,8 @@ #include #include +LUAU_FASTINTVARIABLE(LuauSuggestionDistance, 4) + namespace Luau { @@ -44,6 +46,7 @@ static const char* kWarningNames[] = { "TableOperations", "DuplicateCondition", "MisleadingAndOr", + "CommentDirective", }; // clang-format on @@ -732,13 +735,13 @@ private: bool visit(AstTypeReference* node) override { - if (!node->hasPrefix) + if (!node->prefix) return true; - if (!imports.contains(node->prefix)) + if (!imports.contains(*node->prefix)) return true; - AstLocal* astLocal = imports[node->prefix]; + AstLocal* astLocal = imports[*node->prefix]; Local& local = locals[astLocal]; LUAU_ASSERT(local.import); local.used = true; @@ -2527,13 +2530,108 @@ static void fillBuiltinGlobals(LintContext& context, const AstNameTable& names, } } +static const char* fuzzyMatch(std::string_view str, const char** array, size_t size) +{ + if (FInt::LuauSuggestionDistance == 0) + return nullptr; + + size_t bestDistance = FInt::LuauSuggestionDistance; + size_t bestMatch = size; + + for (size_t i = 0; i < size; ++i) + { + size_t ed = editDistance(str, array[i]); + + if (ed <= bestDistance) + { + bestDistance = ed; + bestMatch = i; + } + } + + return bestMatch < size ? array[bestMatch] : nullptr; +} + +static void lintComments(LintContext& context, const std::vector& hotcomments) +{ + bool seenMode = false; + + for (const HotComment& hc : hotcomments) + { + // We reserve --! for various informational (non-directive) comments + if (hc.content.empty() || hc.content[0] == ' ' || hc.content[0] == '\t') + continue; + + if (!hc.header) + { + emitWarning(context, LintWarning::Code_CommentDirective, hc.location, + "Comment directive is ignored because it is placed after the first non-comment token"); + } + else + { + std::string::size_type space = hc.content.find_first_of(" \t"); + std::string_view first = std::string_view(hc.content).substr(0, space); + + if (first == "nolint") + { + std::string::size_type notspace = hc.content.find_first_not_of(" \t", space); + + if (space == std::string::npos || notspace == std::string::npos) + { + // disables all lints + } + else if (LintWarning::parseName(hc.content.c_str() + notspace) == LintWarning::Code_Unknown) + { + const char* rule = hc.content.c_str() + notspace; + + // skip Unknown + if (const char* suggestion = fuzzyMatch(rule, kWarningNames + 1, LintWarning::Code__Count - 1)) + emitWarning(context, LintWarning::Code_CommentDirective, hc.location, + "nolint directive refers to unknown lint rule '%s'; did you mean '%s'?", rule, suggestion); + else + emitWarning( + context, LintWarning::Code_CommentDirective, hc.location, "nolint directive refers to unknown lint rule '%s'", rule); + } + } + else if (first == "nocheck" || first == "nonstrict" || first == "strict") + { + if (space != std::string::npos) + emitWarning(context, LintWarning::Code_CommentDirective, hc.location, + "Comment directive with the type checking mode has extra symbols at the end of the line"); + else if (seenMode) + emitWarning(context, LintWarning::Code_CommentDirective, hc.location, + "Comment directive with the type checking mode has already been used"); + else + seenMode = true; + } + else + { + static const char* kHotComments[] = { + "nolint", + "nocheck", + "nonstrict", + "strict", + }; + + if (const char* suggestion = fuzzyMatch(first, kHotComments, std::size(kHotComments))) + emitWarning(context, LintWarning::Code_CommentDirective, hc.location, "Unknown comment directive '%.*s'; did you mean '%s'?", + int(first.size()), first.data(), suggestion); + else + emitWarning(context, LintWarning::Code_CommentDirective, hc.location, "Unknown comment directive '%.*s'", int(first.size()), + first.data()); + } + } + } +} + void LintOptions::setDefaults() { // By default, we enable all warnings warningMask = ~0ull; } -std::vector lint(AstStat* root, const AstNameTable& names, const ScopePtr& env, const Module* module, const LintOptions& options) +std::vector lint(AstStat* root, const AstNameTable& names, const ScopePtr& env, const Module* module, + const std::vector& hotcomments, const LintOptions& options) { LintContext context; @@ -2609,6 +2707,9 @@ std::vector lint(AstStat* root, const AstNameTable& names, const Sc if (context.warningEnabled(LintWarning::Code_MisleadingAndOr)) LintMisleadingAndOr::process(context); + if (context.warningEnabled(LintWarning::Code_CommentDirective)) + lintComments(context, hotcomments); + std::sort(context.result.begin(), context.result.end(), WarningComparator()); return context.result; @@ -2630,23 +2731,30 @@ LintWarning::Code LintWarning::parseName(const char* name) return Code_Unknown; } -uint64_t LintWarning::parseMask(const std::vector& hotcomments) +uint64_t LintWarning::parseMask(const std::vector& hotcomments) { uint64_t result = 0; - for (const std::string& hc : hotcomments) + for (const HotComment& hc : hotcomments) { - if (hc.compare(0, 6, "nolint") != 0) + if (!hc.header) continue; - std::string::size_type name = hc.find_first_not_of(" \t", 6); + if (hc.content.compare(0, 6, "nolint") != 0) + continue; + + std::string::size_type name = hc.content.find_first_not_of(" \t", 6); // --!nolint disables everything if (name == std::string::npos) return ~0ull; + // --!nolint needs to be followed by a whitespace character + if (name == 6) + continue; + // --!nolint name disables the specific lint - LintWarning::Code code = LintWarning::parseName(hc.c_str() + name); + LintWarning::Code code = LintWarning::parseName(hc.content.c_str() + name); if (code != LintWarning::Code_Unknown) result |= 1ull << int(code); diff --git a/Analysis/src/Quantify.cpp b/Analysis/src/Quantify.cpp index 04ebffc1..94e169f1 100644 --- a/Analysis/src/Quantify.cpp +++ b/Analysis/src/Quantify.cpp @@ -9,14 +9,12 @@ namespace Luau struct Quantifier { - ModulePtr module; TypeLevel level; std::vector generics; std::vector genericPacks; - Quantifier(ModulePtr module, TypeLevel level) - : module(module) - , level(level) + Quantifier(TypeLevel level) + : level(level) { } @@ -76,9 +74,9 @@ struct Quantifier } }; -void quantify(ModulePtr module, TypeId ty, TypeLevel level) +void quantify(TypeId ty, TypeLevel level) { - Quantifier q{std::move(module), level}; + Quantifier q{level}; DenseHashSet seen{nullptr}; visitTypeVarOnce(ty, q, seen); diff --git a/Analysis/src/Substitution.cpp b/Analysis/src/Substitution.cpp index bacbca76..770c7a47 100644 --- a/Analysis/src/Substitution.cpp +++ b/Analysis/src/Substitution.cpp @@ -226,27 +226,11 @@ TarjanResult Tarjan::loop() return TarjanResult::Ok; } -void Tarjan::clear() -{ - typeToIndex.clear(); - indexToType.clear(); - packToIndex.clear(); - indexToPack.clear(); - lowlink.clear(); - stack.clear(); - onStack.clear(); - - edgesTy.clear(); - edgesTp.clear(); - worklist.clear(); -} - TarjanResult Tarjan::visitRoot(TypeId ty) { childCount = 0; ty = log->follow(ty); - clear(); auto [index, fresh] = indexify(ty); worklist.push_back({index, -1, -1}); return loop(); @@ -257,7 +241,6 @@ TarjanResult Tarjan::visitRoot(TypePackId tp) childCount = 0; tp = log->follow(tp); - clear(); auto [index, fresh] = indexify(tp); worklist.push_back({index, -1, -1}); return loop(); @@ -314,21 +297,17 @@ void FindDirty::visitSCC(int index) TarjanResult FindDirty::findDirty(TypeId ty) { - dirty.clear(); return visitRoot(ty); } TarjanResult FindDirty::findDirty(TypePackId tp) { - dirty.clear(); return visitRoot(tp); } std::optional Substitution::substitute(TypeId ty) { ty = log->follow(ty); - newTypes.clear(); - newPacks.clear(); auto result = findDirty(ty); if (result != TarjanResult::Ok) @@ -347,8 +326,6 @@ std::optional Substitution::substitute(TypeId ty) std::optional Substitution::substitute(TypePackId tp) { tp = log->follow(tp); - newTypes.clear(); - newPacks.clear(); auto result = findDirty(tp); if (result != TarjanResult::Ok) diff --git a/Analysis/src/TopoSortStatements.cpp b/Analysis/src/TopoSortStatements.cpp index dba694be..678001bf 100644 --- a/Analysis/src/TopoSortStatements.cpp +++ b/Analysis/src/TopoSortStatements.cpp @@ -26,9 +26,10 @@ * 3. Cyclic dependencies can be resolved by picking an arbitrary statement to check first. */ -#include "Luau/Parser.h" +#include "Luau/Ast.h" #include "Luau/DenseHash.h" #include "Luau/Common.h" +#include "Luau/StringUtils.h" #include #include diff --git a/Analysis/src/Transpiler.cpp b/Analysis/src/Transpiler.cpp index f5908683..54bd0d5e 100644 --- a/Analysis/src/Transpiler.cpp +++ b/Analysis/src/Transpiler.cpp @@ -933,12 +933,12 @@ struct Printer writer.symbol(")"); - if (writeTypes && func.hasReturnAnnotation) + if (writeTypes && func.returnAnnotation) { writer.symbol(":"); writer.space(); - visualizeTypeList(func.returnAnnotation, false); + visualizeTypeList(*func.returnAnnotation, false); } visualizeBlock(*func.body); @@ -989,9 +989,9 @@ struct Printer advance(typeAnnotation.location.begin); if (const auto& a = typeAnnotation.as()) { - if (a->hasPrefix) + if (a->prefix) { - writer.write(a->prefix.value); + writer.write(a->prefix->value); writer.symbol("."); } diff --git a/Analysis/src/TypeAttach.cpp b/Analysis/src/TypeAttach.cpp index 2208213f..d575e023 100644 --- a/Analysis/src/TypeAttach.cpp +++ b/Analysis/src/TypeAttach.cpp @@ -3,7 +3,6 @@ #include "Luau/Error.h" #include "Luau/Module.h" -#include "Luau/Parser.h" #include "Luau/RecursionCounter.h" #include "Luau/Scope.h" #include "Luau/ToString.h" @@ -476,12 +475,11 @@ public: visitLocal(arg); } - if (!fn->hasReturnAnnotation) + if (!fn->returnAnnotation) { if (auto result = getScope(fn->body->location)) { TypePackId ret = result->returnType; - fn->hasReturnAnnotation = true; AstTypePack* variadicAnnotation = nullptr; const auto& [v, tail] = flatten(ret); diff --git a/Analysis/src/TypeInfer.cpp b/Analysis/src/TypeInfer.cpp index f1c314cd..c29699b7 100644 --- a/Analysis/src/TypeInfer.cpp +++ b/Analysis/src/TypeInfer.cpp @@ -3,7 +3,6 @@ #include "Luau/Common.h" #include "Luau/ModuleResolver.h" -#include "Luau/Parser.h" #include "Luau/Quantify.h" #include "Luau/RecursionCounter.h" #include "Luau/Scope.h" @@ -24,16 +23,12 @@ LUAU_FASTINTVARIABLE(LuauTypeInferTypePackLoopLimit, 5000) LUAU_FASTINTVARIABLE(LuauCheckRecursionLimit, 500) LUAU_FASTFLAG(LuauKnowsTheDataModel3) LUAU_FASTFLAGVARIABLE(LuauEqConstraint, false) -LUAU_FASTFLAGVARIABLE(LuauGroupExpectedType, false) LUAU_FASTFLAGVARIABLE(LuauWeakEqConstraint, false) // Eventually removed as false. LUAU_FASTFLAG(LuauUseCommittingTxnLog) LUAU_FASTFLAGVARIABLE(DebugLuauFreezeDuringUnification, false) LUAU_FASTFLAGVARIABLE(LuauRecursiveTypeParameterRestriction, false) LUAU_FASTFLAGVARIABLE(LuauGenericFunctionsDontCacheTypeParams, false) -LUAU_FASTFLAGVARIABLE(LuauIfElseBranchTypeUnion, false) -LUAU_FASTFLAGVARIABLE(LuauIfElseExpectedType2, false) LUAU_FASTFLAGVARIABLE(LuauImmutableTypes, false) -LUAU_FASTFLAGVARIABLE(LuauLengthOnCompositeType, false) LUAU_FASTFLAGVARIABLE(LuauNoSealedTypeMod, false) LUAU_FASTFLAGVARIABLE(LuauQuantifyInPlace2, false) LUAU_FASTFLAGVARIABLE(LuauSealExports, false) @@ -43,7 +38,6 @@ LUAU_FASTFLAGVARIABLE(LuauTypeAliasDefaults, false) LUAU_FASTFLAGVARIABLE(LuauExpectedTypesOfProperties, false) LUAU_FASTFLAGVARIABLE(LuauErrorRecoveryType, false) LUAU_FASTFLAGVARIABLE(LuauPropertiesGetExpectedType, false) -LUAU_FASTFLAGVARIABLE(LuauPerModuleUnificationCache, false) LUAU_FASTFLAGVARIABLE(LuauProperTypeLevels, false) LUAU_FASTFLAGVARIABLE(LuauAscribeCorrectLevelToInferredProperitesOfFreeTables, false) LUAU_FASTFLAG(LuauUnionTagMatchFix) @@ -293,13 +287,10 @@ ModulePtr TypeChecker::check(const SourceModule& module, Mode mode, std::optiona GenericError{"Free types leaked into this module's public interface. This is an internal Luau error; please report it."}}); } - if (FFlag::LuauPerModuleUnificationCache) - { - // Clear unifier cache since it's keyed off internal types that get deallocated - // This avoids fake cross-module cache hits and keeps cache size at bay when typechecking large module graphs. - unifierState.cachedUnify.clear(); - unifierState.skipCacheForType.clear(); - } + // Clear unifier cache since it's keyed off internal types that get deallocated + // This avoids fake cross-module cache hits and keeps cache size at bay when typechecking large module graphs. + unifierState.cachedUnify.clear(); + unifierState.skipCacheForType.clear(); if (FFlag::LuauTwoPassAliasDefinitionFix) duplicateTypeAliases.clear(); @@ -509,7 +500,7 @@ LUAU_NOINLINE void TypeChecker::checkBlockTypeAliases(const ScopePtr& scope, std { if (const auto& typealias = stat->as()) { - if (FFlag::LuauTwoPassAliasDefinitionFix && typealias->name == Parser::errorName) + if (FFlag::LuauTwoPassAliasDefinitionFix && typealias->name == kParseNameError) continue; auto& bindings = typealias->exported ? scope->exportedTypeBindings : scope->privateTypeBindings; @@ -1193,7 +1184,7 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias Name name = typealias.name.value; // If the alias is missing a name, we can't do anything with it. Ignore it. - if (FFlag::LuauTwoPassAliasDefinitionFix && name == Parser::errorName) + if (FFlag::LuauTwoPassAliasDefinitionFix && name == kParseNameError) return; std::optional binding; @@ -1222,7 +1213,8 @@ void TypeChecker::check(const ScopePtr& scope, const AstStatTypeAlias& typealias if (FFlag::LuauProperTypeLevels) aliasScope->level.subLevel = subLevel; - auto [generics, genericPacks] = createGenericTypes(aliasScope, scope->level, typealias, typealias.generics, typealias.genericPacks, /* useCache = */ true); + auto [generics, genericPacks] = + createGenericTypes(aliasScope, scope->level, typealias, typealias.generics, typealias.genericPacks, /* useCache = */ true); TypeId ty = freshType(aliasScope); FreeTypeVar* ftv = getMutable(ty); @@ -1464,7 +1456,7 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExpr& ExprResult result; if (auto a = expr.as()) - result = checkExpr(scope, *a->expr, FFlag::LuauGroupExpectedType ? expectedType : std::nullopt); + result = checkExpr(scope, *a->expr, expectedType); else if (expr.is()) result = {nilType}; else if (const AstExprConstantBool* bexpr = expr.as()) @@ -1508,7 +1500,7 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExpr& else if (auto a = expr.as()) result = checkExpr(scope, *a); else if (auto a = expr.as()) - result = checkExpr(scope, *a, FFlag::LuauIfElseExpectedType2 ? expectedType : std::nullopt); + result = checkExpr(scope, *a, expectedType); else ice("Unhandled AstExpr?"); @@ -2093,6 +2085,7 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprUn return {numberType}; } case AstExprUnary::Len: + { tablify(operandType); operandType = stripFromNilAndReport(operandType, expr.location); @@ -2100,30 +2093,13 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprUn if (get(operandType)) return {errorRecoveryType(scope)}; - if (FFlag::LuauLengthOnCompositeType) - { - DenseHashSet seen{nullptr}; + DenseHashSet seen{nullptr}; - if (!hasLength(operandType, seen, &recursionCount)) - reportError(TypeError{expr.location, NotATable{operandType}}); - } - else - { - if (get(operandType)) - return {numberType}; // Not strictly correct: metatables permit overriding this - - if (auto p = get(operandType)) - { - if (p->type == PrimitiveTypeVar::String) - return {numberType}; - } - - if (!getTableType(operandType)) - reportError(TypeError{expr.location, NotATable{operandType}}); - } + if (!hasLength(operandType, seen, &recursionCount)) + reportError(TypeError{expr.location, NotATable{operandType}}); return {numberType}; - + } default: ice("Unknown AstExprUnary " + std::to_string(int(expr.op))); } @@ -2618,22 +2594,11 @@ ExprResult TypeChecker::checkExpr(const ScopePtr& scope, const AstExprIf resolve(result.predicates, falseScope, false); ExprResult falseType = checkExpr(falseScope, *expr.falseExpr, expectedType); - if (FFlag::LuauIfElseBranchTypeUnion) - { - if (falseType.type == trueType.type) - return {trueType.type}; - - std::vector types = reduceUnion({trueType.type, falseType.type}); - return {types.size() == 1 ? types[0] : addType(UnionTypeVar{std::move(types)})}; - } - else - { - unify(falseType.type, trueType.type, expr.location); - - // TODO: normalize(UnionTypeVar{{trueType, falseType}}) - // For now both trueType and falseType must be the same type. + if (falseType.type == trueType.type) return {trueType.type}; - } + + std::vector types = reduceUnion({trueType.type, falseType.type}); + return {types.size() == 1 ? types[0] : addType(UnionTypeVar{std::move(types)})}; } TypeId TypeChecker::checkLValue(const ScopePtr& scope, const AstExpr& expr) @@ -2986,8 +2951,8 @@ std::pair TypeChecker::checkFunctionSignature( auto [generics, genericPacks] = createGenericTypes(funScope, std::nullopt, expr, expr.generics, expr.genericPacks); TypePackId retPack; - if (expr.hasReturnAnnotation) - retPack = resolveTypePack(funScope, expr.returnAnnotation); + if (expr.returnAnnotation) + retPack = resolveTypePack(funScope, *expr.returnAnnotation); else if (isNonstrictMode()) retPack = anyTypePack; else if (expectedFunctionType) @@ -3181,7 +3146,7 @@ void TypeChecker::checkFunctionBody(const ScopePtr& scope, TypeId ty, const AstE // If we're in nonstrict mode we want to only report this missing return // statement if there are type annotations on the function. In strict mode // we report it regardless. - if (!isNonstrictMode() || function.hasReturnAnnotation) + if (!isNonstrictMode() || function.returnAnnotation) { reportError(getEndLocation(function), FunctionExitsWithoutReturning{funTy->retType}); } @@ -4403,11 +4368,7 @@ TypeId Instantiation::clean(TypeId ty) // Annoyingly, we have to do this even if there are no generics, // to replace any generic tables. - replaceGenerics.log = log; - replaceGenerics.level = level; - replaceGenerics.currentModule = currentModule; - replaceGenerics.generics.assign(ftv->generics.begin(), ftv->generics.end()); - replaceGenerics.genericPacks.assign(ftv->genericPacks.begin(), ftv->genericPacks.end()); + ReplaceGenerics replaceGenerics{log, arena, level, ftv->generics, ftv->genericPacks}; // TODO: What to do if this returns nullopt? // We don't have access to the error-reporting machinery @@ -4573,16 +4534,11 @@ TypeId TypeChecker::quantify(const ScopePtr& scope, TypeId ty, Location location if (FFlag::LuauQuantifyInPlace2) { - Luau::quantify(currentModule, ty, scope->level); + Luau::quantify(ty, scope->level); return ty; } - quantification.log = TxnLog::empty(); - quantification.level = scope->level; - quantification.generics.clear(); - quantification.genericPacks.clear(); - quantification.currentModule = currentModule; - + Quantification quantification{¤tModule->internalTypes, scope->level}; std::optional qty = quantification.substitute(ty); if (!qty.has_value()) @@ -4596,18 +4552,14 @@ TypeId TypeChecker::quantify(const ScopePtr& scope, TypeId ty, Location location FunctionTypeVar* qftv = getMutable(*qty); LUAU_ASSERT(qftv); - qftv->generics = quantification.generics; - qftv->genericPacks = quantification.genericPacks; + qftv->generics = std::move(quantification.generics); + qftv->genericPacks = std::move(quantification.genericPacks); return *qty; } TypeId TypeChecker::instantiate(const ScopePtr& scope, TypeId ty, Location location, const TxnLog* log) { - LUAU_ASSERT(log != nullptr); - - instantiation.log = FFlag::LuauUseCommittingTxnLog ? log : TxnLog::empty(); - instantiation.level = scope->level; - instantiation.currentModule = currentModule; + Instantiation instantiation{FFlag::LuauUseCommittingTxnLog ? log : TxnLog::empty(), ¤tModule->internalTypes, scope->level}; std::optional instantiated = instantiation.substitute(ty); if (instantiated.has_value()) return *instantiated; @@ -4620,10 +4572,7 @@ TypeId TypeChecker::instantiate(const ScopePtr& scope, TypeId ty, Location locat TypeId TypeChecker::anyify(const ScopePtr& scope, TypeId ty, Location location) { - anyification.log = TxnLog::empty(); - anyification.anyType = anyType; - anyification.anyTypePack = anyTypePack; - anyification.currentModule = currentModule; + Anyification anyification{¤tModule->internalTypes, anyType, anyTypePack}; std::optional any = anyification.substitute(ty); if (any.has_value()) return *any; @@ -4636,10 +4585,7 @@ TypeId TypeChecker::anyify(const ScopePtr& scope, TypeId ty, Location location) TypePackId TypeChecker::anyify(const ScopePtr& scope, TypePackId ty, Location location) { - anyification.log = TxnLog::empty(); - anyification.anyType = anyType; - anyification.anyTypePack = anyTypePack; - anyification.currentModule = currentModule; + Anyification anyification{¤tModule->internalTypes, anyType, anyTypePack}; std::optional any = anyification.substitute(ty); if (any.has_value()) return *any; @@ -4823,7 +4769,8 @@ TypePackId TypeChecker::errorRecoveryTypePack(TypePackId guess) return getSingletonTypes().errorRecoveryTypePack(guess); } -TypeIdPredicate TypeChecker::mkTruthyPredicate(bool sense) { +TypeIdPredicate TypeChecker::mkTruthyPredicate(bool sense) +{ return [this, sense](TypeId ty) -> std::optional { // any/error/free gets a special pass unconditionally because they can't be decided. if (get(ty) || get(ty) || get(ty)) @@ -4904,8 +4851,8 @@ TypeId TypeChecker::resolveType(const ScopePtr& scope, const AstType& annotation if (const auto& lit = annotation.as()) { std::optional tf; - if (lit->hasPrefix) - tf = scope->lookupImportedType(lit->prefix.value, lit->name.value); + if (lit->prefix) + tf = scope->lookupImportedType(lit->prefix->value, lit->name.value); else if (FFlag::DebugLuauMagicTypes && lit->name == "_luau_ice") ice("_luau_ice encountered", lit->location); @@ -4932,12 +4879,12 @@ TypeId TypeChecker::resolveType(const ScopePtr& scope, const AstType& annotation if (!tf) { - if (lit->name == Parser::errorName) + if (lit->name == kParseNameError) return errorRecoveryType(scope); std::string typeName; - if (lit->hasPrefix) - typeName = std::string(lit->prefix.value) + "."; + if (lit->prefix) + typeName = std::string(lit->prefix->value) + "."; typeName += lit->name.value; if (scope->lookupPack(typeName)) @@ -5038,12 +4985,7 @@ TypeId TypeChecker::resolveType(const ScopePtr& scope, const AstType& annotation if (notEnoughParameters && hasDefaultParameters) { // 'applyTypeFunction' is used to substitute default types that reference previous generic types - applyTypeFunction.log = TxnLog::empty(); - applyTypeFunction.typeArguments.clear(); - applyTypeFunction.typePackArguments.clear(); - applyTypeFunction.currentModule = currentModule; - applyTypeFunction.level = scope->level; - applyTypeFunction.encounteredForwardedType = false; + ApplyTypeFunction applyTypeFunction{¤tModule->internalTypes, scope->level}; for (size_t i = 0; i < typesProvided; ++i) applyTypeFunction.typeArguments[tf->typeParams[i].ty] = typeParams[i]; @@ -5362,18 +5304,14 @@ TypeId TypeChecker::instantiateTypeFun(const ScopePtr& scope, const TypeFun& tf, if (tf.typeParams.empty() && tf.typePackParams.empty()) return tf.type; - applyTypeFunction.typeArguments.clear(); + ApplyTypeFunction applyTypeFunction{¤tModule->internalTypes, scope->level}; + for (size_t i = 0; i < tf.typeParams.size(); ++i) applyTypeFunction.typeArguments[tf.typeParams[i].ty] = typeParams[i]; - applyTypeFunction.typePackArguments.clear(); for (size_t i = 0; i < tf.typePackParams.size(); ++i) applyTypeFunction.typePackArguments[tf.typePackParams[i].tp] = typePackParams[i]; - applyTypeFunction.log = TxnLog::empty(); - applyTypeFunction.currentModule = currentModule; - applyTypeFunction.level = scope->level; - applyTypeFunction.encounteredForwardedType = false; std::optional maybeInstantiated = applyTypeFunction.substitute(tf.type); if (!maybeInstantiated.has_value()) { diff --git a/Analysis/src/TypeUtils.cpp b/Analysis/src/TypeUtils.cpp index 8c6d5e49..593b54c8 100644 --- a/Analysis/src/TypeUtils.cpp +++ b/Analysis/src/TypeUtils.cpp @@ -5,6 +5,8 @@ #include "Luau/ToString.h" #include "Luau/TypeInfer.h" +LUAU_FASTFLAGVARIABLE(LuauTerminateCyclicMetatableIndexLookup, false) + namespace Luau { @@ -48,9 +50,19 @@ std::optional findTablePropertyRespectingMeta(ErrorVec& errors, const Sc } std::optional mtIndex = findMetatableEntry(errors, globalScope, ty, "__index", location); + int count = 0; while (mtIndex) { TypeId index = follow(*mtIndex); + + if (FFlag::LuauTerminateCyclicMetatableIndexLookup) + { + if (count >= 100) + return std::nullopt; + + ++count; + } + if (const auto& itt = getTableType(index)) { const auto& fit = itt->props.find(name); diff --git a/Analysis/src/TypeVar.cpp b/Analysis/src/TypeVar.cpp index 7e438e31..b2358c27 100644 --- a/Analysis/src/TypeVar.cpp +++ b/Analysis/src/TypeVar.cpp @@ -23,8 +23,6 @@ LUAU_FASTFLAG(DebugLuauFreezeArena) LUAU_FASTINTVARIABLE(LuauTypeMaximumStringifierLength, 500) LUAU_FASTINTVARIABLE(LuauTableTypeMaximumStringifierLength, 0) LUAU_FASTINT(LuauTypeInferRecursionLimit) -LUAU_FASTFLAG(LuauLengthOnCompositeType) -LUAU_FASTFLAGVARIABLE(LuauMetatableAreEqualRecursion, false) LUAU_FASTFLAGVARIABLE(LuauRefactorTypeVarQuestions, false) LUAU_FASTFLAG(LuauErrorRecoveryType) LUAU_FASTFLAG(LuauUnionTagMatchFix) @@ -385,8 +383,6 @@ bool maybeSingleton(TypeId ty) bool hasLength(TypeId ty, DenseHashSet& seen, int* recursionCount) { - LUAU_ASSERT(FFlag::LuauLengthOnCompositeType); - RecursionLimiter _rl(recursionCount, FInt::LuauTypeInferRecursionLimit); ty = follow(ty); @@ -555,7 +551,7 @@ bool areEqual(SeenSet& seen, const TableTypeVar& lhs, const TableTypeVar& rhs) static bool areEqual(SeenSet& seen, const MetatableTypeVar& lhs, const MetatableTypeVar& rhs) { - if (FFlag::LuauMetatableAreEqualRecursion && areSeen(seen, &lhs, &rhs)) + if (areSeen(seen, &lhs, &rhs)) return true; return areEqual(seen, *lhs.table, *rhs.table) && areEqual(seen, *lhs.metatable, *rhs.metatable); diff --git a/Analysis/src/TypedAllocator.cpp b/Analysis/src/TypedAllocator.cpp index f037351e..c7f31822 100644 --- a/Analysis/src/TypedAllocator.cpp +++ b/Analysis/src/TypedAllocator.cpp @@ -7,6 +7,9 @@ #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include const size_t kPageSize = 4096; diff --git a/Analysis/src/Unifier.cpp b/Analysis/src/Unifier.cpp index a8ad5159..322f6ebf 100644 --- a/Analysis/src/Unifier.cpp +++ b/Analysis/src/Unifier.cpp @@ -14,7 +14,6 @@ LUAU_FASTINT(LuauTypeInferRecursionLimit); LUAU_FASTINT(LuauTypeInferTypePackLoopLimit); -LUAU_FASTFLAGVARIABLE(LuauCommittingTxnLogFreeTpPromote, false) LUAU_FASTFLAG(LuauImmutableTypes) LUAU_FASTFLAG(LuauUseCommittingTxnLog) LUAU_FASTINTVARIABLE(LuauTypeInferIterationLimit, 2000); @@ -23,7 +22,6 @@ LUAU_FASTFLAGVARIABLE(LuauTableUnificationEarlyTest, false) LUAU_FASTFLAG(LuauSingletonTypes) LUAU_FASTFLAG(LuauErrorRecoveryType); LUAU_FASTFLAG(LuauProperTypeLevels); -LUAU_FASTFLAGVARIABLE(LuauUnifyPackTails, false) LUAU_FASTFLAGVARIABLE(LuauUnionTagMatchFix, false) LUAU_FASTFLAGVARIABLE(LuauFollowWithCommittingTxnLogInAnyUnification, false) @@ -116,7 +114,7 @@ struct PromoteTypeLevels { // Surprise, it's actually a BoundTypePack that hasn't been committed yet. // Calling getMutable on this will trigger an assertion. - if (FFlag::LuauCommittingTxnLogFreeTpPromote && FFlag::LuauUseCommittingTxnLog && !log.is(tp)) + if (FFlag::LuauUseCommittingTxnLog && !log.is(tp)) return true; promote(tp, FFlag::LuauUseCommittingTxnLog ? log.getMutable(tp) : getMutable(tp)); @@ -1242,7 +1240,7 @@ void Unifier::tryUnify_(TypePackId subTp, TypePackId superTp, bool isFunctionCal // If both are at the end, we're done if (!superIter.good() && !subIter.good()) { - if (FFlag::LuauUnifyPackTails && subTpv->tail && superTpv->tail) + if (subTpv->tail && superTpv->tail) { tryUnify_(*subTpv->tail, *superTpv->tail); break; @@ -1250,9 +1248,7 @@ void Unifier::tryUnify_(TypePackId subTp, TypePackId superTp, bool isFunctionCal const bool lFreeTail = superTpv->tail && log.getMutable(log.follow(*superTpv->tail)) != nullptr; const bool rFreeTail = subTpv->tail && log.getMutable(log.follow(*subTpv->tail)) != nullptr; - if (!FFlag::LuauUnifyPackTails && lFreeTail && rFreeTail) - tryUnify_(*subTpv->tail, *superTpv->tail); - else if (lFreeTail) + if (lFreeTail) tryUnify_(emptyTp, *superTpv->tail); else if (rFreeTail) tryUnify_(emptyTp, *subTpv->tail); @@ -1448,7 +1444,7 @@ void Unifier::tryUnify_(TypePackId subTp, TypePackId superTp, bool isFunctionCal // If both are at the end, we're done if (!superIter.good() && !subIter.good()) { - if (FFlag::LuauUnifyPackTails && subTpv->tail && superTpv->tail) + if (subTpv->tail && superTpv->tail) { tryUnify_(*subTpv->tail, *superTpv->tail); break; @@ -1456,9 +1452,7 @@ void Unifier::tryUnify_(TypePackId subTp, TypePackId superTp, bool isFunctionCal const bool lFreeTail = superTpv->tail && get(follow(*superTpv->tail)) != nullptr; const bool rFreeTail = subTpv->tail && get(follow(*subTpv->tail)) != nullptr; - if (!FFlag::LuauUnifyPackTails && lFreeTail && rFreeTail) - tryUnify_(*subTpv->tail, *superTpv->tail); - else if (lFreeTail) + if (lFreeTail) tryUnify_(emptyTp, *superTpv->tail); else if (rFreeTail) tryUnify_(emptyTp, *subTpv->tail); diff --git a/Ast/include/Luau/Ast.h b/Ast/include/Luau/Ast.h index ac5950c0..31cd01cc 100644 --- a/Ast/include/Luau/Ast.h +++ b/Ast/include/Luau/Ast.h @@ -594,8 +594,7 @@ public: AstArray genericPacks; AstLocal* self; AstArray args; - bool hasReturnAnnotation; - AstTypeList returnAnnotation; + std::optional returnAnnotation; bool vararg = false; Location varargLocation; AstTypePack* varargAnnotation; @@ -740,7 +739,7 @@ class AstStatIf : public AstStat public: LUAU_RTTI(AstStatIf) - AstStatIf(const Location& location, AstExpr* condition, AstStatBlock* thenbody, AstStat* elsebody, bool hasThen, const Location& thenLocation, + AstStatIf(const Location& location, AstExpr* condition, AstStatBlock* thenbody, AstStat* elsebody, const std::optional& thenLocation, const std::optional& elseLocation, bool hasEnd); void visit(AstVisitor* visitor) override; @@ -749,12 +748,10 @@ public: AstStatBlock* thenbody; AstStat* elsebody; - bool hasThen = false; - Location thenLocation; + std::optional thenLocation; // Active for 'elseif' as well - bool hasElse = false; - Location elseLocation; + std::optional elseLocation; bool hasEnd = false; }; @@ -849,8 +846,7 @@ public: AstArray vars; AstArray values; - bool hasEqualsSign = false; - Location equalsSignLocation; + std::optional equalsSignLocation; }; class AstStatFor : public AstStat @@ -1053,9 +1049,8 @@ public: void visit(AstVisitor* visitor) override; - bool hasPrefix; bool hasParameterList; - AstName prefix; + std::optional prefix; AstName name; AstArray parameters; }; diff --git a/Ast/include/Luau/Lexer.h b/Ast/include/Luau/Lexer.h index 460ef056..d7d867f4 100644 --- a/Ast/include/Luau/Lexer.h +++ b/Ast/include/Luau/Lexer.h @@ -233,4 +233,9 @@ private: bool readNames; }; +inline bool isSpace(char ch) +{ + return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\v' || ch == '\f'; +} + } // namespace Luau diff --git a/Ast/include/Luau/ParseResult.h b/Ast/include/Luau/ParseResult.h new file mode 100644 index 00000000..17ce2e3b --- /dev/null +++ b/Ast/include/Luau/ParseResult.h @@ -0,0 +1,69 @@ +// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details +#pragma once + +#include "Luau/Common.h" +#include "Luau/Location.h" +#include "Luau/Lexer.h" +#include "Luau/StringUtils.h" + +namespace Luau +{ + +class AstStatBlock; + +class ParseError : public std::exception +{ +public: + ParseError(const Location& location, const std::string& message); + + virtual const char* what() const throw(); + + const Location& getLocation() const; + const std::string& getMessage() const; + + static LUAU_NORETURN void raise(const Location& location, const char* format, ...) LUAU_PRINTF_ATTR(2, 3); + +private: + Location location; + std::string message; +}; + +class ParseErrors : public std::exception +{ +public: + ParseErrors(std::vector errors); + + virtual const char* what() const throw(); + + const std::vector& getErrors() const; + +private: + std::vector errors; + std::string message; +}; + +struct HotComment +{ + bool header; + Location location; + std::string content; +}; + +struct Comment +{ + Lexeme::Type type; // Comment, BlockComment, or BrokenComment + Location location; +}; + +struct ParseResult +{ + AstStatBlock* root; + std::vector hotcomments; + std::vector errors; + + std::vector commentLocations; +}; + +static constexpr const char* kParseNameError = "%error-id%"; + +} // namespace Luau diff --git a/Ast/include/Luau/Parser.h b/Ast/include/Luau/Parser.h index 40ecdcdd..4b5ae315 100644 --- a/Ast/include/Luau/Parser.h +++ b/Ast/include/Luau/Parser.h @@ -4,6 +4,7 @@ #include "Luau/Ast.h" #include "Luau/Lexer.h" #include "Luau/ParseOptions.h" +#include "Luau/ParseResult.h" #include "Luau/StringUtils.h" #include "Luau/DenseHash.h" #include "Luau/Common.h" @@ -14,37 +15,6 @@ namespace Luau { -class ParseError : public std::exception -{ -public: - ParseError(const Location& location, const std::string& message); - - virtual const char* what() const throw(); - - const Location& getLocation() const; - const std::string& getMessage() const; - - static LUAU_NORETURN void raise(const Location& location, const char* format, ...) LUAU_PRINTF_ATTR(2, 3); - -private: - Location location; - std::string message; -}; - -class ParseErrors : public std::exception -{ -public: - ParseErrors(std::vector errors); - - virtual const char* what() const throw(); - - const std::vector& getErrors() const; - -private: - std::vector errors; - std::string message; -}; - template class TempVector { @@ -80,34 +50,17 @@ private: size_t size_; }; -struct Comment -{ - Lexeme::Type type; // Comment, BlockComment, or BrokenComment - Location location; -}; - -struct ParseResult -{ - AstStatBlock* root; - std::vector hotcomments; - std::vector errors; - - std::vector commentLocations; -}; - class Parser { public: static ParseResult parse( const char* buffer, std::size_t bufferSize, AstNameTable& names, Allocator& allocator, ParseOptions options = ParseOptions()); - static constexpr const char* errorName = "%error-id%"; - private: struct Name; struct Binding; - Parser(const char* buffer, std::size_t bufferSize, AstNameTable& names, Allocator& allocator); + Parser(const char* buffer, std::size_t bufferSize, AstNameTable& names, Allocator& allocator, const ParseOptions& options); bool blockFollow(const Lexeme& l); @@ -330,7 +283,7 @@ private: AstTypeError* reportTypeAnnotationError(const Location& location, const AstArray& types, bool isMissing, const char* format, ...) LUAU_PRINTF_ATTR(5, 6); - const Lexeme& nextLexeme(); + void nextLexeme(); struct Function { @@ -386,6 +339,9 @@ private: Allocator& allocator; std::vector commentLocations; + std::vector hotcomments; + + bool hotcommentHeader = true; unsigned int recursionCounter; diff --git a/Ast/src/Ast.cpp b/Ast/src/Ast.cpp index 9b5bc0c7..24a280da 100644 --- a/Ast/src/Ast.cpp +++ b/Ast/src/Ast.cpp @@ -167,8 +167,7 @@ AstExprFunction::AstExprFunction(const Location& location, const AstArrayreturnAnnotation = *returnAnnotation; } void AstExprFunction::visit(AstVisitor* visitor) @@ -195,8 +192,8 @@ void AstExprFunction::visit(AstVisitor* visitor) if (varargAnnotation) varargAnnotation->visit(visitor); - if (hasReturnAnnotation) - visitTypeList(visitor, returnAnnotation); + if (returnAnnotation) + visitTypeList(visitor, *returnAnnotation); body->visit(visitor); } @@ -375,21 +372,16 @@ void AstStatBlock::visit(AstVisitor* visitor) } } -AstStatIf::AstStatIf(const Location& location, AstExpr* condition, AstStatBlock* thenbody, AstStat* elsebody, bool hasThen, - const Location& thenLocation, const std::optional& elseLocation, bool hasEnd) +AstStatIf::AstStatIf(const Location& location, AstExpr* condition, AstStatBlock* thenbody, AstStat* elsebody, + const std::optional& thenLocation, const std::optional& elseLocation, bool hasEnd) : AstStat(ClassIndex(), location) , condition(condition) , thenbody(thenbody) , elsebody(elsebody) - , hasThen(hasThen) , thenLocation(thenLocation) + , elseLocation(elseLocation) , hasEnd(hasEnd) { - if (bool(elseLocation)) - { - hasElse = true; - this->elseLocation = *elseLocation; - } } void AstStatIf::visit(AstVisitor* visitor) @@ -492,12 +484,8 @@ AstStatLocal::AstStatLocal( : AstStat(ClassIndex(), location) , vars(vars) , values(values) + , equalsSignLocation(equalsSignLocation) { - if (bool(equalsSignLocation)) - { - hasEqualsSign = true; - this->equalsSignLocation = *equalsSignLocation; - } } void AstStatLocal::visit(AstVisitor* visitor) @@ -750,9 +738,8 @@ void AstStatError::visit(AstVisitor* visitor) AstTypeReference::AstTypeReference( const Location& location, std::optional prefix, AstName name, bool hasParameterList, const AstArray& parameters) : AstType(ClassIndex(), location) - , hasPrefix(bool(prefix)) , hasParameterList(hasParameterList) - , prefix(prefix ? *prefix : AstName()) + , prefix(prefix) , name(name) , parameters(parameters) { diff --git a/Ast/src/Lexer.cpp b/Ast/src/Lexer.cpp index a7aa24ca..d56c8860 100644 --- a/Ast/src/Lexer.cpp +++ b/Ast/src/Lexer.cpp @@ -101,11 +101,6 @@ Lexeme::Lexeme(const Location& location, Type type, const char* name) LUAU_ASSERT(type == Name || (type >= Reserved_BEGIN && type < Lexeme::Reserved_END)); } -static bool isComment(const Lexeme& lexeme) -{ - return lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment; -} - static const char* kReserved[] = {"and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while"}; @@ -282,11 +277,6 @@ AstName AstNameTable::get(const char* name) const return getWithType(name, strlen(name)).first; } -inline bool isSpace(char ch) -{ - return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\v' || ch == '\f'; -} - inline bool isAlpha(char ch) { // use or trick to convert to lower case and unsigned comparison to do range check @@ -372,7 +362,7 @@ const Lexeme& Lexer::next(bool skipComments) prevLocation = lexeme.location; lexeme = readNext(); - } while (skipComments && isComment(lexeme)); + } while (skipComments && (lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment)); return lexeme; } diff --git a/Ast/src/Parser.cpp b/Ast/src/Parser.cpp index 30b32f91..235d6349 100644 --- a/Ast/src/Parser.cpp +++ b/Ast/src/Parser.cpp @@ -13,18 +13,15 @@ LUAU_FASTINTVARIABLE(LuauParseErrorLimit, 100) LUAU_FASTFLAGVARIABLE(LuauParseSingletonTypes, false) LUAU_FASTFLAGVARIABLE(LuauParseTypeAliasDefaults, false) LUAU_FASTFLAGVARIABLE(LuauParseRecoverTypePackEllipsis, false) -LUAU_FASTFLAGVARIABLE(LuauStartingBrokenComment, false) +LUAU_FASTFLAGVARIABLE(LuauParseAllHotComments, false) +LUAU_FASTFLAGVARIABLE(LuauTableFieldFunctionDebugname, false) namespace Luau { -inline bool isSpace(char ch) -{ - return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\v' || ch == '\f'; -} - static bool isComment(const Lexeme& lexeme) { + LUAU_ASSERT(!FFlag::LuauParseAllHotComments); return lexeme.type == Lexeme::Comment || lexeme.type == Lexeme::BlockComment; } @@ -151,31 +148,37 @@ ParseResult Parser::parse(const char* buffer, size_t bufferSize, AstNameTable& n { LUAU_TIMETRACE_SCOPE("Parser::parse", "Parser"); - Parser p(buffer, bufferSize, names, allocator); + Parser p(buffer, bufferSize, names, allocator, FFlag::LuauParseAllHotComments ? options : ParseOptions()); try { - std::vector hotcomments; - - while (isComment(p.lexer.current()) || p.lexer.current().type == Lexeme::BrokenComment) + if (FFlag::LuauParseAllHotComments) { - const char* text = p.lexer.current().data; - unsigned int length = p.lexer.current().length; + AstStatBlock* root = p.parseChunk(); - if (length && text[0] == '!') + return ParseResult{root, std::move(p.hotcomments), std::move(p.parseErrors), std::move(p.commentLocations)}; + } + else + { + std::vector hotcomments; + + while (isComment(p.lexer.current()) || p.lexer.current().type == Lexeme::BrokenComment) { - unsigned int end = length; - while (end > 0 && isSpace(text[end - 1])) - --end; + const char* text = p.lexer.current().data; + unsigned int length = p.lexer.current().length; - hotcomments.push_back(std::string(text + 1, text + end)); - } + if (length && text[0] == '!') + { + unsigned int end = length; + while (end > 0 && isSpace(text[end - 1])) + --end; - const Lexeme::Type type = p.lexer.current().type; - const Location loc = p.lexer.current().location; + hotcomments.push_back({true, p.lexer.current().location, std::string(text + 1, text + end)}); + } + + const Lexeme::Type type = p.lexer.current().type; + const Location loc = p.lexer.current().location; - if (FFlag::LuauStartingBrokenComment) - { if (options.captureComments) p.commentLocations.push_back(Comment{type, loc}); @@ -184,22 +187,15 @@ ParseResult Parser::parse(const char* buffer, size_t bufferSize, AstNameTable& n p.lexer.next(); } - else - { - p.lexer.next(); - if (options.captureComments) - p.commentLocations.push_back(Comment{type, loc}); - } + p.lexer.setSkipComments(true); + + p.options = options; + + AstStatBlock* root = p.parseChunk(); + + return ParseResult{root, hotcomments, p.parseErrors, std::move(p.commentLocations)}; } - - p.lexer.setSkipComments(true); - - p.options = options; - - AstStatBlock* root = p.parseChunk(); - - return ParseResult{root, hotcomments, p.parseErrors, std::move(p.commentLocations)}; } catch (ParseError& err) { @@ -210,8 +206,9 @@ ParseResult Parser::parse(const char* buffer, size_t bufferSize, AstNameTable& n } } -Parser::Parser(const char* buffer, size_t bufferSize, AstNameTable& names, Allocator& allocator) - : lexer(buffer, bufferSize, names) +Parser::Parser(const char* buffer, size_t bufferSize, AstNameTable& names, Allocator& allocator, const ParseOptions& options) + : options(options) + , lexer(buffer, bufferSize, names) , allocator(allocator) , recursionCounter(0) , endMismatchSuspect(Location(), Lexeme::Eof) @@ -224,14 +221,20 @@ Parser::Parser(const char* buffer, size_t bufferSize, AstNameTable& names, Alloc nameSelf = names.addStatic("self"); nameNumber = names.addStatic("number"); - nameError = names.addStatic(errorName); + nameError = names.addStatic(kParseNameError); nameNil = names.getOrAdd("nil"); // nil is a reserved keyword matchRecoveryStopOnToken.assign(Lexeme::Type::Reserved_END, 0); matchRecoveryStopOnToken[Lexeme::Type::Eof] = 1; + if (FFlag::LuauParseAllHotComments) + lexer.setSkipComments(true); + // read first lexeme nextLexeme(); + + // all hot comments parsed after the first non-comment lexeme are special in that they don't affect type checking / linting mode + hotcommentHeader = false; } bool Parser::blockFollow(const Lexeme& l) @@ -396,7 +399,9 @@ AstStat* Parser::parseIf() AstExpr* cond = parseExpr(); Lexeme matchThen = lexer.current(); - bool hasThen = expectAndConsume(Lexeme::ReservedThen, "if statement"); + std::optional thenLocation; + if (expectAndConsume(Lexeme::ReservedThen, "if statement")) + thenLocation = matchThen.location; AstStatBlock* thenbody = parseBlock(); @@ -434,7 +439,7 @@ AstStat* Parser::parseIf() hasEnd = expectMatchEndAndConsume(Lexeme::ReservedEnd, matchThenElse); } - return allocator.alloc(Location(start, end), cond, thenbody, elsebody, hasThen, matchThen.location, elseLocation, hasEnd); + return allocator.alloc(Location(start, end), cond, thenbody, elsebody, thenLocation, elseLocation, hasEnd); } // while exp do block end @@ -769,7 +774,7 @@ AstStat* Parser::parseTypeAlias(const Location& start, bool exported) { // note: `type` token is already parsed for us, so we just need to parse the rest - auto name = parseNameOpt("type name"); + std::optional name = parseNameOpt("type name"); // Use error name if the name is missing if (!name) @@ -925,7 +930,7 @@ AstStat* Parser::parseDeclaration(const Location& start) return allocator.alloc(Location(classStart, classEnd), className.name, superName, copy(props)); } - else if (auto globalName = parseNameOpt("global variable name")) + else if (std::optional globalName = parseNameOpt("global variable name")) { expectAndConsume(':', "global variable declaration"); @@ -1066,7 +1071,7 @@ void Parser::parseExprList(TempVector& result) Parser::Binding Parser::parseBinding() { - auto name = parseNameOpt("variable name"); + std::optional name = parseNameOpt("variable name"); // Use placeholder if the name is missing if (!name) @@ -1325,7 +1330,7 @@ AstType* Parser::parseTableTypeAnnotation() } else { - auto name = parseNameOpt("table field"); + std::optional name = parseNameOpt("table field"); if (!name) break; @@ -1422,7 +1427,7 @@ AstType* Parser::parseFunctionTypeAnnotationTail(const Lexeme& begin, AstArray(Location(begin.location, endLocation), generics, genericPacks, paramTypes, paramNames, returnTypeList); @@ -1869,7 +1874,7 @@ AstExpr* Parser::parseExpr(unsigned int limit) // NAME AstExpr* Parser::parseNameExpr(const char* context) { - auto name = parseNameOpt(context); + std::optional name = parseNameOpt(context); if (!name) return allocator.alloc(lexer.current().location, copy({}), unsigned(parseErrors.size() - 1)); @@ -2233,6 +2238,12 @@ AstExpr* Parser::parseTableConstructor() AstExpr* key = allocator.alloc(name.location, nameString); AstExpr* value = parseExpr(); + if (FFlag::LuauTableFieldFunctionDebugname) + { + if (AstExprFunction* func = value->as()) + func->debugname = name.name; + } + items.push_back({AstExprTable::Item::Record, key, value}); } else @@ -2313,7 +2324,7 @@ std::optional Parser::parseNameOpt(const char* context) Parser::Name Parser::parseName(const char* context) { - if (auto name = parseNameOpt(context)) + if (std::optional name = parseNameOpt(context)) return *name; Location location = lexer.current().location; @@ -2324,7 +2335,7 @@ Parser::Name Parser::parseName(const char* context) Parser::Name Parser::parseIndexName(const char* context, const Position& previous) { - if (auto name = parseNameOpt(context)) + if (std::optional name = parseNameOpt(context)) return *name; // If we have a reserved keyword next at the same line, assume it's an incomplete name @@ -2379,7 +2390,7 @@ std::pair, AstArray> Parser::parseG if (shouldParseTypePackAnnotation(lexer)) { - auto typePack = parseTypePackAnnotation(); + AstTypePack* typePack = parseTypePackAnnotation(); namePacks.push_back({name, nameLocation, typePack}); } @@ -2451,7 +2462,7 @@ AstArray Parser::parseTypeParams() { if (shouldParseTypePackAnnotation(lexer)) { - auto typePack = parseTypePackAnnotation(); + AstTypePack* typePack = parseTypePackAnnotation(); parameters.push_back({{}, typePack}); } @@ -2821,25 +2832,57 @@ AstTypeError* Parser::reportTypeAnnotationError(const Location& location, const return allocator.alloc(location, types, isMissing, unsigned(parseErrors.size() - 1)); } -const Lexeme& Parser::nextLexeme() +void Parser::nextLexeme() { if (options.captureComments) { - while (true) + if (FFlag::LuauParseAllHotComments) { - const Lexeme& lexeme = lexer.next(/*skipComments*/ false); - // Subtlety: Broken comments are weird because we record them as comments AND pass them to the parser as a lexeme. - // The parser will turn this into a proper syntax error. - if (lexeme.type == Lexeme::BrokenComment) + Lexeme::Type type = lexer.next(/* skipComments= */ false).type; + + while (type == Lexeme::BrokenComment || type == Lexeme::Comment || type == Lexeme::BlockComment) + { + const Lexeme& lexeme = lexer.current(); commentLocations.push_back(Comment{lexeme.type, lexeme.location}); - if (isComment(lexeme)) - commentLocations.push_back(Comment{lexeme.type, lexeme.location}); - else - return lexeme; + + // Subtlety: Broken comments are weird because we record them as comments AND pass them to the parser as a lexeme. + // The parser will turn this into a proper syntax error. + if (lexeme.type == Lexeme::BrokenComment) + return; + + // Comments starting with ! are called "hot comments" and contain directives for type checking / linting + if (lexeme.type == Lexeme::Comment && lexeme.length && lexeme.data[0] == '!') + { + const char* text = lexeme.data; + + unsigned int end = lexeme.length; + while (end > 0 && isSpace(text[end - 1])) + --end; + + hotcomments.push_back({hotcommentHeader, lexeme.location, std::string(text + 1, text + end)}); + } + + type = lexer.next(/* skipComments= */ false).type; + } + } + else + { + while (true) + { + const Lexeme& lexeme = lexer.next(/*skipComments*/ false); + // Subtlety: Broken comments are weird because we record them as comments AND pass them to the parser as a lexeme. + // The parser will turn this into a proper syntax error. + if (lexeme.type == Lexeme::BrokenComment) + commentLocations.push_back(Comment{lexeme.type, lexeme.location}); + if (isComment(lexeme)) + commentLocations.push_back(Comment{lexeme.type, lexeme.location}); + else + return; + } } } else - return lexer.next(); + lexer.next(); } } // namespace Luau diff --git a/Ast/src/TimeTrace.cpp b/Ast/src/TimeTrace.cpp index ded50e53..8079830b 100644 --- a/Ast/src/TimeTrace.cpp +++ b/Ast/src/TimeTrace.cpp @@ -9,6 +9,12 @@ #include #ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include #endif diff --git a/CLI/FileUtils.cpp b/CLI/FileUtils.cpp index fe005aec..fb6ac373 100644 --- a/CLI/FileUtils.cpp +++ b/CLI/FileUtils.cpp @@ -4,8 +4,13 @@ #include "Luau/Common.h" #ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN -#include +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include #else #include #include diff --git a/CLI/Repl.cpp b/CLI/Repl.cpp index 9a6e25c2..13304d57 100644 --- a/CLI/Repl.cpp +++ b/CLI/Repl.cpp @@ -37,6 +37,8 @@ enum class CompileFormat Binary }; +constexpr int MaxTraversalLimit = 50; + struct GlobalOptions { int optimizationLevel = 1; @@ -243,10 +245,115 @@ std::string runCode(lua_State* L, const std::string& source) return std::string(); } +// Replaces the top of the lua stack with the metatable __index for the value +// if it exists. Returns true iff __index exists. +static bool tryReplaceTopWithIndex(lua_State* L) +{ + if (luaL_getmetafield(L, -1, "__index")) + { + // Remove the table leaving __index on the top of stack + lua_remove(L, -2); + return true; + } + return false; +} + + +// This function is similar to lua_gettable, but it avoids calling any +// lua callback functions (e.g. __index) which might modify the Lua VM state. +static void safeGetTable(lua_State* L, int tableIndex) +{ + lua_pushvalue(L, tableIndex); // Duplicate the table + + // The loop invariant is that the table to search is at -1 + // and the key is at -2. + for (int loopCount = 0;; loopCount++) + { + lua_pushvalue(L, -2); // Duplicate the key + lua_rawget(L, -2); // Try to find the key + if (!lua_isnil(L, -1) || loopCount >= MaxTraversalLimit) + { + // Either the key has been found, and/or we have reached the max traversal limit + break; + } + else + { + lua_pop(L, 1); // Pop the nil result + if (!luaL_getmetafield(L, -1, "__index")) + { + lua_pushnil(L); + break; + } + else if (lua_istable(L, -1)) + { + // Replace the current table being searched with __index table + lua_replace(L, -2); + } + else + { + lua_pop(L, 1); // Pop the value + lua_pushnil(L); + break; + } + } + } + + lua_remove(L, -2); // Remove the table + lua_remove(L, -2); // Remove the original key +} + +// completePartialMatches finds keys that match the specified 'prefix' +// Note: the table/object to be searched must be on the top of the Lua stack +static void completePartialMatches(lua_State* L, bool completeOnlyFunctions, const std::string& editBuffer, std::string_view prefix, + const AddCompletionCallback& addCompletionCallback) +{ + for (int i = 0; i < MaxTraversalLimit && lua_istable(L, -1); i++) + { + // table, key + lua_pushnil(L); + + // Loop over all the keys in the current table + while (lua_next(L, -2) != 0) + { + if (lua_type(L, -2) == LUA_TSTRING) + { + // table, key, value + std::string_view key = lua_tostring(L, -2); + int valueType = lua_type(L, -1); + + // If the last separator was a ':' (i.e. a method call) then only functions should be completed. + bool requiredValueType = (!completeOnlyFunctions || valueType == LUA_TFUNCTION); + + if (!key.empty() && requiredValueType && Luau::startsWith(key, prefix)) + { + std::string completedComponent(key.substr(prefix.size())); + std::string completion(editBuffer + completedComponent); + if (valueType == LUA_TFUNCTION) + { + // Add an opening paren for function calls by default. + completion += "("; + } + addCompletionCallback(completion, std::string(key)); + } + } + lua_pop(L, 1); + } + + // Replace the current table being searched with an __index table if one exists + if (!tryReplaceTopWithIndex(L)) + { + break; + } + } +} + static void completeIndexer(lua_State* L, const std::string& editBuffer, const AddCompletionCallback& addCompletionCallback) { std::string_view lookup = editBuffer; - char lastSep = 0; + bool completeOnlyFunctions = false; + + // Push the global variable table to begin the search + lua_pushvalue(L, LUA_GLOBALSINDEX); for (;;) { @@ -255,60 +362,26 @@ static void completeIndexer(lua_State* L, const std::string& editBuffer, const A if (sep == std::string_view::npos) { - // table, key - lua_pushnil(L); - - while (lua_next(L, -2) != 0) - { - if (lua_type(L, -2) == LUA_TSTRING) - { - // table, key, value - std::string_view key = lua_tostring(L, -2); - int valueType = lua_type(L, -1); - - // If the last separator was a ':' (i.e. a method call) then only functions should be completed. - bool requiredValueType = (lastSep != ':' || valueType == LUA_TFUNCTION); - - if (!key.empty() && requiredValueType && Luau::startsWith(key, prefix)) - { - std::string completedComponent(key.substr(prefix.size())); - std::string completion(editBuffer + completedComponent); - if (valueType == LUA_TFUNCTION) - { - // Add an opening paren for function calls by default. - completion += "("; - } - addCompletionCallback(completion, std::string(key)); - } - } - lua_pop(L, 1); - } - + completePartialMatches(L, completeOnlyFunctions, editBuffer, prefix, addCompletionCallback); break; } else { // find the key in the table lua_pushlstring(L, prefix.data(), prefix.size()); - lua_rawget(L, -2); + safeGetTable(L, -2); lua_remove(L, -2); - if (lua_type(L, -1) == LUA_TSTRING) + if (lua_istable(L, -1) || tryReplaceTopWithIndex(L)) { - // Replace the string object with the string class to perform further lookups of string functions - // Note: We retrieve the string class from _G to prevent issues if the user assigns to `string`. - lua_pop(L, 1); // Pop the string instance - lua_getglobal(L, "_G"); - lua_pushlstring(L, "string", 6); - lua_rawget(L, -2); - lua_remove(L, -2); // Remove the global table - LUAU_ASSERT(lua_istable(L, -1)); + completeOnlyFunctions = lookup[sep] == ':'; + lookup.remove_prefix(sep + 1); } - else if (!lua_istable(L, -1)) + else + { + // Unable to search for keys, so stop searching break; - - lastSep = lookup[sep]; - lookup.remove_prefix(sep + 1); + } } } @@ -317,12 +390,6 @@ static void completeIndexer(lua_State* L, const std::string& editBuffer, const A void getCompletions(lua_State* L, const std::string& editBuffer, const AddCompletionCallback& addCompletionCallback) { - // look the value up in current global table first - lua_pushvalue(L, LUA_GLOBALSINDEX); - completeIndexer(L, editBuffer, addCompletionCallback); - - // and in actual global table after that - lua_getglobal(L, "_G"); completeIndexer(L, editBuffer, addCompletionCallback); } @@ -365,9 +432,7 @@ struct LinenoiseScopedHistory ic_set_history(historyFilepath.c_str(), -1 /* default entries (= 200) */); } - ~LinenoiseScopedHistory() - { - } + ~LinenoiseScopedHistory() {} std::string historyFilepath; }; diff --git a/CMakeLists.txt b/CMakeLists.txt index c19d2b40..c6ccebc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,6 +104,20 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1924) set_source_files_properties(VM/src/lvmexecute.cpp PROPERTIES COMPILE_FLAGS /d2ssa-pre-) endif() +# embed .natvis inside the library debug information +if(MSVC) + target_link_options(Luau.Ast INTERFACE /NATVIS:${CMAKE_CURRENT_SOURCE_DIR}/tools/natvis/Ast.natvis) + target_link_options(Luau.Analysis INTERFACE /NATVIS:${CMAKE_CURRENT_SOURCE_DIR}/tools/natvis/Analysis.natvis) + target_link_options(Luau.VM INTERFACE /NATVIS:${CMAKE_CURRENT_SOURCE_DIR}/tools/natvis/VM.natvis) +endif() + +# make .natvis visible inside the solution +if(MSVC_IDE) + target_sources(Luau.Ast PRIVATE tools/natvis/Ast.natvis) + target_sources(Luau.Analysis PRIVATE tools/natvis/Analysis.natvis) + target_sources(Luau.VM PRIVATE tools/natvis/VM.natvis) +endif() + if(LUAU_BUILD_CLI) target_compile_options(Luau.Repl.CLI PRIVATE ${LUAU_OPTIONS}) target_compile_options(Luau.Analyze.CLI PRIVATE ${LUAU_OPTIONS}) diff --git a/Sources.cmake b/Sources.cmake index 773f6f35..615641eb 100644 --- a/Sources.cmake +++ b/Sources.cmake @@ -8,6 +8,7 @@ target_sources(Luau.Ast PRIVATE Ast/include/Luau/Location.h Ast/include/Luau/ParseOptions.h Ast/include/Luau/Parser.h + Ast/include/Luau/ParseResult.h Ast/include/Luau/StringUtils.h Ast/include/Luau/TimeTrace.h diff --git a/VM/src/lapi.cpp b/VM/src/lapi.cpp index 5cffba63..29d5f397 100644 --- a/VM/src/lapi.cpp +++ b/VM/src/lapi.cpp @@ -36,12 +36,9 @@ const char* luau_ident = "$Luau: Copyright (C) 2019-2022 Roblox Corporation $\n" static Table* getcurrenv(lua_State* L) { if (L->ci == L->base_ci) /* no enclosing function? */ - return hvalue(gt(L)); /* use global table as environment */ + return L->gt; /* use global table as environment */ else - { - Closure* func = curr_func(L); - return func->env; - } + return curr_func(L)->env; } static LUAU_NOINLINE TValue* pseudo2addr(lua_State* L, int idx) @@ -53,11 +50,14 @@ static LUAU_NOINLINE TValue* pseudo2addr(lua_State* L, int idx) return registry(L); case LUA_ENVIRONINDEX: { - sethvalue(L, &L->env, getcurrenv(L)); - return &L->env; + sethvalue(L, &L->global->pseudotemp, getcurrenv(L)); + return &L->global->pseudotemp; } case LUA_GLOBALSINDEX: - return gt(L); + { + sethvalue(L, &L->global->pseudotemp, L->gt); + return &L->global->pseudotemp; + } default: { Closure* func = curr_func(L); @@ -237,6 +237,11 @@ void lua_replace(lua_State* L, int idx) func->env = hvalue(L->top - 1); luaC_barrier(L, func, L->top - 1); } + else if (idx == LUA_GLOBALSINDEX) + { + api_check(L, ttistable(L->top - 1)); + L->gt = hvalue(L->top - 1); + } else { setobj(L, o, L->top - 1); @@ -783,7 +788,7 @@ void lua_getfenv(lua_State* L, int idx) sethvalue(L, L->top, clvalue(o)->env); break; case LUA_TTHREAD: - setobj2s(L, L->top, gt(thvalue(o))); + sethvalue(L, L->top, thvalue(o)->gt); break; default: setnilvalue(L->top); @@ -914,7 +919,7 @@ int lua_setfenv(lua_State* L, int idx) clvalue(o)->env = hvalue(L->top - 1); break; case LUA_TTHREAD: - sethvalue(L, gt(thvalue(o)), hvalue(L->top - 1)); + thvalue(o)->gt = hvalue(L->top - 1); break; default: res = 0; diff --git a/VM/src/ldebug.cpp b/VM/src/ldebug.cpp index e9930f7a..a4f93c62 100644 --- a/VM/src/ldebug.cpp +++ b/VM/src/ldebug.cpp @@ -419,7 +419,7 @@ static void getcoverage(Proto* p, int depth, int* buffer, size_t size, void* con } const char* debugname = p->debugname ? getstr(p->debugname) : NULL; - int linedefined = luaG_getline(p, 0); + int linedefined = getlinedefined(p); callback(context, debugname, linedefined, depth, buffer, size); diff --git a/VM/src/ldo.cpp b/VM/src/ldo.cpp index a3982bc6..d87f0661 100644 --- a/VM/src/ldo.cpp +++ b/VM/src/ldo.cpp @@ -17,6 +17,8 @@ #include +LUAU_FASTFLAG(LuauReduceStackReallocs) + /* ** {====================================================== ** Error-recovery functions @@ -164,13 +166,14 @@ static void correctstack(lua_State* L, TValue* oldstack) void luaD_reallocstack(lua_State* L, int newsize) { TValue* oldstack = L->stack; - int realsize = newsize + 1 + EXTRA_STACK; - LUAU_ASSERT(L->stack_last - L->stack == L->stacksize - EXTRA_STACK - 1); + int realsize = newsize + (FFlag::LuauReduceStackReallocs ? EXTRA_STACK : 1 + EXTRA_STACK); + LUAU_ASSERT(L->stack_last - L->stack == L->stacksize - (FFlag::LuauReduceStackReallocs ? EXTRA_STACK : 1 + EXTRA_STACK)); luaM_reallocarray(L, L->stack, L->stacksize, realsize, TValue, L->memcat); + TValue* newstack = L->stack; for (int i = L->stacksize; i < realsize; i++) - setnilvalue(L->stack + i); /* erase new segment */ + setnilvalue(newstack + i); /* erase new segment */ L->stacksize = realsize; - L->stack_last = L->stack + newsize; + L->stack_last = newstack + newsize; correctstack(L, oldstack); } @@ -512,7 +515,7 @@ static void callerrfunc(lua_State* L, void* ud) static void restore_stack_limit(lua_State* L) { - LUAU_ASSERT(L->stack_last - L->stack == L->stacksize - EXTRA_STACK - 1); + LUAU_ASSERT(L->stack_last - L->stack == L->stacksize - (FFlag::LuauReduceStackReallocs ? EXTRA_STACK : 1 + EXTRA_STACK)); if (L->size_ci > LUAI_MAXCALLS) { /* there was an overflow? */ int inuse = cast_int(L->ci - L->base_ci); diff --git a/VM/src/ldo.h b/VM/src/ldo.h index 72807f0f..1c1480d6 100644 --- a/VM/src/ldo.h +++ b/VM/src/ldo.h @@ -11,7 +11,7 @@ if ((char*)L->stack_last - (char*)L->top <= (n) * (int)sizeof(TValue)) \ luaD_growstack(L, n); \ else \ - condhardstacktests(luaD_reallocstack(L, L->stacksize - EXTRA_STACK - 1)); + condhardstacktests(luaD_reallocstack(L, L->stacksize - (FFlag::LuauReduceStackReallocs ? EXTRA_STACK : 1 + EXTRA_STACK))); #define incr_top(L) \ { \ diff --git a/VM/src/lgc.cpp b/VM/src/lgc.cpp index 835572fa..724b24b2 100644 --- a/VM/src/lgc.cpp +++ b/VM/src/lgc.cpp @@ -268,7 +268,7 @@ static void traverseclosure(global_State* g, Closure* cl) static void traversestack(global_State* g, lua_State* l, bool clearstack) { - markvalue(g, gt(l)); + markobject(g, l->gt); if (l->namecall) stringmark(l->namecall); for (StkId o = l->stack; o < l->top; o++) @@ -643,7 +643,7 @@ static void markroot(lua_State* L) g->weak = NULL; markobject(g, g->mainthread); /* make global table be traversed before main stack */ - markvalue(g, gt(g->mainthread)); + markobject(g, g->mainthread->gt); markvalue(g, registry(L)); markmt(g); g->gcstate = GCSpropagate; diff --git a/VM/src/lgc.h b/VM/src/lgc.h index 528d0944..2acb5d8a 100644 --- a/VM/src/lgc.h +++ b/VM/src/lgc.h @@ -77,7 +77,7 @@ #define luaC_checkGC(L) \ { \ - condhardstacktests(luaD_reallocstack(L, L->stacksize - EXTRA_STACK - 1)); \ + condhardstacktests(luaD_reallocstack(L, L->stacksize - (FFlag::LuauReduceStackReallocs ? EXTRA_STACK : 1 + EXTRA_STACK))); \ if (L->global->totalbytes >= L->global->GCthreshold) \ { \ condhardmemtests(luaC_validate(L), 1); \ diff --git a/VM/src/lgcdebug.cpp b/VM/src/lgcdebug.cpp index ce196520..30242e52 100644 --- a/VM/src/lgcdebug.cpp +++ b/VM/src/lgcdebug.cpp @@ -88,7 +88,7 @@ static void validateclosure(global_State* g, Closure* cl) static void validatestack(global_State* g, lua_State* l) { - validateref(g, obj2gco(l), gt(l)); + validateobjref(g, obj2gco(l), obj2gco(l->gt)); for (CallInfo* ci = l->base_ci; ci <= l->ci; ++ci) { @@ -370,6 +370,7 @@ static void dumpclosure(FILE* f, Closure* cl) fprintf(f, ",\"env\":"); dumpref(f, obj2gco(cl->env)); + if (cl->isC) { if (cl->nupvalues) @@ -411,11 +412,8 @@ static void dumpthread(FILE* f, lua_State* th) fprintf(f, "{\"type\":\"thread\",\"cat\":%d,\"size\":%d", th->memcat, int(size)); - if (iscollectable(&th->l_gt)) - { - fprintf(f, ",\"env\":"); - dumpref(f, gcvalue(&th->l_gt)); - } + fprintf(f, ",\"env\":"); + dumpref(f, obj2gco(th->gt)); Closure* tcl = 0; for (CallInfo* ci = th->base_ci; ci <= th->ci; ++ci) diff --git a/VM/src/lperf.cpp b/VM/src/lperf.cpp index 2f6c7297..da68e376 100644 --- a/VM/src/lperf.cpp +++ b/VM/src/lperf.cpp @@ -3,6 +3,12 @@ #include "lua.h" #ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include #endif diff --git a/VM/src/lstate.cpp b/VM/src/lstate.cpp index 6762c638..d6d127c0 100644 --- a/VM/src/lstate.cpp +++ b/VM/src/lstate.cpp @@ -11,6 +11,7 @@ #include "ldebug.h" LUAU_FASTFLAG(LuauGcPagedSweep) +LUAU_FASTFLAGVARIABLE(LuauReduceStackReallocs, false) /* ** Main thread combines a thread state and the global state @@ -31,10 +32,11 @@ static void stack_init(lua_State* L1, lua_State* L) /* initialize stack array */ L1->stack = luaM_newarray(L, BASIC_STACK_SIZE + EXTRA_STACK, TValue, L1->memcat); L1->stacksize = BASIC_STACK_SIZE + EXTRA_STACK; + TValue* stack = L1->stack; for (int i = 0; i < BASIC_STACK_SIZE + EXTRA_STACK; i++) - setnilvalue(L1->stack + i); /* erase new stack */ - L1->top = L1->stack; - L1->stack_last = L1->stack + (L1->stacksize - EXTRA_STACK) - 1; + setnilvalue(stack + i); /* erase new stack */ + L1->top = stack; + L1->stack_last = stack + (L1->stacksize - (FFlag::LuauReduceStackReallocs ? EXTRA_STACK : 1 + EXTRA_STACK)); /* initialize first ci */ L1->ci->func = L1->top; setnilvalue(L1->top++); /* `function' entry for this `ci' */ @@ -55,7 +57,7 @@ static void f_luaopen(lua_State* L, void* ud) { global_State* g = L->global; stack_init(L, L); /* init stack */ - sethvalue(L, gt(L), luaH_new(L, 0, 2)); /* table of globals */ + L->gt = luaH_new(L, 0, 2); /* table of globals */ sethvalue(L, registry(L), luaH_new(L, 0, 2)); /* registry */ luaS_resize(L, LUA_MINSTRTABSIZE); /* initial size of string table */ luaT_init(L); @@ -69,6 +71,7 @@ static void preinit_state(lua_State* L, global_State* g) L->global = g; L->stack = NULL; L->stacksize = 0; + L->gt = NULL; L->openupval = NULL; L->size_ci = 0; L->nCcalls = L->baseCcalls = 0; @@ -80,7 +83,6 @@ static void preinit_state(lua_State* L, global_State* g) L->stackstate = 0; L->activememcat = 0; L->userdata = NULL; - setnilvalue(gt(L)); } static void close_state(lua_State* L) @@ -116,7 +118,7 @@ lua_State* luaE_newthread(lua_State* L) preinit_state(L1, L->global); L1->activememcat = L->activememcat; // inherit the active memory category stack_init(L1, L); /* init stack */ - setobj2n(L, gt(L1), gt(L)); /* share table of globals */ + L1->gt = L->gt; /* share table of globals */ L1->singlestep = L->singlestep; LUAU_ASSERT(iswhite(obj2gco(L1))); return L1; @@ -144,14 +146,30 @@ void lua_resetthread(lua_State* L) ci->top = ci->base + LUA_MINSTACK; setnilvalue(ci->func); L->ci = ci; - luaD_reallocCI(L, BASIC_CI_SIZE); + if (FFlag::LuauReduceStackReallocs) + { + if (L->size_ci != BASIC_CI_SIZE) + luaD_reallocCI(L, BASIC_CI_SIZE); + } + else + { + luaD_reallocCI(L, BASIC_CI_SIZE); + } /* clear thread state */ L->status = LUA_OK; L->base = L->ci->base; L->top = L->ci->base; L->nCcalls = L->baseCcalls = 0; /* clear thread stack */ - luaD_reallocstack(L, BASIC_STACK_SIZE); + if (FFlag::LuauReduceStackReallocs) + { + if (L->stacksize != BASIC_STACK_SIZE + EXTRA_STACK) + luaD_reallocstack(L, BASIC_STACK_SIZE); + } + else + { + luaD_reallocstack(L, BASIC_STACK_SIZE); + } for (int i = 0; i < L->stacksize; i++) setnilvalue(L->stack + i); } @@ -193,6 +211,7 @@ lua_State* lua_newstate(lua_Alloc f, void* ud) g->strt.size = 0; g->strt.nuse = 0; g->strt.hash = NULL; + setnilvalue(&g->pseudotemp); setnilvalue(registry(L)); g->gcstate = GCSpause; if (!FFlag::LuauGcPagedSweep) diff --git a/VM/src/lstate.h b/VM/src/lstate.h index 0708b71f..6dd89138 100644 --- a/VM/src/lstate.h +++ b/VM/src/lstate.h @@ -5,9 +5,6 @@ #include "lobject.h" #include "ltm.h" -/* table of globals */ -#define gt(L) (&L->l_gt) - /* registry */ #define registry(L) (&L->global->registry) @@ -177,6 +174,8 @@ typedef struct global_State TString* ttname[LUA_T_COUNT]; /* names for basic types */ TString* tmname[TM_N]; /* array with tag-method names */ + TValue pseudotemp; /* storage for temporary values used in pseudo2addr */ + TValue registry; /* registry table, used by lua_ref and LUA_REGISTRYINDEX */ int registryfree; /* next free slot in registry */ @@ -231,8 +230,7 @@ struct lua_State int cachedslot; /* when table operations or INDEX/NEWINDEX is invoked from Luau, what is the expected slot for lookup? */ - TValue l_gt; /* table of globals */ - TValue env; /* temporary place for environments */ + Table* gt; /* table of globals */ UpVal* openupval; /* list of open upvalues in this stack */ GCObject* gclist; diff --git a/VM/src/lvmexecute.cpp b/VM/src/lvmexecute.cpp index c3b662a2..6c31d36f 100644 --- a/VM/src/lvmexecute.cpp +++ b/VM/src/lvmexecute.cpp @@ -39,7 +39,7 @@ // When calling luau_callTM, we usually push the arguments to the top of the stack. // This is safe to do for complicated reasons: -// - stack guarantees 1 + EXTRA_STACK room beyond stack_last (see luaD_reallocstack) +// - stack guarantees EXTRA_STACK room beyond stack_last (see luaD_reallocstack) // - stack reallocation copies values past stack_last // All external function calls that can cause stack realloc or Lua calls have to be wrapped in VM_PROTECT diff --git a/VM/src/lvmload.cpp b/VM/src/lvmload.cpp index 2472cd90..4e5435b7 100644 --- a/VM/src/lvmload.cpp +++ b/VM/src/lvmload.cpp @@ -116,12 +116,12 @@ static void resolveImportSafe(lua_State* L, Table* env, TValue* k, uint32_t id) // note: we call getimport with nil propagation which means that accesses to table chains like A.B.C will resolve in nil // this is technically not necessary but it reduces the number of exceptions when loading scripts that rely on getfenv/setfenv for global // injection - luaV_getimport(L, hvalue(gt(L)), self->k, self->id, /* propagatenil= */ true); + luaV_getimport(L, L->gt, self->k, self->id, /* propagatenil= */ true); } }; ResolveImport ri = {k, id}; - if (hvalue(gt(L))->safeenv) + if (L->gt->safeenv) { // luaD_pcall will make sure that if any C/Lua calls during import resolution fail, the thread state is restored back int oldTop = lua_gettop(L); @@ -171,7 +171,7 @@ int luau_load(lua_State* L, const char* chunkname, const char* data, size_t size L->global->GCthreshold = SIZE_MAX; // env is 0 for current environment and a stack index otherwise - Table* envt = (env == 0) ? hvalue(gt(L)) : hvalue(luaA_toobject(L, env)); + Table* envt = (env == 0) ? L->gt : hvalue(luaA_toobject(L, env)); TString* source = luaS_new(L, chunkname); diff --git a/VM/src/lvmutils.cpp b/VM/src/lvmutils.cpp index 31dd59c8..8a18a4d4 100644 --- a/VM/src/lvmutils.cpp +++ b/VM/src/lvmutils.cpp @@ -55,7 +55,7 @@ static void callTMres(lua_State* L, StkId res, const TValue* f, const TValue* p1 { ptrdiff_t result = savestack(L, res); // using stack room beyond top is technically safe here, but for very complicated reasons: - // * The stack guarantees 1 + EXTRA_STACK room beyond stack_last (see luaD_reallocstack) will be allocated + // * The stack guarantees EXTRA_STACK room beyond stack_last (see luaD_reallocstack) will be allocated // * we cannot move luaD_checkstack above because the arguments are *sometimes* pointers to the lua // stack and checkstack may invalidate those pointers // * we cannot use savestack/restorestack because the arguments are sometimes on the C++ stack @@ -76,7 +76,7 @@ static void callTMres(lua_State* L, StkId res, const TValue* f, const TValue* p1 static void callTM(lua_State* L, const TValue* f, const TValue* p1, const TValue* p2, const TValue* p3) { // using stack room beyond top is technically safe here, but for very complicated reasons: - // * The stack guarantees 1 + EXTRA_STACK room beyond stack_last (see luaD_reallocstack) will be allocated + // * The stack guarantees EXTRA_STACK room beyond stack_last (see luaD_reallocstack) will be allocated // * we cannot move luaD_checkstack above because the arguments are *sometimes* pointers to the lua // stack and checkstack may invalidate those pointers // * we cannot use savestack/restorestack because the arguments are sometimes on the C++ stack diff --git a/fuzz/linter.cpp b/fuzz/linter.cpp index 55e0888b..04638d23 100644 --- a/fuzz/linter.cpp +++ b/fuzz/linter.cpp @@ -32,7 +32,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) Luau::LintOptions lintOptions; lintOptions.warningMask = ~0ull; - Luau::lint(parseResult.root, names, typeck.globalScope, nullptr, lintOptions); + Luau::lint(parseResult.root, names, typeck.globalScope, nullptr, {}, lintOptions); } return 0; diff --git a/fuzz/proto.cpp b/fuzz/proto.cpp index 27e53492..f407248a 100644 --- a/fuzz/proto.cpp +++ b/fuzz/proto.cpp @@ -227,7 +227,7 @@ DEFINE_PROTO_FUZZER(const luau::StatBlock& message) if (kFuzzLinter) { Luau::LintOptions lintOptions = {~0u}; - Luau::lint(parseResult.root, names, sharedEnv.globalScope, module.get(), lintOptions); + Luau::lint(parseResult.root, names, sharedEnv.globalScope, module.get(), {}, lintOptions); } } diff --git a/tests/Autocomplete.test.cpp b/tests/Autocomplete.test.cpp index 59e12574..1978a0d3 100644 --- a/tests/Autocomplete.test.cpp +++ b/tests/Autocomplete.test.cpp @@ -1,7 +1,6 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/Autocomplete.h" #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" #include "Luau/VisitTypeVar.h" @@ -2610,6 +2609,27 @@ a = if temp then even elseif true then temp else e@9 CHECK(ac.entryMap.count("elseif") == 0); } +TEST_CASE_FIXTURE(ACFixture, "autocomplete_if_else_regression") +{ + ScopedFastFlag FFlagLuauIfElseExprFixCompletionIssue("LuauIfElseExprFixCompletionIssue", true); + check(R"( +local abcdef = 0; +local temp = false +local even = true; +local a +a = if temp then even else@1 +a = if temp then even else @2 +a = if temp then even else abc@3 + )"); + + auto ac = autocomplete('1'); + CHECK(ac.entryMap.count("else") == 0); + ac = autocomplete('2'); + CHECK(ac.entryMap.count("else") == 0); + ac = autocomplete('3'); + CHECK(ac.entryMap.count("abcdef")); +} + TEST_CASE_FIXTURE(ACFixture, "autocomplete_explicit_type_pack") { check(R"( diff --git a/tests/Compiler.test.cpp b/tests/Compiler.test.cpp index cd7a21d8..f982c86f 100644 --- a/tests/Compiler.test.cpp +++ b/tests/Compiler.test.cpp @@ -10,6 +10,11 @@ #include #include +namespace Luau +{ +std::string rep(const std::string& s, size_t n); +} + using namespace Luau; static std::string compileFunction(const char* source, uint32_t id) @@ -1960,15 +1965,6 @@ RETURN R8 -1 )"); } -static std::string rep(const std::string& s, size_t n) -{ - std::string r; - r.reserve(s.length() * n); - for (size_t i = 0; i < n; ++i) - r += s; - return r; -} - TEST_CASE("RecursionParse") { // The test forcibly pushes the stack limit during compilation; in NoOpt, the stack consumption is much larger so we need to reduce the limit to diff --git a/tests/Conformance.test.cpp b/tests/Conformance.test.cpp index 8b58d2ce..b09c1efb 100644 --- a/tests/Conformance.test.cpp +++ b/tests/Conformance.test.cpp @@ -492,6 +492,8 @@ TEST_CASE("DateTime") TEST_CASE("Debug") { + ScopedFastFlag luauTableFieldFunctionDebugname{"LuauTableFieldFunctionDebugname", true}; + runConformance("debug.lua"); } @@ -890,6 +892,12 @@ TEST_CASE("Coverage") lua_pushstring(L, function); lua_setfield(L, -2, "name"); + lua_pushinteger(L, linedefined); + lua_setfield(L, -2, "linedefined"); + + lua_pushinteger(L, depth); + lua_setfield(L, -2, "depth"); + for (size_t i = 0; i < size; ++i) if (hits[i] != -1) { diff --git a/tests/Fixture.cpp b/tests/Fixture.cpp index c74bfa27..dbdd06a4 100644 --- a/tests/Fixture.cpp +++ b/tests/Fixture.cpp @@ -2,6 +2,7 @@ #include "Fixture.h" #include "Luau/AstQuery.h" +#include "Luau/Parser.h" #include "Luau/TypeVar.h" #include "Luau/TypeAttach.h" #include "Luau/Transpiler.h" @@ -112,7 +113,7 @@ AstStatBlock* Fixture::parse(const std::string& source, const ParseOptions& pars sourceModule->name = fromString(mainModuleName); sourceModule->root = result.root; sourceModule->mode = parseMode(result.hotcomments); - sourceModule->ignoreLints = LintWarning::parseMask(result.hotcomments); + sourceModule->hotcomments = std::move(result.hotcomments); if (!result.errors.empty()) { @@ -157,6 +158,7 @@ CheckResult Fixture::check(const std::string& source) LintResult Fixture::lint(const std::string& source, const std::optional& lintOptions) { ParseOptions parseOptions; + parseOptions.captureComments = true; configResolver.defaultConfig.mode = Mode::Nonstrict; parse(source, parseOptions); diff --git a/tests/Fixture.h b/tests/Fixture.h index ab852ef6..4e45a952 100644 --- a/tests/Fixture.h +++ b/tests/Fixture.h @@ -8,7 +8,6 @@ #include "Luau/Linter.h" #include "Luau/Location.h" #include "Luau/ModuleResolver.h" -#include "Luau/Parser.h" #include "Luau/ToString.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/Frontend.test.cpp b/tests/Frontend.test.cpp index ea1a08fe..8a59acd1 100644 --- a/tests/Frontend.test.cpp +++ b/tests/Frontend.test.cpp @@ -2,7 +2,6 @@ #include "Luau/AstQuery.h" #include "Luau/BuiltinDefinitions.h" #include "Luau/Frontend.h" -#include "Luau/Parser.h" #include "Luau/RequireTracer.h" #include "Fixture.h" @@ -897,8 +896,6 @@ TEST_CASE_FIXTURE(FrontendFixture, "clearStats") TEST_CASE_FIXTURE(FrontendFixture, "typecheck_twice_for_ast_types") { - ScopedFastFlag sffs("LuauTypeCheckTwice", true); - fileResolver.source["Module/A"] = R"( local a = 1 )"; diff --git a/tests/Linter.test.cpp b/tests/Linter.test.cpp index 577415fc..d4b97360 100644 --- a/tests/Linter.test.cpp +++ b/tests/Linter.test.cpp @@ -1395,12 +1395,10 @@ TEST_CASE_FIXTURE(Fixture, "DeprecatedApi") TypeId colorType = typeChecker.globalTypes.addType(TableTypeVar{{}, std::nullopt, typeChecker.globalScope->level, Luau::TableState::Sealed}); - getMutable(colorType)->props = { - {"toHSV", {typeChecker.anyType, /* deprecated= */ true, "Color3:ToHSV"} } - }; + getMutable(colorType)->props = {{"toHSV", {typeChecker.anyType, /* deprecated= */ true, "Color3:ToHSV"}}}; addGlobalBinding(typeChecker, "Color3", Binding{colorType, {}}); - + freeze(typeChecker.globalTypes); LintResult result = lintTyped(R"( @@ -1554,8 +1552,46 @@ _ = (math.random() < 0.5 and false) or 42 -- currently ignored )"); REQUIRE_EQ(result.warnings.size(), 2); - CHECK_EQ(result.warnings[0].text, "The and-or expression always evaluates to the second alternative because the first alternative is false; consider using if-then-else expression instead"); - CHECK_EQ(result.warnings[1].text, "The and-or expression always evaluates to the second alternative because the first alternative is nil; consider using if-then-else expression instead"); + CHECK_EQ(result.warnings[0].text, "The and-or expression always evaluates to the second alternative because the first alternative is false; " + "consider using if-then-else expression instead"); + CHECK_EQ(result.warnings[1].text, "The and-or expression always evaluates to the second alternative because the first alternative is nil; " + "consider using if-then-else expression instead"); +} + +TEST_CASE_FIXTURE(Fixture, "WrongComment") +{ + ScopedFastFlag sff("LuauParseAllHotComments", true); + + LintResult result = lint(R"( +--!strict +--!struct +--!nolintGlobal +--!nolint Global +--!nolint KnownGlobal +--!nolint UnknownGlobal +--! no more lint +--!strict here +do end +--!nolint +)"); + + REQUIRE_EQ(result.warnings.size(), 6); + CHECK_EQ(result.warnings[0].text, "Unknown comment directive 'struct'; did you mean 'strict'?"); + CHECK_EQ(result.warnings[1].text, "Unknown comment directive 'nolintGlobal'"); + CHECK_EQ(result.warnings[2].text, "nolint directive refers to unknown lint rule 'Global'"); + CHECK_EQ(result.warnings[3].text, "nolint directive refers to unknown lint rule 'KnownGlobal'; did you mean 'UnknownGlobal'?"); + CHECK_EQ(result.warnings[4].text, "Comment directive with the type checking mode has extra symbols at the end of the line"); + CHECK_EQ(result.warnings[5].text, "Comment directive is ignored because it is placed after the first non-comment token"); +} + +TEST_CASE_FIXTURE(Fixture, "WrongCommentMuteSelf") +{ + LintResult result = lint(R"( +--!nolint +--!struct +)"); + + REQUIRE_EQ(result.warnings.size(), 0); // --!nolint disables WrongComment lint :) } TEST_SUITE_END(); diff --git a/tests/NonstrictMode.test.cpp b/tests/NonstrictMode.test.cpp index 931a8403..5bad9901 100644 --- a/tests/NonstrictMode.test.cpp +++ b/tests/NonstrictMode.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/Scope.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/Parser.test.cpp b/tests/Parser.test.cpp index c1a8887b..0d4c088d 100644 --- a/tests/Parser.test.cpp +++ b/tests/Parser.test.cpp @@ -1,6 +1,5 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/Parser.h" -#include "Luau/TypeInfer.h" #include "Fixture.h" #include "ScopedFlags.h" @@ -300,8 +299,9 @@ TEST_CASE_FIXTURE(Fixture, "functions_can_have_return_annotations") AstStatFunction* statFunction = block->body.data[0]->as(); REQUIRE(statFunction != nullptr); - CHECK_EQ(statFunction->func->returnAnnotation.types.size, 1); - CHECK(statFunction->func->returnAnnotation.tailType == nullptr); + REQUIRE(statFunction->func->returnAnnotation.has_value()); + CHECK_EQ(statFunction->func->returnAnnotation->types.size, 1); + CHECK(statFunction->func->returnAnnotation->tailType == nullptr); } TEST_CASE_FIXTURE(Fixture, "functions_can_have_a_function_type_annotation") @@ -316,9 +316,9 @@ TEST_CASE_FIXTURE(Fixture, "functions_can_have_a_function_type_annotation") AstStatFunction* statFunc = block->body.data[0]->as(); REQUIRE(statFunc != nullptr); - AstArray& retTypes = statFunc->func->returnAnnotation.types; - REQUIRE(statFunc->func->hasReturnAnnotation); - CHECK(statFunc->func->returnAnnotation.tailType == nullptr); + REQUIRE(statFunc->func->returnAnnotation.has_value()); + CHECK(statFunc->func->returnAnnotation->tailType == nullptr); + AstArray& retTypes = statFunc->func->returnAnnotation->types; REQUIRE(retTypes.size == 1); AstTypeFunction* funTy = retTypes.data[0]->as(); @@ -337,9 +337,9 @@ TEST_CASE_FIXTURE(Fixture, "function_return_type_should_disambiguate_from_functi AstStatFunction* statFunc = block->body.data[0]->as(); REQUIRE(statFunc != nullptr); - AstArray& retTypes = statFunc->func->returnAnnotation.types; - REQUIRE(statFunc->func->hasReturnAnnotation); - CHECK(statFunc->func->returnAnnotation.tailType == nullptr); + REQUIRE(statFunc->func->returnAnnotation.has_value()); + CHECK(statFunc->func->returnAnnotation->tailType == nullptr); + AstArray& retTypes = statFunc->func->returnAnnotation->types; REQUIRE(retTypes.size == 2); AstTypeReference* ty0 = retTypes.data[0]->as(); @@ -363,9 +363,9 @@ TEST_CASE_FIXTURE(Fixture, "function_return_type_should_parse_as_function_type_a AstStatFunction* statFunc = block->body.data[0]->as(); REQUIRE(statFunc != nullptr); - AstArray& retTypes = statFunc->func->returnAnnotation.types; - REQUIRE(statFunc->func->hasReturnAnnotation); - CHECK(statFunc->func->returnAnnotation.tailType == nullptr); + REQUIRE(statFunc->func->returnAnnotation.has_value()); + CHECK(statFunc->func->returnAnnotation->tailType == nullptr); + AstArray& retTypes = statFunc->func->returnAnnotation->types; REQUIRE(retTypes.size == 1); AstTypeFunction* funTy = retTypes.data[0]->as(); @@ -707,12 +707,25 @@ TEST_CASE_FIXTURE(Fixture, "mode_is_unset_if_no_hot_comment") TEST_CASE_FIXTURE(Fixture, "sense_hot_comment_on_first_line") { - ParseResult result = parseEx(" --!strict "); + ParseOptions options; + options.captureComments = true; + + ParseResult result = parseEx(" --!strict ", options); std::optional mode = parseMode(result.hotcomments); REQUIRE(bool(mode)); CHECK_EQ(int(*mode), int(Mode::Strict)); } +TEST_CASE_FIXTURE(Fixture, "non_header_hot_comments") +{ + ParseOptions options; + options.captureComments = true; + + ParseResult result = parseEx("do end --!strict", options); + std::optional mode = parseMode(result.hotcomments); + REQUIRE(!mode); +} + TEST_CASE_FIXTURE(Fixture, "stop_if_line_ends_with_hyphen") { CHECK_THROWS_AS(parse(" -"), std::exception); @@ -720,7 +733,10 @@ TEST_CASE_FIXTURE(Fixture, "stop_if_line_ends_with_hyphen") TEST_CASE_FIXTURE(Fixture, "nonstrict_mode") { - ParseResult result = parseEx("--!nonstrict"); + ParseOptions options; + options.captureComments = true; + + ParseResult result = parseEx("--!nonstrict", options); CHECK(result.errors.empty()); std::optional mode = parseMode(result.hotcomments); REQUIRE(bool(mode)); @@ -729,7 +745,10 @@ TEST_CASE_FIXTURE(Fixture, "nonstrict_mode") TEST_CASE_FIXTURE(Fixture, "nocheck_mode") { - ParseResult result = parseEx("--!nocheck"); + ParseOptions options; + options.captureComments = true; + + ParseResult result = parseEx("--!nocheck", options); CHECK(result.errors.empty()); std::optional mode = parseMode(result.hotcomments); REQUIRE(bool(mode)); @@ -1498,8 +1517,6 @@ return TEST_CASE_FIXTURE(Fixture, "parse_error_broken_comment") { - ScopedFastFlag luauStartingBrokenComment{"LuauStartingBrokenComment", true}; - const char* expected = "Expected identifier when parsing expression, got unfinished comment"; matchParseError("--[[unfinished work", expected); diff --git a/tests/Repl.test.cpp b/tests/Repl.test.cpp index 1f9c9739..87a1e1e2 100644 --- a/tests/Repl.test.cpp +++ b/tests/Repl.test.cpp @@ -73,7 +73,7 @@ public: private: std::unique_ptr luaState; - // This is a simplicitic and incomplete pretty printer. + // This is a simplistic and incomplete pretty printer. // It is included here to test that the pretty printer hook is being called. // More elaborate tests to ensure correct output can be added if we introduce // a more feature rich pretty printer. @@ -158,12 +158,25 @@ TEST_CASE_FIXTURE(ReplFixture, "CompleteGlobalVariables") myvariable1 = 5 myvariable2 = 5 )"); - CompletionSet completions = getCompletionSet("myvar"); + { + // Try to complete globals that are added by the user's script + CompletionSet completions = getCompletionSet("myvar"); - std::string prefix = ""; - CHECK(completions.size() == 2); - CHECK(checkCompletion(completions, prefix, "myvariable1")); - CHECK(checkCompletion(completions, prefix, "myvariable2")); + std::string prefix = ""; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "myvariable1")); + CHECK(checkCompletion(completions, prefix, "myvariable2")); + } + { + // Try completing some builtin functions + CompletionSet completions = getCompletionSet("math.m"); + + std::string prefix = "math."; + CHECK(completions.size() == 3); + CHECK(checkCompletion(completions, prefix, "max(")); + CHECK(checkCompletion(completions, prefix, "min(")); + CHECK(checkCompletion(completions, prefix, "modf(")); + } } TEST_CASE_FIXTURE(ReplFixture, "CompleteTableKeys") @@ -206,4 +219,188 @@ TEST_CASE_FIXTURE(ReplFixture, "StringMethods") } } +TEST_CASE_FIXTURE(ReplFixture, "TableWithMetatableIndexTable") +{ + runCode(L, R"( + -- Create 't' which is a table with a metatable with an __index table + mt = {} + mt.__index = mt + + t = {} + setmetatable(t, mt) + + mt.mtkey1 = {x="x value", y="y value", 1, 2} + mt.mtkey2 = 2 + + t.tkey1 = {data1 = 2, data2 = "str", 3, 4} + t.tkey2 = 4 +)"); + { + CompletionSet completions = getCompletionSet("t.t"); + + std::string prefix = "t."; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "tkey1")); + CHECK(checkCompletion(completions, prefix, "tkey2")); + } + { + CompletionSet completions = getCompletionSet("t.tkey1.data2:re"); + + std::string prefix = "t.tkey1.data2:"; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "rep(")); + CHECK(checkCompletion(completions, prefix, "reverse(")); + } + { + CompletionSet completions = getCompletionSet("t.mtk"); + + std::string prefix = "t."; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "mtkey1")); + CHECK(checkCompletion(completions, prefix, "mtkey2")); + } + { + CompletionSet completions = getCompletionSet("t.mtkey1."); + + std::string prefix = "t.mtkey1."; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "x")); + CHECK(checkCompletion(completions, prefix, "y")); + } +} + +TEST_CASE_FIXTURE(ReplFixture, "TableWithMetatableIndexFunction") +{ + runCode(L, R"( + -- Create 't' which is a table with a metatable with an __index function + mt = {} + mt.__index = function(table, key) + print("mt.__index called") + if key == "foo" then + return "FOO" + elseif key == "bar" then + return "BAR" + else + return nil + end + end + + t = {} + setmetatable(t, mt) + t.tkey = 0 +)"); + { + CompletionSet completions = getCompletionSet("t.t"); + + std::string prefix = "t."; + CHECK(completions.size() == 1); + CHECK(checkCompletion(completions, prefix, "tkey")); + } + { + // t.foo is a valid key, but should not be completed because it requires calling an __index function + CompletionSet completions = getCompletionSet("t.foo"); + + CHECK(completions.size() == 0); + } + { + // t.foo is a valid key, but should not be found because it requires calling an __index function + CompletionSet completions = getCompletionSet("t.foo:"); + + CHECK(completions.size() == 0); + } +} + +TEST_CASE_FIXTURE(ReplFixture, "TableWithMultipleMetatableIndexTables") +{ + runCode(L, R"( + -- Create a table with a chain of metatables + mt2 = {} + mt2.__index = mt2 + + mt = {} + mt.__index = mt + setmetatable(mt, mt2) + + t = {} + setmetatable(t, mt) + + mt2.mt2key = {x=1, y=2} + mt.mtkey = 2 + t.tkey = 3 +)"); + { + CompletionSet completions = getCompletionSet("t."); + + std::string prefix = "t."; + CHECK(completions.size() == 4); + CHECK(checkCompletion(completions, prefix, "__index")); + CHECK(checkCompletion(completions, prefix, "tkey")); + CHECK(checkCompletion(completions, prefix, "mtkey")); + CHECK(checkCompletion(completions, prefix, "mt2key")); + } + { + CompletionSet completions = getCompletionSet("t.__index."); + + std::string prefix = "t.__index."; + CHECK(completions.size() == 3); + CHECK(checkCompletion(completions, prefix, "__index")); + CHECK(checkCompletion(completions, prefix, "mtkey")); + CHECK(checkCompletion(completions, prefix, "mt2key")); + } + { + CompletionSet completions = getCompletionSet("t.mt2key."); + + std::string prefix = "t.mt2key."; + CHECK(completions.size() == 2); + CHECK(checkCompletion(completions, prefix, "x")); + CHECK(checkCompletion(completions, prefix, "y")); + } +} + +TEST_CASE_FIXTURE(ReplFixture, "TableWithDeepMetatableIndexTables") +{ + runCode(L, R"( +-- Creates a table with a chain of metatables of length `count` +function makeChainedTable(count) + local result = {} + result.__index = result + result[string.format("entry%d", count)] = { count = count } + if count == 0 then + return result + else + return setmetatable(result, makeChainedTable(count - 1)) + end +end + +t30 = makeChainedTable(30) +t60 = makeChainedTable(60) +)"); + { + // Check if entry0 exists + CompletionSet completions = getCompletionSet("t30.entry0"); + + std::string prefix = "t30."; + CHECK(checkCompletion(completions, prefix, "entry0")); + } + { + // Check if entry0.count exists + CompletionSet completions = getCompletionSet("t30.entry0.co"); + + std::string prefix = "t30.entry0."; + CHECK(checkCompletion(completions, prefix, "count")); + } + { + // Check if entry0 exists. With the max traversal limit of 50 in the repl, this should fail. + CompletionSet completions = getCompletionSet("t60.entry0"); + + CHECK(completions.size() == 0); + } + { + // Check if entry0.count exists. With the max traversal limit of 50 in the repl, this should fail. + CompletionSet completions = getCompletionSet("t60.entry0.co"); + + CHECK(completions.size() == 0); + } +} + TEST_SUITE_END(); diff --git a/tests/RequireTracer.test.cpp b/tests/RequireTracer.test.cpp index b9fd04d6..ba03f363 100644 --- a/tests/RequireTracer.test.cpp +++ b/tests/RequireTracer.test.cpp @@ -1,6 +1,6 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/RequireTracer.h" +#include "Luau/Parser.h" #include "Fixture.h" diff --git a/tests/TypeInfer.aliases.test.cpp b/tests/TypeInfer.aliases.test.cpp index a8729268..31d7ef10 100644 --- a/tests/TypeInfer.aliases.test.cpp +++ b/tests/TypeInfer.aliases.test.cpp @@ -598,15 +598,13 @@ TEST_CASE_FIXTURE(Fixture, "generic_typevars_are_not_considered_to_escape_their_ /* * The two-pass alias definition system starts by ascribing a free TypeVar to each alias. It then * circles back to fill in the actual type later on. - * + * * If this free type is unified with something degenerate like `any`, we need to take extra care * to ensure that the alias actually binds to the type that the user expected. */ TEST_CASE_FIXTURE(Fixture, "forward_declared_alias_is_not_clobbered_by_prior_unification_with_any") { - ScopedFastFlag sff[] = { - {"LuauTwoPassAliasDefinitionFix", true} - }; + ScopedFastFlag sff[] = {{"LuauTwoPassAliasDefinitionFix", true}}; CheckResult result = check(R"( local function x() diff --git a/tests/TypeInfer.annotations.test.cpp b/tests/TypeInfer.annotations.test.cpp index 572b882d..2ad11d01 100644 --- a/tests/TypeInfer.annotations.test.cpp +++ b/tests/TypeInfer.annotations.test.cpp @@ -1,6 +1,5 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeInfer.builtins.test.cpp b/tests/TypeInfer.builtins.test.cpp index df06884d..f3dfb214 100644 --- a/tests/TypeInfer.builtins.test.cpp +++ b/tests/TypeInfer.builtins.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/BuiltinDefinitions.h" diff --git a/tests/TypeInfer.classes.test.cpp b/tests/TypeInfer.classes.test.cpp index 0283ae19..98fa66eb 100644 --- a/tests/TypeInfer.classes.test.cpp +++ b/tests/TypeInfer.classes.test.cpp @@ -1,6 +1,5 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeInfer.definitions.test.cpp b/tests/TypeInfer.definitions.test.cpp index 2652486b..c6d55793 100644 --- a/tests/TypeInfer.definitions.test.cpp +++ b/tests/TypeInfer.definitions.test.cpp @@ -1,6 +1,5 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeInfer.generics.test.cpp b/tests/TypeInfer.generics.test.cpp index 8a2c6f27..f8fccf6b 100644 --- a/tests/TypeInfer.generics.test.cpp +++ b/tests/TypeInfer.generics.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeInfer.intersectionTypes.test.cpp b/tests/TypeInfer.intersectionTypes.test.cpp index 93c0baf6..d677e28d 100644 --- a/tests/TypeInfer.intersectionTypes.test.cpp +++ b/tests/TypeInfer.intersectionTypes.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeInfer.provisional.test.cpp b/tests/TypeInfer.provisional.test.cpp index 2bcd840c..eee0e0f1 100644 --- a/tests/TypeInfer.provisional.test.cpp +++ b/tests/TypeInfer.provisional.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Fixture.h" diff --git a/tests/TypeInfer.singletons.test.cpp b/tests/TypeInfer.singletons.test.cpp index df365fda..9021700d 100644 --- a/tests/TypeInfer.singletons.test.cpp +++ b/tests/TypeInfer.singletons.test.cpp @@ -426,8 +426,6 @@ TEST_CASE_FIXTURE(Fixture, "if_then_else_expression_singleton_options") {"LuauSingletonTypes", true}, {"LuauParseSingletonTypes", true}, {"LuauExpectedTypesOfProperties", true}, - {"LuauIfElseExpectedType2", true}, - {"LuauIfElseBranchTypeUnion", true}, }; CheckResult result = check(R"( diff --git a/tests/TypeInfer.tables.test.cpp b/tests/TypeInfer.tables.test.cpp index f19cb618..6bcd4b99 100644 --- a/tests/TypeInfer.tables.test.cpp +++ b/tests/TypeInfer.tables.test.cpp @@ -1,6 +1,5 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" @@ -2168,8 +2167,6 @@ b() TEST_CASE_FIXTURE(Fixture, "length_operator_union") { - ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true}; - CheckResult result = check(R"( local x: {number} | {string} local y = #x @@ -2180,8 +2177,6 @@ local y = #x TEST_CASE_FIXTURE(Fixture, "length_operator_intersection") { - ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true}; - CheckResult result = check(R"( local x: {number} & {z:string} -- mixed tables are evil local y = #x @@ -2192,8 +2187,6 @@ local y = #x TEST_CASE_FIXTURE(Fixture, "length_operator_non_table_union") { - ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true}; - CheckResult result = check(R"( local x: {number} | any | string local y = #x @@ -2204,8 +2197,6 @@ local y = #x TEST_CASE_FIXTURE(Fixture, "length_operator_union_errors") { - ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true}; - CheckResult result = check(R"( local x: {number} | number | string local y = #x @@ -2214,4 +2205,38 @@ local y = #x LUAU_REQUIRE_ERROR_COUNT(1, result); } +TEST_CASE_FIXTURE(Fixture, "dont_hang_when_trying_to_look_up_in_cyclic_metatable_index") +{ + ScopedFastFlag sff{"LuauTerminateCyclicMetatableIndexLookup", true}; + + // t :: t1 where t1 = {metatable {__index: t1, __tostring: (t1) -> string}} + CheckResult result = check(R"( + local mt = {} + local t = setmetatable({}, mt) + mt.__index = t + + function mt:__tostring() + return t.p + end + )"); + + LUAU_REQUIRE_ERROR_COUNT(1, result); + CHECK_EQ("Type 't' does not have key 'p'", toString(result.errors[0])); +} + +TEST_CASE_FIXTURE(Fixture, "give_up_after_one_metatable_index_look_up") +{ + CheckResult result = check(R"( + local data = { x = 5 } + local t1 = setmetatable({}, { __index = data }) + local t2 = setmetatable({}, t1) -- note: must be t1, not a new table + + local x1 = t1.x -- ok + local x2 = t2.x -- nope + )"); + + LUAU_REQUIRE_ERROR_COUNT(1, result); + CHECK_EQ("Type 't2' does not have key 'x'", toString(result.errors[0])); +} + TEST_SUITE_END(); diff --git a/tests/TypeInfer.test.cpp b/tests/TypeInfer.test.cpp index 531a382f..32358571 100644 --- a/tests/TypeInfer.test.cpp +++ b/tests/TypeInfer.test.cpp @@ -2,7 +2,6 @@ #include "Luau/AstQuery.h" #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/Scope.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" @@ -4654,8 +4653,6 @@ a = setmetatable(a, { __call = function(x) end }) TEST_CASE_FIXTURE(Fixture, "infer_through_group_expr") { - ScopedFastFlag luauGroupExpectedType{"LuauGroupExpectedType", true}; - CheckResult result = check(R"( local function f(a: (number, number) -> number) return a(1, 3) end f(((function(a, b) return a + b end))) @@ -4735,21 +4732,14 @@ local a = if false then "a" elseif false then "b" else "c" TEST_CASE_FIXTURE(Fixture, "tc_if_else_expressions_type_union") { - ScopedFastFlag sff3{"LuauIfElseBranchTypeUnion", true}; + CheckResult result = check(R"(local a: number? = if true then 42 else nil)"); - { - CheckResult result = check(R"(local a: number? = if true then 42 else nil)"); - - LUAU_REQUIRE_NO_ERRORS(result); - CHECK_EQ(toString(requireType("a"), {true}), "number?"); - } + LUAU_REQUIRE_NO_ERRORS(result); + CHECK_EQ(toString(requireType("a"), {true}), "number?"); } TEST_CASE_FIXTURE(Fixture, "tc_if_else_expressions_expected_type_1") { - ScopedFastFlag luauIfElseExpectedType2{"LuauIfElseExpectedType2", true}; - ScopedFastFlag luauIfElseBranchTypeUnion{"LuauIfElseBranchTypeUnion", true}; - CheckResult result = check(R"( type X = {number | string} local a: X = if true then {"1", 2, 3} else {4, 5, 6} @@ -4761,9 +4751,6 @@ local a: X = if true then {"1", 2, 3} else {4, 5, 6} TEST_CASE_FIXTURE(Fixture, "tc_if_else_expressions_expected_type_2") { - ScopedFastFlag luauIfElseExpectedType2{"LuauIfElseExpectedType2", true}; - ScopedFastFlag luauIfElseBranchTypeUnion{"LuauIfElseBranchTypeUnion", true}; - CheckResult result = check(R"( local a: number? = if true then 1 else nil )"); @@ -4773,8 +4760,6 @@ local a: number? = if true then 1 else nil TEST_CASE_FIXTURE(Fixture, "tc_if_else_expressions_expected_type_3") { - ScopedFastFlag luauIfElseExpectedType2{"LuauIfElseExpectedType2", true}; - CheckResult result = check(R"( local function times(n: any, f: () -> T) local result: {T} = {} @@ -5058,8 +5043,6 @@ end TEST_CASE_FIXTURE(Fixture, "recursive_metatable_crash") { - ScopedFastFlag luauMetatableAreEqualRecursion{"LuauMetatableAreEqualRecursion", true}; - CheckResult result = check(R"( local function getIt() local y @@ -5076,8 +5059,6 @@ local c = a or b TEST_CASE_FIXTURE(Fixture, "bound_typepack_promote") { - ScopedFastFlag luauCommittingTxnLogFreeTpPromote{"LuauCommittingTxnLogFreeTpPromote", true}; - // No assertions should trigger check(R"( local function p() @@ -5251,7 +5232,6 @@ TEST_CASE_FIXTURE(Fixture, "taking_the_length_of_string_singleton") {"LuauDiscriminableUnions2", true}, {"LuauRefactorTypeVarQuestions", true}, {"LuauSingletonTypes", true}, - {"LuauLengthOnCompositeType", true}, }; CheckResult result = check(R"( @@ -5272,7 +5252,6 @@ TEST_CASE_FIXTURE(Fixture, "taking_the_length_of_union_of_string_singleton") {"LuauDiscriminableUnions2", true}, {"LuauRefactorTypeVarQuestions", true}, {"LuauSingletonTypes", true}, - {"LuauLengthOnCompositeType", true}, }; CheckResult result = check(R"( diff --git a/tests/TypeInfer.tryUnify.test.cpp b/tests/TypeInfer.tryUnify.test.cpp index 8c7fb79a..4669ea8e 100644 --- a/tests/TypeInfer.tryUnify.test.cpp +++ b/tests/TypeInfer.tryUnify.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/Scope.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeInfer.typePacks.cpp b/tests/TypeInfer.typePacks.cpp index 079870f5..cbe2e48f 100644 --- a/tests/TypeInfer.typePacks.cpp +++ b/tests/TypeInfer.typePacks.cpp @@ -1,6 +1,5 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/BuiltinDefinitions.h" -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" @@ -930,8 +929,6 @@ type R = { m: F } TEST_CASE_FIXTURE(Fixture, "pack_tail_unification_check") { - ScopedFastFlag luauUnifyPackTails{"LuauUnifyPackTails", true}; - CheckResult result = check(R"( local a: () -> (number, ...string) local b: () -> (number, ...boolean) diff --git a/tests/TypeInfer.unionTypes.test.cpp b/tests/TypeInfer.unionTypes.test.cpp index 759794e6..3b53ddfe 100644 --- a/tests/TypeInfer.unionTypes.test.cpp +++ b/tests/TypeInfer.unionTypes.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypePack.test.cpp b/tests/TypePack.test.cpp index 8b056544..c4931578 100644 --- a/tests/TypePack.test.cpp +++ b/tests/TypePack.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/TypeVar.test.cpp b/tests/TypeVar.test.cpp index 329e7b1f..e43161fa 100644 --- a/tests/TypeVar.test.cpp +++ b/tests/TypeVar.test.cpp @@ -1,5 +1,4 @@ // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details -#include "Luau/Parser.h" #include "Luau/Scope.h" #include "Luau/TypeInfer.h" #include "Luau/TypeVar.h" diff --git a/tests/conformance/coroutine.lua b/tests/conformance/coroutine.lua index f2ecc96b..b4f81bba 100644 --- a/tests/conformance/coroutine.lua +++ b/tests/conformance/coroutine.lua @@ -371,6 +371,15 @@ do st, msg = coroutine.close(co) assert(st and msg == nil) assert(f() == 42) + + -- closing a coroutine with a large stack + co = coroutine.create(function() + local function f(depth) return if depth > 0 then f(depth - 1) + depth else 0 end + coroutine.yield(f(100)) + end) + assert(coroutine.resume(co)) + st, msg = coroutine.close(co) + assert(st and msg == nil) end return 'OK' diff --git a/tests/conformance/coverage.lua b/tests/conformance/coverage.lua index f899603f..14d843a4 100644 --- a/tests/conformance/coverage.lua +++ b/tests/conformance/coverage.lua @@ -49,16 +49,24 @@ foo() c = getcoverage(foo) assert(#c == 1) assert(c[1].name == "foo") +assert(c[1].linedefined == 4) +assert(c[1].depth == 0) assert(validate(c[1], {5, 6, 7}, {})) bar() c = getcoverage(bar) assert(#c == 3) assert(c[1].name == "bar") +assert(c[1].linedefined == 10) +assert(c[1].depth == 0) assert(validate(c[1], {11, 15, 19}, {})) assert(c[2].name == "one") +assert(c[2].linedefined == 11) +assert(c[2].depth == 1) assert(validate(c[2], {12}, {})) assert(c[3].name == nil) +assert(c[3].linedefined == 15) +assert(c[3].depth == 1) assert(validate(c[3], {}, {16})) return 'OK' diff --git a/tests/conformance/debug.lua b/tests/conformance/debug.lua index 0e410000..0c8cc2d8 100644 --- a/tests/conformance/debug.lua +++ b/tests/conformance/debug.lua @@ -76,6 +76,9 @@ assert(baz(co, 2, "n") == nil) assert(baz(math.sqrt, "n") == "sqrt") assert(baz(math.sqrt, "f") == math.sqrt) -- yes this is pointless +local t = { foo = function() return 1 end } +assert(baz(t.foo, "n") == "foo") + -- info multi-arg returns function quux(...) return {debug.info(...)} diff --git a/tests/main.cpp b/tests/main.cpp index cd24e100..2af9f702 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -10,6 +10,9 @@ #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include // IsDebuggerPresent #endif @@ -52,7 +55,7 @@ static bool debuggerPresent() #endif } -static int assertionHandler(const char* expr, const char* file, int line, const char* function) +static int testAssertionHandler(const char* expr, const char* file, int line, const char* function) { if (debuggerPresent()) LUAU_DEBUGBREAK(); @@ -218,7 +221,7 @@ static void setFastFlags(const std::vector& flags) int main(int argc, char** argv) { - Luau::assertHandler() = assertionHandler; + Luau::assertHandler() = testAssertionHandler; doctest::registerReporter("boost", 0, true); diff --git a/tools/natvis/Analysis.natvis b/tools/natvis/Analysis.natvis new file mode 100644 index 00000000..5de0140e --- /dev/null +++ b/tools/natvis/Analysis.natvis @@ -0,0 +1,78 @@ + + + + + AnyTypeVar + + + + {{ index=0, value={*($T1*)storage} }} + {{ index=1, value={*($T2*)storage} }} + {{ index=2, value={*($T3*)storage} }} + {{ index=3, value={*($T4*)storage} }} + {{ index=4, value={*($T5*)storage} }} + {{ index=5, value={*($T6*)storage} }} + {{ index=6, value={*($T7*)storage} }} + {{ index=7, value={*($T8*)storage} }} + {{ index=8, value={*($T9*)storage} }} + {{ index=9, value={*($T10*)storage} }} + {{ index=10, value={*($T11*)storage} }} + {{ index=11, value={*($T12*)storage} }} + {{ index=12, value={*($T13*)storage} }} + {{ index=13, value={*($T14*)storage} }} + {{ index=14, value={*($T15*)storage} }} + {{ index=15, value={*($T16*)storage} }} + {{ index=16, value={*($T17*)storage} }} + {{ index=17, value={*($T18*)storage} }} + {{ index=18, value={*($T19*)storage} }} + {{ index=19, value={*($T20*)storage} }} + {{ index=20, value={*($T21*)storage} }} + {{ index=21, value={*($T22*)storage} }} + {{ index=22, value={*($T23*)storage} }} + {{ index=23, value={*($T24*)storage} }} + {{ index=24, value={*($T25*)storage} }} + {{ index=25, value={*($T26*)storage} }} + {{ index=26, value={*($T27*)storage} }} + {{ index=27, value={*($T28*)storage} }} + {{ index=28, value={*($T29*)storage} }} + {{ index=29, value={*($T30*)storage} }} + {{ index=30, value={*($T31*)storage} }} + {{ index=31, value={*($T32*)storage} }} + + typeId + *($T1*)storage + *($T2*)storage + *($T3*)storage + *($T4*)storage + *($T5*)storage + *($T6*)storage + *($T7*)storage + *($T8*)storage + *($T9*)storage + *($T10*)storage + *($T11*)storage + *($T12*)storage + *($T13*)storage + *($T14*)storage + *($T15*)storage + *($T16*)storage + *($T17*)storage + *($T18*)storage + *($T19*)storage + *($T20*)storage + *($T21*)storage + *($T22*)storage + *($T23*)storage + *($T24*)storage + *($T25*)storage + *($T26*)storage + *($T27*)storage + *($T28*)storage + *($T29*)storage + *($T30*)storage + *($T31*)storage + *($T32*)storage + + + + diff --git a/tools/natvis/Ast.natvis b/tools/natvis/Ast.natvis new file mode 100644 index 00000000..322eb8f6 --- /dev/null +++ b/tools/natvis/Ast.natvis @@ -0,0 +1,25 @@ + + + + + AstArray size={size} + + size + + size + data + + + + + + + size_ + + size_ + storage._Mypair._Myval2._Myfirst + offset + + + + + diff --git a/tools/natvis/VM.natvis b/tools/natvis/VM.natvis new file mode 100644 index 00000000..9924e194 --- /dev/null +++ b/tools/natvis/VM.natvis @@ -0,0 +1,269 @@ + + + + + nil + {(bool)value.b} + lightuserdata {value.p} + number = {value.n} + vector = {value.v[0]}, {value.v[1]}, {*(float*)&extra} + {value.gc->ts} + {value.gc->h} + function {value.gc->cl,view(short)} + userdata {value.gc->u} + thread {value.gc->th} + proto {value.gc->p} + upvalue {value.gc->uv} + deadkey + empty + + value.p + value.gc->ts + value.gc->h + value.gc->cl + value.gc->cl + value.gc->u + value.gc->th + value.gc->p + value.gc->uv + + fixed ({(int)value.gc->gch.marked}) + black ({(int)value.gc->gch.marked}) + white ({(int)value.gc->gch.marked}) + white ({(int)value.gc->gch.marked}) + gray ({(int)value.gc->gch.marked}) + + + + + + nil + {(bool)value.b} + lightuserdata {value.p} + number = {value.n} + vector = {value.v[0]}, {value.v[1]}, {*(float*)&extra} + {value.gc->ts} + {value.gc->h} + function {value.gc->cl,view(short)} + userdata {value.gc->u} + thread {value.gc->th} + proto {value.gc->p} + upvalue {value.gc->uv} + deadkey + empty + + (void**)value.p + value.gc->ts + value.gc->h + value.gc->cl + value.gc->cl + value.gc->u + value.gc->th + value.gc->p + value.gc->uv + + next + + + + + {key,na} = {val} + --- + + + + table + + metatable + + + [size] {1<<lsizenode} + + + 1<<lsizenode + node[$i] + + + + + [size] {sizearray} + + + sizearray + array[$i] + + + + + + + + + + + + + 1 + + + + + metatable->node[i].val + + + + i = i + 1 + + + "unknown",sb + + + tag + len + metatable + data + + + + + {c.f,na} + {l.p,na} + {c} + {l} + invalid + + + + {data,s} + + + + + {ci->func->value.gc->cl.c.f,na} + + + {ci->func->value.gc->cl.l.p->source->data,sb}:{ci->func->value.gc->cl.l.p->linedefined,d} {ci->func->value.gc->cl.l.p->debugname->data,sb} + + + {ci->func->value.gc->cl.l.p->source->data,sb}:{ci->func->value.gc->cl.l.p->linedefined,d} + + thread + + + {ci-base_ci} frames + + + ci-base_ci + + + base_ci[ci-base_ci - $i].func->value.gc->cl,view(short) + + + + + + {top-base} values + + + top-base + base + + + + + {top-stack} values + + + top-stack + stack + + + + + + + openupval + u.l.next + this + + + + l_gt + env + userdata + + + + + {source->data,sb}:{linedefined} function {debugname->data,sb} [{(int)numparams} arg, {(int)nups} upval] + {source->data,sb}:{linedefined} [{(int)numparams} arg, {(int)nups} upval] + + debugname + + constants + + + sizek + k[$i] + + + + + locals + + + sizelocvars + locvars[$i] + + + + + bytecode + + + sizecode + code[$i] + + + + + functions + + + sizep + p[$i] + + + + + upvals + + + sizeupvalues + upvalues[$i] + + + + + source + + + + + + + {(lua_Type)tt} + + + fixed ({(int)marked}) + black ({(int)marked}) + white ({(int)marked}) + white ({(int)marked}) + gray ({(int)marked}) + unknown + + memcat + + + + From a9bdce6cc06577cb412c38db757e44ea783f7c05 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Fri, 18 Feb 2022 10:04:38 -0800 Subject: [PATCH 18/25] Rename tests to tests.py (#374) --- .github/workflows/prototyping.yml | 2 +- prototyping/{tests => tests.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename prototyping/{tests => tests.py} (99%) diff --git a/.github/workflows/prototyping.yml b/.github/workflows/prototyping.yml index 637ed840..fe458673 100644 --- a/.github/workflows/prototyping.yml +++ b/.github/workflows/prototyping.yml @@ -73,7 +73,7 @@ jobs: working-directory: prototyping run: | mkdir test-failures - python tests -l ../build/luau-ast --write-diff-failures --diff-failure-location test-failures/ + python tests.py -l ../build/luau-ast --write-diff-failures --diff-failure-location test-failures/ - uses: actions/upload-artifact@v2 if: failure() with: diff --git a/prototyping/tests b/prototyping/tests.py similarity index 99% rename from prototyping/tests rename to prototyping/tests.py index 89e105d0..20070ae0 100755 --- a/prototyping/tests +++ b/prototyping/tests.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/usr/bin/python import argparse import difflib From 7f867ac166c39a3b291a5a66e70500335e3211a3 Mon Sep 17 00:00:00 2001 From: Lily Brown Date: Fri, 18 Feb 2022 11:09:00 -0800 Subject: [PATCH 19/25] Prototyping: numbers (#368) Adds number support to the prototype. Binary operators are next. --- prototyping/Examples/OpSem.agda | 1 - prototyping/Examples/Run.agda | 7 +++++-- prototyping/FFI/Data/Scientific.agda | 15 +++++++++++++++ prototyping/Interpreter.agda | 1 - prototyping/Luau/RuntimeError.agda | 5 +++-- prototyping/Luau/RuntimeError/ToString.agda | 8 ++++++-- prototyping/Luau/Substitution.agda | 4 ++-- prototyping/Luau/Syntax.agda | 3 ++- prototyping/Luau/Syntax/FromJSON.agda | 8 +++++++- prototyping/Luau/Syntax/ToString.agda | 4 +++- prototyping/Luau/Type.agda | 4 +++- prototyping/Luau/Type/FromJSON.agda | 3 ++- prototyping/Luau/Type/ToString.agda | 3 ++- prototyping/Luau/Value.agda | 7 ++++--- prototyping/Luau/Value/ToString.agda | 5 +++-- prototyping/PrettyPrinter.agda | 1 - prototyping/Properties/Step.agda | 8 +++++--- prototyping/Tests/PrettyPrinter/smoke_test/in.lua | 1 + .../Tests/PrettyPrinter/smoke_test/out.txt | 1 + 19 files changed, 64 insertions(+), 25 deletions(-) diff --git a/prototyping/Examples/OpSem.agda b/prototyping/Examples/OpSem.agda index ec8bce7b..c4cfc5e5 100644 --- a/prototyping/Examples/OpSem.agda +++ b/prototyping/Examples/OpSem.agda @@ -6,4 +6,3 @@ open import Luau.Heap using (∅) ex1 : ∅ ⊢ (local (var "x") ← nil ∙ return (var "x") ∙ done) ⟶ᴮ (return nil ∙ done) ⊣ ∅ ex1 = subst - diff --git a/prototyping/Examples/Run.agda b/prototyping/Examples/Run.agda index 88ca39b6..3cac564d 100644 --- a/prototyping/Examples/Run.agda +++ b/prototyping/Examples/Run.agda @@ -3,8 +3,8 @@ module Examples.Run where open import Agda.Builtin.Equality using (_≡_; refl) -open import Luau.Syntax using (nil; var; _$_; function_is_end; return; _∙_; done; _⟨_⟩) -open import Luau.Value using (nil) +open import Luau.Syntax using (nil; var; _$_; function_is_end; return; _∙_; done; _⟨_⟩; number) +open import Luau.Value using (nil; number) open import Luau.Run using (run; return) open import Luau.Heap using (lookup-next; next-emp; lookup-next-emp) @@ -13,3 +13,6 @@ import Agda.Builtin.Equality.Rewrite ex1 : (run (function "id" ⟨ var "x" ⟩ is return (var "x") ∙ done end ∙ return (var "id" $ nil) ∙ done) ≡ return nil _) ex1 = refl + +ex2 : (run (function "fn" ⟨ var "x" ⟩ is return (number 123.0) ∙ done end ∙ return (var "fn" $ nil) ∙ done) ≡ return (number 123.0) _) +ex2 = refl diff --git a/prototyping/FFI/Data/Scientific.agda b/prototyping/FFI/Data/Scientific.agda index 8a5be39e..772d3367 100644 --- a/prototyping/FFI/Data/Scientific.agda +++ b/prototyping/FFI/Data/Scientific.agda @@ -1,6 +1,21 @@ module FFI.Data.Scientific where +open import Agda.Builtin.Float using (Float) +open import FFI.Data.String using (String) +open import FFI.Data.HaskellString using (HaskellString; pack; unpack) + {-# FOREIGN GHC import qualified Data.Scientific #-} +{-# FOREIGN GHC import qualified Text.Show #-} postulate Scientific : Set {-# COMPILE GHC Scientific = type Data.Scientific.Scientific #-} + +postulate + showHaskell : Scientific → HaskellString + toFloat : Scientific → Float + +{-# COMPILE GHC showHaskell = \x -> Text.Show.show x #-} +{-# COMPILE GHC toFloat = \x -> Data.Scientific.toRealFloat x #-} + +show : Scientific → String +show x = pack (showHaskell x) diff --git a/prototyping/Interpreter.agda b/prototyping/Interpreter.agda index 3ec5b8d1..fe311e57 100644 --- a/prototyping/Interpreter.agda +++ b/prototyping/Interpreter.agda @@ -36,4 +36,3 @@ runString txt | (Right value) = runJSON value main : IO ⊤ main = getContents >>= runString - diff --git a/prototyping/Luau/RuntimeError.agda b/prototyping/Luau/RuntimeError.agda index e514dc9d..f9e684b3 100644 --- a/prototyping/Luau/RuntimeError.agda +++ b/prototyping/Luau/RuntimeError.agda @@ -3,13 +3,15 @@ module Luau.RuntimeError where open import Agda.Builtin.Equality using (_≡_) open import Luau.Heap using (Heap; _[_]) open import FFI.Data.Maybe using (just; nothing) -open import Luau.Syntax using (Block; Expr; nil; var; addr; block_is_end; _$_; local_←_; return; done; _∙_) +open import FFI.Data.String using (String) +open import Luau.Syntax using (Block; Expr; nil; var; addr; block_is_end; _$_; local_←_; return; done; _∙_; number) data RuntimeErrorᴮ {a} (H : Heap a) : Block a → Set data RuntimeErrorᴱ {a} (H : Heap a) : Expr a → Set data RuntimeErrorᴱ H where NilIsNotAFunction : ∀ {M} → RuntimeErrorᴱ H (nil $ M) + NumberIsNotAFunction : ∀ n {M} → RuntimeErrorᴱ H ((number n) $ M) UnboundVariable : ∀ x → RuntimeErrorᴱ H (var x) SEGV : ∀ a → (H [ a ] ≡ nothing) → RuntimeErrorᴱ H (addr a) app : ∀ {M N} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (M $ N) @@ -18,4 +20,3 @@ data RuntimeErrorᴱ H where data RuntimeErrorᴮ H where local : ∀ x {M B} → RuntimeErrorᴱ H M → RuntimeErrorᴮ H (local x ← M ∙ B) return : ∀ {M B} → RuntimeErrorᴱ H M → RuntimeErrorᴮ H (return M ∙ B) - diff --git a/prototyping/Luau/RuntimeError/ToString.agda b/prototyping/Luau/RuntimeError/ToString.agda index e8287157..ac760fab 100644 --- a/prototyping/Luau/RuntimeError/ToString.agda +++ b/prototyping/Luau/RuntimeError/ToString.agda @@ -1,15 +1,18 @@ module Luau.RuntimeError.ToString where +open import Agda.Builtin.Float using (primShowFloat) open import FFI.Data.String using (String; _++_) -open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; UnboundVariable; SEGV; app; block) +open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app; block) open import Luau.Addr.ToString using (addrToString) +open import Luau.Syntax.ToString using (exprToString) open import Luau.Var.ToString using (varToString) -open import Luau.Syntax using (name) +open import Luau.Syntax using (name; _$_) errToStringᴱ : ∀ {a H B} → RuntimeErrorᴱ {a} H B → String errToStringᴮ : ∀ {a H B} → RuntimeErrorᴮ {a} H B → String errToStringᴱ NilIsNotAFunction = "nil is not a function" +errToStringᴱ (NumberIsNotAFunction n) = "number " ++ primShowFloat n ++ " is not a function" errToStringᴱ (UnboundVariable x) = "variable " ++ varToString x ++ " is unbound" errToStringᴱ (SEGV a x) = "address " ++ addrToString a ++ " is unallocated" errToStringᴱ (app E) = errToStringᴱ E @@ -17,3 +20,4 @@ errToStringᴱ (block b E) = errToStringᴮ E ++ "\n in call of function " ++ v errToStringᴮ (local x E) = errToStringᴱ E ++ "\n in definition of " ++ varToString (name x) errToStringᴮ (return E) = errToStringᴱ E ++ "\n in return statement" + \ No newline at end of file diff --git a/prototyping/Luau/Substitution.agda b/prototyping/Luau/Substitution.agda index b956aeae..e364a000 100644 --- a/prototyping/Luau/Substitution.agda +++ b/prototyping/Luau/Substitution.agda @@ -1,6 +1,6 @@ module Luau.Substitution where -open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; _⟨_⟩ ; name; fun; arg) +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; _⟨_⟩ ; name; fun; arg; number) open import Luau.Value using (Value; val) open import Luau.Var using (Var; _≡ⱽ_) open import Properties.Dec using (Dec; yes; no) @@ -13,6 +13,7 @@ _[_/_]ᴮunless_ : ∀ {a P} → Block a → Value → Var → (Dec P) → Block nil [ v / x ]ᴱ = nil var y [ v / x ]ᴱ = var y [ v / x ]ᴱwhenever (x ≡ⱽ y) addr a [ v / x ]ᴱ = addr a +(number y) [ v / x ]ᴱ = number y (M $ N) [ v / x ]ᴱ = (M [ v / x ]ᴱ) $ (N [ v / x ]ᴱ) function F is C end [ v / x ]ᴱ = function F is C [ v / x ]ᴮunless (x ≡ⱽ name(arg F)) end block b is C end [ v / x ]ᴱ = block b is C [ v / x ]ᴮ end @@ -27,4 +28,3 @@ var y [ v / x ]ᴱwhenever no p = var y B [ v / x ]ᴮunless yes p = B B [ v / x ]ᴮunless no p = B [ v / x ]ᴮ - diff --git a/prototyping/Luau/Syntax.agda b/prototyping/Luau/Syntax.agda index 04970b62..1313456b 100644 --- a/prototyping/Luau/Syntax.agda +++ b/prototyping/Luau/Syntax.agda @@ -1,6 +1,7 @@ module Luau.Syntax where open import Agda.Builtin.Equality using (_≡_) +open import Agda.Builtin.Float using (Float) open import Properties.Dec using (⊥) open import Luau.Var using (Var) open import Luau.Addr using (Addr) @@ -52,4 +53,4 @@ data Expr a where _$_ : Expr a → Expr a → Expr a function_is_end : FunDec a → Block a → Expr a block_is_end : Var → Block a → Expr a - + number : Float → Expr a diff --git a/prototyping/Luau/Syntax/FromJSON.agda b/prototyping/Luau/Syntax/FromJSON.agda index 8ae620bb..8191e9e7 100644 --- a/prototyping/Luau/Syntax/FromJSON.agda +++ b/prototyping/Luau/Syntax/FromJSON.agda @@ -1,6 +1,6 @@ module Luau.Syntax.FromJSON where -open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; var_∈_; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec) +open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; var_∈_; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec; number) open import Luau.Type.FromJSON using (typeFromJSON) open import Agda.Builtin.List using (List; _∷_; []) @@ -9,6 +9,7 @@ open import FFI.Data.Aeson using (Value; Array; Object; object; array; string; f open import FFI.Data.Bool using (true; false) open import FFI.Data.Either using (Either; Left; Right) open import FFI.Data.Maybe using (Maybe; nothing; just) +open import FFI.Data.Scientific using (toFloat) open import FFI.Data.String using (String; _++_) open import FFI.Data.Vector using (head; tail; null; empty) @@ -19,6 +20,7 @@ lokal = fromString "local" list = fromString "list" name = fromString "name" type = fromString "type" +value = fromString "value" values = fromString "values" vars = fromString "vars" @@ -83,6 +85,10 @@ exprFromObject obj | just (string "AstExprLocal") | just x with varDecFromJSON x exprFromObject obj | just (string "AstExprLocal") | just x | Right x′ = Right (var (Luau.Syntax.name x′)) exprFromObject obj | just (string "AstExprLocal") | just x | Left err = Left err exprFromObject obj | just (string "AstExprLocal") | nothing = Left "AstExprLocal missing local" +exprFromObject obj | just (string "AstExprConstantNumber") with lookup value obj +exprFromObject obj | just (string "AstExprConstantNumber") | just (FFI.Data.Aeson.Value.number x) = Right (number (toFloat x)) +exprFromObject obj | just (string "AstExprConstantNumber") | just _ = Left "AstExprConstantNumber value is not a number" +exprFromObject obj | just (string "AstExprConstantNumber") | nothing = Left "AstExprConstantNumber missing value" exprFromObject obj | just (string ty) = Left ("TODO: Unsupported AstExpr " ++ ty) exprFromObject obj | just _ = Left "AstExpr type not a string" exprFromObject obj | nothing = Left "AstExpr missing type" diff --git a/prototyping/Luau/Syntax/ToString.agda b/prototyping/Luau/Syntax/ToString.agda index 18d36175..a20b360b 100644 --- a/prototyping/Luau/Syntax/ToString.agda +++ b/prototyping/Luau/Syntax/ToString.agda @@ -1,6 +1,7 @@ module Luau.Syntax.ToString where -open import Luau.Syntax using (Block; Stat; Expr; VarDec; FunDec; nil; var; var_∈_; addr; _$_; function_is_end; return; local_←_; _∙_; done; block_is_end; _⟨_⟩; _⟨_⟩∈_) +open import Agda.Builtin.Float using (primShowFloat) +open import Luau.Syntax using (Block; Stat; Expr; VarDec; FunDec; nil; var; var_∈_; addr; _$_; function_is_end; return; local_←_; _∙_; done; block_is_end; _⟨_⟩; _⟨_⟩∈_; number) open import FFI.Data.String using (String; _++_) open import Luau.Addr.ToString using (addrToString) open import Luau.Type.ToString using (typeToString) @@ -36,6 +37,7 @@ exprToString′ lb (block b is B end) = "(" ++ b ++ "()" ++ lb ++ " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ "end)()" +exprToString′ lb (number x) = primShowFloat x statToString′ lb (function F is B end) = "local " ++ funDecToString F ++ lb ++ diff --git a/prototyping/Luau/Type.agda b/prototyping/Luau/Type.agda index 6e384c3b..af5f857c 100644 --- a/prototyping/Luau/Type.agda +++ b/prototyping/Luau/Type.agda @@ -7,6 +7,7 @@ data Type : Set where _⇒_ : Type → Type → Type none : Type any : Type + number : Type _∪_ : Type → Type → Type _∩_ : Type → Type → Type @@ -15,6 +16,7 @@ src nil = none src (S ⇒ T) = S src none = none src any = any +src number = none src (S ∪ T) = (src S) ∪ (src T) src (S ∩ T) = (src S) ∩ (src T) @@ -23,6 +25,7 @@ tgt nil = none tgt (S ⇒ T) = T tgt none = none tgt any = any +tgt number = none tgt (S ∪ T) = (tgt S) ∪ (tgt T) tgt (S ∩ T) = (tgt S) ∩ (tgt T) @@ -40,4 +43,3 @@ normalizeOptional (S ∪ T) | S′ | nil = optional S′ normalizeOptional (S ∪ T) | nil | T′ = optional T′ normalizeOptional (S ∪ T) | S′ | T′ = S′ ∪ T′ normalizeOptional T = T - diff --git a/prototyping/Luau/Type/FromJSON.agda b/prototyping/Luau/Type/FromJSON.agda index 45bda5f3..c585c3f3 100644 --- a/prototyping/Luau/Type/FromJSON.agda +++ b/prototyping/Luau/Type/FromJSON.agda @@ -1,6 +1,6 @@ module Luau.Type.FromJSON where -open import Luau.Type using (Type; nil; _⇒_; _∪_; _∩_; any) +open import Luau.Type using (Type; nil; _⇒_; _∪_; _∩_; any; number) open import Agda.Builtin.List using (List; _∷_; []) @@ -41,6 +41,7 @@ typeFromJSON (object o) | just (string "AstTypeFunction") | nothing | nothing = typeFromJSON (object o) | just (string "AstTypeReference") with lookup name o typeFromJSON (object o) | just (string "AstTypeReference") | just (string "nil") = Right nil typeFromJSON (object o) | just (string "AstTypeReference") | just (string "any") = Right any +typeFromJSON (object o) | just (string "AstTypeReference") | just (string "number") = Right number typeFromJSON (object o) | just (string "AstTypeReference") | _ = Left "Unknown referenced type" typeFromJSON (object o) | just (string "AstTypeUnion") with lookup types o diff --git a/prototyping/Luau/Type/ToString.agda b/prototyping/Luau/Type/ToString.agda index 698d6e8e..e7a7c1c0 100644 --- a/prototyping/Luau/Type/ToString.agda +++ b/prototyping/Luau/Type/ToString.agda @@ -1,7 +1,7 @@ module Luau.Type.ToString where open import FFI.Data.String using (String; _++_) -open import Luau.Type using (Type; nil; _⇒_; none; any; _∪_; _∩_; normalizeOptional) +open import Luau.Type using (Type; nil; _⇒_; none; any; number; _∪_; _∩_; normalizeOptional) {-# TERMINATING #-} typeToString : Type → String @@ -12,6 +12,7 @@ typeToString nil = "nil" typeToString (S ⇒ T) = "(" ++ (typeToString S) ++ ") -> " ++ (typeToString T) typeToString none = "none" typeToString any = "any" +typeToString number = "number" typeToString (S ∪ T) with normalizeOptional(S ∪ T) typeToString (S ∪ T) | ((S′ ⇒ T′) ∪ nil) = "(" ++ typeToString (S′ ⇒ T′) ++ ")?" typeToString (S ∪ T) | (S′ ∪ nil) = typeToString S′ ++ "?" diff --git a/prototyping/Luau/Value.agda b/prototyping/Luau/Value.agda index 4768f859..1086d39c 100644 --- a/prototyping/Luau/Value.agda +++ b/prototyping/Luau/Value.agda @@ -1,15 +1,16 @@ module Luau.Value where +open import Agda.Builtin.Float using (Float) open import Luau.Addr using (Addr) -open import Luau.Syntax using (Block; Expr; nil; addr) +open import Luau.Syntax using (Block; Expr; nil; addr; number) open import Luau.Var using (Var) data Value : Set where nil : Value addr : Addr → Value + number : Float → Value val : ∀ {a} → Value → Expr a val nil = nil val (addr a) = addr a - - +val (number x) = number x diff --git a/prototyping/Luau/Value/ToString.agda b/prototyping/Luau/Value/ToString.agda index 3cac3ee7..51c3fa78 100644 --- a/prototyping/Luau/Value/ToString.agda +++ b/prototyping/Luau/Value/ToString.agda @@ -1,10 +1,11 @@ module Luau.Value.ToString where open import Agda.Builtin.String using (String) -open import Luau.Value using (Value; nil; addr) +open import Agda.Builtin.Float using (primShowFloat) +open import Luau.Value using (Value; nil; addr; number) open import Luau.Addr.ToString using (addrToString) valueToString : Value → String valueToString nil = "nil" valueToString (addr a) = addrToString a - +valueToString (number x) = primShowFloat x diff --git a/prototyping/PrettyPrinter.agda b/prototyping/PrettyPrinter.agda index 4a6c7dd6..fdad32d6 100644 --- a/prototyping/PrettyPrinter.agda +++ b/prototyping/PrettyPrinter.agda @@ -30,4 +30,3 @@ runString txt | (Right value) = runJSON value main : IO ⊤ main = getContents >>= runString - diff --git a/prototyping/Properties/Step.agda b/prototyping/Properties/Step.agda index eeda4ef2..620dc618 100644 --- a/prototyping/Properties/Step.agda +++ b/prototyping/Properties/Step.agda @@ -3,11 +3,11 @@ module Properties.Step where open import Agda.Builtin.Equality using (_≡_; refl) open import FFI.Data.Maybe using (just; nothing) open import Luau.Heap using (Heap; _[_]; alloc; ok; function_is_end) -open import Luau.Syntax using (Block; Expr; nil; var; addr; function_is_end; block_is_end; _$_; local_←_; return; done; _∙_; name; fun; arg) +open import Luau.Syntax using (Block; Expr; nil; var; addr; function_is_end; block_is_end; _$_; local_←_; return; done; _∙_; name; fun; arg; number) open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app ; beta; function; block; return; done; local; subst) -open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; UnboundVariable; SEGV; app; block; local; return) +open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app; block; local; return) open import Luau.Substitution using (_[_/_]ᴮ) -open import Luau.Value using (nil; addr; val) +open import Luau.Value using (nil; addr; val; number) open import Properties.Remember using (remember; _,_) data StepResultᴮ {a} (H : Heap a) (B : Block a) : Set @@ -30,9 +30,11 @@ stepᴮ : ∀ {a} H B → StepResultᴮ {a} H B stepᴱ H nil = value nil refl stepᴱ H (var x) = error (UnboundVariable x) stepᴱ H (addr a) = value (addr a) refl +stepᴱ H (number x) = value (number x) refl stepᴱ H (M $ N) with stepᴱ H M stepᴱ H (M $ N) | step H′ M′ D = step H′ (M′ $ N) (app D) stepᴱ H (nil $ N) | value nil refl = error NilIsNotAFunction +stepᴱ H ((number _) $ N) | value (number x) refl = error (NumberIsNotAFunction x) stepᴱ H (addr a $ N) | value (addr a) refl with remember (H [ a ]) stepᴱ H (addr a $ N) | value (addr a) refl | (nothing , p) = error (app (SEGV a p)) stepᴱ H (addr a $ N) | value (addr a) refl | (just(function F is B end) , p) = step H (block fun F is (local arg F ← N) ∙ B end) (beta p) diff --git a/prototyping/Tests/PrettyPrinter/smoke_test/in.lua b/prototyping/Tests/PrettyPrinter/smoke_test/in.lua index 0d23bbb2..d26e3a0d 100644 --- a/prototyping/Tests/PrettyPrinter/smoke_test/in.lua +++ b/prototyping/Tests/PrettyPrinter/smoke_test/in.lua @@ -15,4 +15,5 @@ local b : nil = nil local c : (nil) -> nil = nil local d : (any & nil) = nil local e : any? = nil +local f : number = 123 return id2(nil2) diff --git a/prototyping/Tests/PrettyPrinter/smoke_test/out.txt b/prototyping/Tests/PrettyPrinter/smoke_test/out.txt index e663534a..34e0c4fe 100644 --- a/prototyping/Tests/PrettyPrinter/smoke_test/out.txt +++ b/prototyping/Tests/PrettyPrinter/smoke_test/out.txt @@ -15,4 +15,5 @@ local b : nil = nil local c : (nil) -> nil = nil local d : (any & nil) = nil local e : any? = nil +local f : number = 123.0 return id2(nil2) From 0b783d89327eb5de1d20223559aa6ac9111075b7 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey <403333+asajeffrey@users.noreply.github.com> Date: Fri, 18 Feb 2022 16:47:04 -0600 Subject: [PATCH 20/25] Add Properties.Equality to prototyping (#376) --- prototyping/Properties.agda | 2 ++ prototyping/Properties/Contradiction.agda | 9 +++++++++ prototyping/Properties/Equality.agda | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 prototyping/Properties/Contradiction.agda create mode 100644 prototyping/Properties/Equality.agda diff --git a/prototyping/Properties.agda b/prototyping/Properties.agda index b08a3a81..1a6f92fa 100644 --- a/prototyping/Properties.agda +++ b/prototyping/Properties.agda @@ -1,5 +1,7 @@ module Properties where +import Properties.Contradiction import Properties.Dec +import Properties.Equality import Properties.Step import Properties.Remember diff --git a/prototyping/Properties/Contradiction.agda b/prototyping/Properties/Contradiction.agda new file mode 100644 index 00000000..e9a92ad5 --- /dev/null +++ b/prototyping/Properties/Contradiction.agda @@ -0,0 +1,9 @@ +module Properties.Contradiction where + +data ⊥ : Set where + +¬ : Set → Set +¬ A = A → ⊥ + +CONTRADICTION : ∀ {A : Set} → ⊥ → A +CONTRADICTION () diff --git a/prototyping/Properties/Equality.agda b/prototyping/Properties/Equality.agda new file mode 100644 index 00000000..c027bee3 --- /dev/null +++ b/prototyping/Properties/Equality.agda @@ -0,0 +1,23 @@ +module Properties.Equality where + +open import Agda.Builtin.Equality using (_≡_; refl) +open import Properties.Contradiction using (¬) + +sym : ∀ {A : Set} {a b : A} → (a ≡ b) → (b ≡ a) +sym refl = refl + +trans : ∀ {A : Set} {a b c : A} → (a ≡ b) → (b ≡ c) → (a ≡ c) +trans refl refl = refl + +cong : ∀ {A B : Set} {a b : A} (f : A → B) → (a ≡ b) → (f a ≡ f b) +cong f refl = refl + +subst₁ : ∀ {A : Set} {a b : A} (F : A → Set) → (a ≡ b) → (F a) → (F b) +subst₁ F refl x = x + +subst₂ : ∀ {A B : Set} {a b : A} {c d : B} (F : A → B → Set) → (a ≡ b) → (c ≡ d) → (F a c) → (F b d) +subst₂ F refl refl x = x + +_≢_ : ∀ {A : Set} → A → A → Set +(a ≢ b) = ¬(a ≡ b) + From fc33b0c7026eea4f51af73fcf811ea5b3f3b497b Mon Sep 17 00:00:00 2001 From: Alan Jeffrey <403333+asajeffrey@users.noreply.github.com> Date: Fri, 18 Feb 2022 16:47:23 -0600 Subject: [PATCH 21/25] Fix evaluation rule for function application (#375) --- prototyping/Luau/OpSem.agda | 14 ++++++++++---- prototyping/Luau/RuntimeError.agda | 8 +++++--- prototyping/Luau/RuntimeError/ToString.agda | 7 ++++--- prototyping/Properties/Step.agda | 21 ++++++++++++--------- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/prototyping/Luau/OpSem.agda b/prototyping/Luau/OpSem.agda index 878b8bd0..89564769 100644 --- a/prototyping/Luau/OpSem.agda +++ b/prototyping/Luau/OpSem.agda @@ -23,17 +23,23 @@ data _⊢_⟶ᴱ_⊣_ where ------------------------------------------- H ⊢ (function F is B end) ⟶ᴱ (addr a) ⊣ H′ - app : ∀ {H H′ M M′ N} → + app₁ : ∀ {H H′ M M′ N} → H ⊢ M ⟶ᴱ M′ ⊣ H′ → ----------------------------- H ⊢ (M $ N) ⟶ᴱ (M′ $ N) ⊣ H′ - beta : ∀ {H M a F B} → + app₂ : ∀ {H H′ V N N′} → + + H ⊢ N ⟶ᴱ N′ ⊣ H′ → + ----------------------------- + H ⊢ (val V $ N) ⟶ᴱ (val V $ N′) ⊣ H′ + + beta : ∀ {H a F B V} → H [ a ] ≡ just(function F is B end) → - ----------------------------------------------------- - H ⊢ (addr a $ M) ⟶ᴱ (block (fun F) is local (arg F) ← M ∙ B end) ⊣ H + ----------------------------------------------------------------------------- + H ⊢ (addr a $ val V) ⟶ᴱ (block (fun F) is (B [ V / name(arg F) ]ᴮ) end) ⊣ H block : ∀ {H H′ B B′ b} → diff --git a/prototyping/Luau/RuntimeError.agda b/prototyping/Luau/RuntimeError.agda index f9e684b3..a54d48fe 100644 --- a/prototyping/Luau/RuntimeError.agda +++ b/prototyping/Luau/RuntimeError.agda @@ -5,16 +5,18 @@ open import Luau.Heap using (Heap; _[_]) open import FFI.Data.Maybe using (just; nothing) open import FFI.Data.String using (String) open import Luau.Syntax using (Block; Expr; nil; var; addr; block_is_end; _$_; local_←_; return; done; _∙_; number) +open import Luau.Value using (val) data RuntimeErrorᴮ {a} (H : Heap a) : Block a → Set data RuntimeErrorᴱ {a} (H : Heap a) : Expr a → Set data RuntimeErrorᴱ H where - NilIsNotAFunction : ∀ {M} → RuntimeErrorᴱ H (nil $ M) - NumberIsNotAFunction : ∀ n {M} → RuntimeErrorᴱ H ((number n) $ M) + NilIsNotAFunction : ∀ {V} → RuntimeErrorᴱ H (nil $ val V) + NumberIsNotAFunction : ∀ n {V} → RuntimeErrorᴱ H (number n $ val V) UnboundVariable : ∀ x → RuntimeErrorᴱ H (var x) SEGV : ∀ a → (H [ a ] ≡ nothing) → RuntimeErrorᴱ H (addr a) - app : ∀ {M N} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (M $ N) + app₁ : ∀ {M N} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (M $ N) + app₂ : ∀ {M N} → RuntimeErrorᴱ H N → RuntimeErrorᴱ H (M $ N) block : ∀ b {B} → RuntimeErrorᴮ H B → RuntimeErrorᴱ H (block b is B end) data RuntimeErrorᴮ H where diff --git a/prototyping/Luau/RuntimeError/ToString.agda b/prototyping/Luau/RuntimeError/ToString.agda index ac760fab..10c0fe08 100644 --- a/prototyping/Luau/RuntimeError/ToString.agda +++ b/prototyping/Luau/RuntimeError/ToString.agda @@ -2,7 +2,7 @@ module Luau.RuntimeError.ToString where open import Agda.Builtin.Float using (primShowFloat) open import FFI.Data.String using (String; _++_) -open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app; block) +open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app₁; app₂; block) open import Luau.Addr.ToString using (addrToString) open import Luau.Syntax.ToString using (exprToString) open import Luau.Var.ToString using (varToString) @@ -15,9 +15,10 @@ errToStringᴱ NilIsNotAFunction = "nil is not a function" errToStringᴱ (NumberIsNotAFunction n) = "number " ++ primShowFloat n ++ " is not a function" errToStringᴱ (UnboundVariable x) = "variable " ++ varToString x ++ " is unbound" errToStringᴱ (SEGV a x) = "address " ++ addrToString a ++ " is unallocated" -errToStringᴱ (app E) = errToStringᴱ E +errToStringᴱ (app₁ E) = errToStringᴱ E +errToStringᴱ (app₂ E) = errToStringᴱ E errToStringᴱ (block b E) = errToStringᴮ E ++ "\n in call of function " ++ varToString b errToStringᴮ (local x E) = errToStringᴱ E ++ "\n in definition of " ++ varToString (name x) errToStringᴮ (return E) = errToStringᴱ E ++ "\n in return statement" - \ No newline at end of file + diff --git a/prototyping/Properties/Step.agda b/prototyping/Properties/Step.agda index 620dc618..914c3468 100644 --- a/prototyping/Properties/Step.agda +++ b/prototyping/Properties/Step.agda @@ -4,8 +4,8 @@ open import Agda.Builtin.Equality using (_≡_; refl) open import FFI.Data.Maybe using (just; nothing) open import Luau.Heap using (Heap; _[_]; alloc; ok; function_is_end) open import Luau.Syntax using (Block; Expr; nil; var; addr; function_is_end; block_is_end; _$_; local_←_; return; done; _∙_; name; fun; arg; number) -open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app ; beta; function; block; return; done; local; subst) -open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app; block; local; return) +open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app₁ ; app₂ ; beta; function; block; return; done; local; subst) +open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app₁; app₂; block; local; return) open import Luau.Substitution using (_[_/_]ᴮ) open import Luau.Value using (nil; addr; val; number) open import Properties.Remember using (remember; _,_) @@ -32,13 +32,16 @@ stepᴱ H (var x) = error (UnboundVariable x) stepᴱ H (addr a) = value (addr a) refl stepᴱ H (number x) = value (number x) refl stepᴱ H (M $ N) with stepᴱ H M -stepᴱ H (M $ N) | step H′ M′ D = step H′ (M′ $ N) (app D) -stepᴱ H (nil $ N) | value nil refl = error NilIsNotAFunction -stepᴱ H ((number _) $ N) | value (number x) refl = error (NumberIsNotAFunction x) -stepᴱ H (addr a $ N) | value (addr a) refl with remember (H [ a ]) -stepᴱ H (addr a $ N) | value (addr a) refl | (nothing , p) = error (app (SEGV a p)) -stepᴱ H (addr a $ N) | value (addr a) refl | (just(function F is B end) , p) = step H (block fun F is (local arg F ← N) ∙ B end) (beta p) -stepᴱ H (M $ N) | error E = error (app E) +stepᴱ H (M $ N) | step H′ M′ D = step H′ (M′ $ N) (app₁ D) +stepᴱ H (_ $ N) | value V refl with stepᴱ H N +stepᴱ H (_ $ N) | value V refl | step H′ N′ s = step H′ (val V $ N′) (app₂ s) +stepᴱ H (_ $ _) | value nil refl | value W refl = error NilIsNotAFunction +stepᴱ H (_ $ _) | value (number n) refl | value W refl = error (NumberIsNotAFunction n) +stepᴱ H (_ $ _) | value (addr a) refl | value W refl with remember (H [ a ]) +stepᴱ H (_ $ _) | value (addr a) refl | value W refl | (nothing , p) = error (app₁ (SEGV a p)) +stepᴱ H (_ $ _) | value (addr a) refl | value W refl | (just(function F is B end) , p) = step H (block fun F is B [ W / name (arg F) ]ᴮ end) (beta p) +stepᴱ H (M $ N) | value V p | error E = error (app₂ E) +stepᴱ H (M $ N) | error E = error (app₁ E) stepᴱ H (block b is B end) with stepᴮ H B stepᴱ H (block b is B end) | step H′ B′ D = step H′ (block b is B′ end) (block D) stepᴱ H (block b is (return _ ∙ B′) end) | return V refl = step H (val V) return From 3c42b3a01349b78c9c1abc1e15c67b390754f2e2 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Sat, 19 Feb 2022 10:57:29 -0800 Subject: [PATCH 22/25] Revert "Mark singleton types RFC as implemented (#370)" (#378) This reverts commit 731e197757ead8908e558cfc63f9d518f53940d5. --- rfcs/STATUS.md | 6 ++++++ rfcs/syntax-singleton-types.md | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rfcs/STATUS.md b/rfcs/STATUS.md index 742cb501..72db2003 100644 --- a/rfcs/STATUS.md +++ b/rfcs/STATUS.md @@ -23,6 +23,12 @@ This document tracks unimplemented RFCs. **Status**: Implemented but depends on new transaction log implementation that is not fully live yet. +## Singleton types + +[RFC: Singleton types](https://github.com/Roblox/luau/blob/master/rfcs/syntax-singleton-types.md) + +**Status**: Implemented but not fully rolled out yet. + ## Nil-forgiving operator [RFC: nil-forgiving postfix operator !](https://github.com/Roblox/luau/blob/master/rfcs/syntax-nil-forgiving-operator.md) diff --git a/rfcs/syntax-singleton-types.md b/rfcs/syntax-singleton-types.md index 2c1f5442..26ea3028 100644 --- a/rfcs/syntax-singleton-types.md +++ b/rfcs/syntax-singleton-types.md @@ -2,8 +2,6 @@ > Note: this RFC was adapted from an internal proposal that predates RFC process -**Status**: Implemented - ## Summary Introduce a new kind of type variable, called singleton types. They are just like normal types but has the capability to represent a constant runtime value as a type. From 1334db600f9b1923b627f58a47cab76c7c370aff Mon Sep 17 00:00:00 2001 From: James Napora <85808999+TheGreatSageEqualToHeaven@users.noreply.github.com> Date: Tue, 22 Feb 2022 20:24:15 +0100 Subject: [PATCH 23/25] Update grammar.md (#379) --- docs/_pages/grammar.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_pages/grammar.md b/docs/_pages/grammar.md index 585bac73..ffb1dbf6 100644 --- a/docs/_pages/grammar.md +++ b/docs/_pages/grammar.md @@ -68,8 +68,8 @@ SingletonType = STRING | 'true' | 'false' Type = SimpleType ['?'] | - SimpleType ['|' Type] | - SimpleType ['&' Type] + Type ['|' Type] | + Type ['&' Type] GenericTypePackParameter = NAME '...' ['=' (TypePack | VariadicTypePack | GenericTypePack)] GenericTypeParameterList = NAME ['=' Type] [',' GenericTypeParameterList] | GenericTypePackParameter {',' GenericTypePackParameter} From cd18adc20ecb805b8eeb770a9e5ef8e0cd123734 Mon Sep 17 00:00:00 2001 From: Lily Brown Date: Tue, 22 Feb 2022 15:52:56 -0800 Subject: [PATCH 24/25] Prototyping: binary operations (#377) Adds support for binary operations on numbers. --- prototyping/Examples/Run.agda | 5 ++- prototyping/Luau/OpSem.agda | 31 ++++++++++++++++--- prototyping/Luau/RuntimeError.agda | 9 ++++-- prototyping/Luau/RuntimeError/ToString.agda | 9 ++++-- prototyping/Luau/RuntimeType.agda | 13 ++++++++ prototyping/Luau/RuntimeType/ToString.agda | 9 ++++++ prototyping/Luau/Substitution.agda | 3 +- prototyping/Luau/Syntax.agda | 7 +++++ prototyping/Luau/Syntax/FromJSON.agda | 26 +++++++++++++++- prototyping/Luau/Syntax/ToString.agda | 9 +++++- prototyping/Properties/Step.agda | 23 +++++++++++--- .../Tests/Interpreter/binary_exps/in.lua | 1 + .../Tests/Interpreter/binary_exps/out.txt | 1 + 13 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 prototyping/Luau/RuntimeType.agda create mode 100644 prototyping/Luau/RuntimeType/ToString.agda create mode 100644 prototyping/Tests/Interpreter/binary_exps/in.lua create mode 100644 prototyping/Tests/Interpreter/binary_exps/out.txt diff --git a/prototyping/Examples/Run.agda b/prototyping/Examples/Run.agda index 3cac564d..534bb122 100644 --- a/prototyping/Examples/Run.agda +++ b/prototyping/Examples/Run.agda @@ -3,7 +3,7 @@ module Examples.Run where open import Agda.Builtin.Equality using (_≡_; refl) -open import Luau.Syntax using (nil; var; _$_; function_is_end; return; _∙_; done; _⟨_⟩; number) +open import Luau.Syntax using (nil; var; _$_; function_is_end; return; _∙_; done; _⟨_⟩; number; binexp; +) open import Luau.Value using (nil; number) open import Luau.Run using (run; return) open import Luau.Heap using (lookup-next; next-emp; lookup-next-emp) @@ -16,3 +16,6 @@ ex1 = refl ex2 : (run (function "fn" ⟨ var "x" ⟩ is return (number 123.0) ∙ done end ∙ return (var "fn" $ nil) ∙ done) ≡ return (number 123.0) _) ex2 = refl + +ex3 : (run (function "fn" ⟨ var "x" ⟩ is return (binexp (number 1.0) + (number 2.0)) ∙ done end ∙ return (var "fn" $ nil) ∙ done) ≡ return (number 3.0) _) +ex3 = refl diff --git a/prototyping/Luau/OpSem.agda b/prototyping/Luau/OpSem.agda index 89564769..bfc63a4a 100644 --- a/prototyping/Luau/OpSem.agda +++ b/prototyping/Luau/OpSem.agda @@ -1,11 +1,18 @@ module Luau.OpSem where open import Agda.Builtin.Equality using (_≡_) +open import Agda.Builtin.Float using (Float; primFloatPlus; primFloatMinus; primFloatTimes; primFloatDiv) open import FFI.Data.Maybe using (just) open import Luau.Heap using (Heap; _≡_⊕_↦_; _[_]; function_is_end) open import Luau.Substitution using (_[_/_]ᴮ) -open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; name; fun; arg) -open import Luau.Value using (addr; val) +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; name; fun; arg; binexp; BinaryOperator; +; -; *; /; number) +open import Luau.Value using (addr; val; number) + +evalBinOp : Float → BinaryOperator → Float → Float +evalBinOp x + y = primFloatPlus x y +evalBinOp x - y = primFloatMinus x y +evalBinOp x * y = primFloatTimes x y +evalBinOp x / y = primFloatDiv x y data _⊢_⟶ᴮ_⊣_ {a} : Heap a → Block a → Block a → Heap a → Set data _⊢_⟶ᴱ_⊣_ {a} : Heap a → Expr a → Expr a → Heap a → Set @@ -57,6 +64,24 @@ data _⊢_⟶ᴱ_⊣_ where --------------------------------- H ⊢ (block b is done end) ⟶ᴱ nil ⊣ H + binOpEval : + ∀ {H x op y} → + -------------------------------------------------------------------------- + H ⊢ (binexp (number x) op (number y)) ⟶ᴱ (number (evalBinOp x op y)) ⊣ H + + binOp₁ : + ∀ {H H′ x x′ op y} → + H ⊢ x ⟶ᴱ x′ ⊣ H′ → + --------------------------------------------- + H ⊢ (binexp x op y) ⟶ᴱ (binexp x′ op y) ⊣ H′ + + binOp₂ : + ∀ {H H′ x op y y′} → + H ⊢ y ⟶ᴱ y′ ⊣ H′ → + --------------------------------------------- + H ⊢ (binexp x op y) ⟶ᴱ (binexp x op y′) ⊣ H′ + + data _⊢_⟶ᴮ_⊣_ where local : ∀ {H H′ x M M′ B} → @@ -94,5 +119,3 @@ data _⊢_⟶*_⊣_ {a} : Heap a → Block a → Block a → Heap a → Set wher H′ ⊢ B′ ⟶* B″ ⊣ H″ → ------------------ H ⊢ B ⟶* B″ ⊣ H″ - - diff --git a/prototyping/Luau/RuntimeError.agda b/prototyping/Luau/RuntimeError.agda index a54d48fe..b479a72a 100644 --- a/prototyping/Luau/RuntimeError.agda +++ b/prototyping/Luau/RuntimeError.agda @@ -4,20 +4,23 @@ open import Agda.Builtin.Equality using (_≡_) open import Luau.Heap using (Heap; _[_]) open import FFI.Data.Maybe using (just; nothing) open import FFI.Data.String using (String) -open import Luau.Syntax using (Block; Expr; nil; var; addr; block_is_end; _$_; local_←_; return; done; _∙_; number) +open import Luau.Syntax using (Block; Expr; nil; var; addr; block_is_end; _$_; local_←_; return; done; _∙_; number; binexp) +open import Luau.RuntimeType using (RuntimeType; valueType) open import Luau.Value using (val) +open import Properties.Equality using (_≢_) data RuntimeErrorᴮ {a} (H : Heap a) : Block a → Set data RuntimeErrorᴱ {a} (H : Heap a) : Expr a → Set data RuntimeErrorᴱ H where - NilIsNotAFunction : ∀ {V} → RuntimeErrorᴱ H (nil $ val V) - NumberIsNotAFunction : ∀ n {V} → RuntimeErrorᴱ H (number n $ val V) + TypeMismatch : ∀ t v → (t ≢ valueType v) → RuntimeErrorᴱ H (val v) UnboundVariable : ∀ x → RuntimeErrorᴱ H (var x) SEGV : ∀ a → (H [ a ] ≡ nothing) → RuntimeErrorᴱ H (addr a) app₁ : ∀ {M N} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (M $ N) app₂ : ∀ {M N} → RuntimeErrorᴱ H N → RuntimeErrorᴱ H (M $ N) block : ∀ b {B} → RuntimeErrorᴮ H B → RuntimeErrorᴱ H (block b is B end) + bin₁ : ∀ {M N op} → RuntimeErrorᴱ H M → RuntimeErrorᴱ H (binexp M op N) + bin₂ : ∀ {M N op} → RuntimeErrorᴱ H N → RuntimeErrorᴱ H (binexp M op N) data RuntimeErrorᴮ H where local : ∀ x {M B} → RuntimeErrorᴱ H M → RuntimeErrorᴮ H (local x ← M ∙ B) diff --git a/prototyping/Luau/RuntimeError/ToString.agda b/prototyping/Luau/RuntimeError/ToString.agda index 10c0fe08..d5528c8b 100644 --- a/prototyping/Luau/RuntimeError/ToString.agda +++ b/prototyping/Luau/RuntimeError/ToString.agda @@ -2,22 +2,25 @@ module Luau.RuntimeError.ToString where open import Agda.Builtin.Float using (primShowFloat) open import FFI.Data.String using (String; _++_) -open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app₁; app₂; block) +open import Luau.RuntimeError using (RuntimeErrorᴮ; RuntimeErrorᴱ; local; return; TypeMismatch; UnboundVariable; SEGV; app₁; app₂; block; bin₁; bin₂) +open import Luau.RuntimeType.ToString using (runtimeTypeToString) open import Luau.Addr.ToString using (addrToString) open import Luau.Syntax.ToString using (exprToString) open import Luau.Var.ToString using (varToString) +open import Luau.Value.ToString using (valueToString) open import Luau.Syntax using (name; _$_) errToStringᴱ : ∀ {a H B} → RuntimeErrorᴱ {a} H B → String errToStringᴮ : ∀ {a H B} → RuntimeErrorᴮ {a} H B → String -errToStringᴱ NilIsNotAFunction = "nil is not a function" -errToStringᴱ (NumberIsNotAFunction n) = "number " ++ primShowFloat n ++ " is not a function" errToStringᴱ (UnboundVariable x) = "variable " ++ varToString x ++ " is unbound" errToStringᴱ (SEGV a x) = "address " ++ addrToString a ++ " is unallocated" errToStringᴱ (app₁ E) = errToStringᴱ E errToStringᴱ (app₂ E) = errToStringᴱ E +errToStringᴱ (bin₁ E) = errToStringᴱ E +errToStringᴱ (bin₂ E) = errToStringᴱ E errToStringᴱ (block b E) = errToStringᴮ E ++ "\n in call of function " ++ varToString b +errToStringᴱ (TypeMismatch t v _) = "value " ++ valueToString v ++ " is not a " ++ runtimeTypeToString t errToStringᴮ (local x E) = errToStringᴱ E ++ "\n in definition of " ++ varToString (name x) errToStringᴮ (return E) = errToStringᴱ E ++ "\n in return statement" diff --git a/prototyping/Luau/RuntimeType.agda b/prototyping/Luau/RuntimeType.agda new file mode 100644 index 00000000..25c0283a --- /dev/null +++ b/prototyping/Luau/RuntimeType.agda @@ -0,0 +1,13 @@ +module Luau.RuntimeType where + +open import Luau.Value using (Value; nil; addr; number) + +data RuntimeType : Set where + function : RuntimeType + number : RuntimeType + nil : RuntimeType + +valueType : Value → RuntimeType +valueType nil = nil +valueType (addr x) = function +valueType (number x) = number diff --git a/prototyping/Luau/RuntimeType/ToString.agda b/prototyping/Luau/RuntimeType/ToString.agda new file mode 100644 index 00000000..be67ee0c --- /dev/null +++ b/prototyping/Luau/RuntimeType/ToString.agda @@ -0,0 +1,9 @@ +module Luau.RuntimeType.ToString where + +open import FFI.Data.String using (String) +open import Luau.RuntimeType using (RuntimeType; function; number; nil) + +runtimeTypeToString : RuntimeType → String +runtimeTypeToString function = "function" +runtimeTypeToString number = "number" +runtimeTypeToString nil = "nil" diff --git a/prototyping/Luau/Substitution.agda b/prototyping/Luau/Substitution.agda index e364a000..61287d36 100644 --- a/prototyping/Luau/Substitution.agda +++ b/prototyping/Luau/Substitution.agda @@ -1,6 +1,6 @@ module Luau.Substitution where -open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; _⟨_⟩ ; name; fun; arg; number) +open import Luau.Syntax using (Expr; Stat; Block; nil; addr; var; function_is_end; _$_; block_is_end; local_←_; _∙_; done; return; _⟨_⟩ ; name; fun; arg; number; binexp) open import Luau.Value using (Value; val) open import Luau.Var using (Var; _≡ⱽ_) open import Properties.Dec using (Dec; yes; no) @@ -17,6 +17,7 @@ addr a [ v / x ]ᴱ = addr a (M $ N) [ v / x ]ᴱ = (M [ v / x ]ᴱ) $ (N [ v / x ]ᴱ) function F is C end [ v / x ]ᴱ = function F is C [ v / x ]ᴮunless (x ≡ⱽ name(arg F)) end block b is C end [ v / x ]ᴱ = block b is C [ v / x ]ᴮ end +(binexp e₁ op e₂) [ v / x ]ᴱ = binexp (e₁ [ v / x ]ᴱ) op (e₂ [ v / x ]ᴱ) (function F is C end ∙ B) [ v / x ]ᴮ = function F is (C [ v / x ]ᴮunless (x ≡ⱽ name(arg F))) end ∙ (B [ v / x ]ᴮunless (x ≡ⱽ fun F)) (local y ← M ∙ B) [ v / x ]ᴮ = local y ← (M [ v / x ]ᴱ) ∙ (B [ v / x ]ᴮunless (x ≡ⱽ name y)) diff --git a/prototyping/Luau/Syntax.agda b/prototyping/Luau/Syntax.agda index 1313456b..c1c31e42 100644 --- a/prototyping/Luau/Syntax.agda +++ b/prototyping/Luau/Syntax.agda @@ -33,6 +33,12 @@ arg : ∀ {a} → FunDec a → VarDec a arg (f ⟨ x ⟩∈ T) = x arg (f ⟨ x ⟩) = x +data BinaryOperator : Set where + + : BinaryOperator + - : BinaryOperator + * : BinaryOperator + / : BinaryOperator + data Block (a : Annotated) : Set data Stat (a : Annotated) : Set data Expr (a : Annotated) : Set @@ -54,3 +60,4 @@ data Expr a where function_is_end : FunDec a → Block a → Expr a block_is_end : Var → Block a → Expr a number : Float → Expr a + binexp : Expr a → BinaryOperator → Expr a → Expr a diff --git a/prototyping/Luau/Syntax/FromJSON.agda b/prototyping/Luau/Syntax/FromJSON.agda index 8191e9e7..a2c9a42f 100644 --- a/prototyping/Luau/Syntax/FromJSON.agda +++ b/prototyping/Luau/Syntax/FromJSON.agda @@ -1,6 +1,6 @@ module Luau.Syntax.FromJSON where -open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; var_∈_; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec; number) +open import Luau.Syntax using (Block; Stat ; Expr; nil; _$_; var; var_∈_; function_is_end; _⟨_⟩; local_←_; return; done; _∙_; maybe; VarDec; number; binexp; BinaryOperator; +; -; *; /) open import Luau.Type.FromJSON using (typeFromJSON) open import Agda.Builtin.List using (List; _∷_; []) @@ -23,6 +23,9 @@ type = fromString "type" value = fromString "value" values = fromString "values" vars = fromString "vars" +op = fromString "op" +left = fromString "left" +right = fromString "right" data Lookup : Set where _,_ : String → Value → Lookup @@ -34,6 +37,8 @@ lookupIn (key ∷ keys) obj with lookup (fromString key) obj lookupIn (key ∷ keys) obj | nothing = lookupIn keys obj lookupIn (key ∷ keys) obj | just value = (key , value) +binOpFromJSON : Value → Either String BinaryOperator +binOpFromString : String → Either String BinaryOperator varDecFromJSON : Value → Either String (VarDec maybe) varDecFromObject : Object → Either String (VarDec maybe) exprFromJSON : Value → Either String (Expr maybe) @@ -43,6 +48,15 @@ statFromObject : Object → Either String (Stat maybe) blockFromJSON : Value → Either String (Block maybe) blockFromArray : Array → Either String (Block maybe) +binOpFromJSON (string s) = binOpFromString s +binOpFromJSON val = Left "Binary operator not a string" + +binOpFromString "Add" = Right + +binOpFromString "Sub" = Right - +binOpFromString "Mul" = Right * +binOpFromString "Div" = Right / +binOpFromString s = Left ("'" ++ s ++ "' is not a valid operator") + varDecFromJSON (object arg) = varDecFromObject arg varDecFromJSON val = Left "VarDec not an object" @@ -89,6 +103,15 @@ exprFromObject obj | just (string "AstExprConstantNumber") with lookup value obj exprFromObject obj | just (string "AstExprConstantNumber") | just (FFI.Data.Aeson.Value.number x) = Right (number (toFloat x)) exprFromObject obj | just (string "AstExprConstantNumber") | just _ = Left "AstExprConstantNumber value is not a number" exprFromObject obj | just (string "AstExprConstantNumber") | nothing = Left "AstExprConstantNumber missing value" +exprFromObject obj | just (string "AstExprBinary") with lookup op obj | lookup left obj | lookup right obj +exprFromObject obj | just (string "AstExprBinary") | just o | just l | just r with binOpFromJSON o | exprFromJSON l | exprFromJSON r +exprFromObject obj | just (string "AstExprBinary") | just o | just l | just r | Right o′ | Right l′ | Right r′ = Right (binexp l′ o′ r′) +exprFromObject obj | just (string "AstExprBinary") | just o | just l | just r | Left err | _ | _ = Left err +exprFromObject obj | just (string "AstExprBinary") | just o | just l | just r | _ | Left err | _ = Left err +exprFromObject obj | just (string "AstExprBinary") | just o | just l | just r | _ | _ | Left err = Left err +exprFromObject obj | just (string "AstExprBinary") | nothing | _ | _ = Left "Missing 'op' in AstExprBinary" +exprFromObject obj | just (string "AstExprBinary") | _ | nothing | _ = Left "Missing 'left' in AstExprBinary" +exprFromObject obj | just (string "AstExprBinary") | _ | _ | nothing = Left "Missing 'right' in AstExprBinary" exprFromObject obj | just (string ty) = Left ("TODO: Unsupported AstExpr " ++ ty) exprFromObject obj | just _ = Left "AstExpr type not a string" exprFromObject obj | nothing = Left "AstExpr missing type" @@ -146,3 +169,4 @@ blockFromArray arr | just value | Left err = Left err blockFromArray arr | just value | Right S with blockFromArray(tail arr) blockFromArray arr | just value | Right S | Left err = Left (err) blockFromArray arr | just value | Right S | Right B = Right (S ∙ B) + \ No newline at end of file diff --git a/prototyping/Luau/Syntax/ToString.agda b/prototyping/Luau/Syntax/ToString.agda index a20b360b..4a3061ba 100644 --- a/prototyping/Luau/Syntax/ToString.agda +++ b/prototyping/Luau/Syntax/ToString.agda @@ -1,7 +1,7 @@ module Luau.Syntax.ToString where open import Agda.Builtin.Float using (primShowFloat) -open import Luau.Syntax using (Block; Stat; Expr; VarDec; FunDec; nil; var; var_∈_; addr; _$_; function_is_end; return; local_←_; _∙_; done; block_is_end; _⟨_⟩; _⟨_⟩∈_; number) +open import Luau.Syntax using (Block; Stat; Expr; VarDec; FunDec; nil; var; var_∈_; addr; _$_; function_is_end; return; local_←_; _∙_; done; block_is_end; _⟨_⟩; _⟨_⟩∈_; number; BinaryOperator; +; -; *; /; binexp) open import FFI.Data.String using (String; _++_) open import Luau.Addr.ToString using (addrToString) open import Luau.Type.ToString using (typeToString) @@ -17,6 +17,12 @@ funDecToString ("" ⟨ x ⟩) = "function(" ++ varDecToString x ++ ")" funDecToString (f ⟨ x ⟩∈ T) = "function " ++ varToString f ++ "(" ++ varDecToString x ++ "): " ++ typeToString T funDecToString (f ⟨ x ⟩) = "function " ++ varToString f ++ "(" ++ varDecToString x ++ ")" +binOpToString : BinaryOperator → String +binOpToString + = "+" +binOpToString - = "-" +binOpToString * = "*" +binOpToString / = "/" + exprToString′ : ∀ {a} → String → Expr a → String statToString′ : ∀ {a} → String → Stat a → String blockToString′ : ∀ {a} → String → Block a → String @@ -38,6 +44,7 @@ exprToString′ lb (block b is B end) = " " ++ (blockToString′ (lb ++ " ") B) ++ lb ++ "end)()" exprToString′ lb (number x) = primShowFloat x +exprToString′ lb (binexp x op y) = exprToString′ lb x ++ " " ++ binOpToString op ++ " " ++ exprToString′ lb y statToString′ lb (function F is B end) = "local " ++ funDecToString F ++ lb ++ diff --git a/prototyping/Properties/Step.agda b/prototyping/Properties/Step.agda index 914c3468..2a0978ff 100644 --- a/prototyping/Properties/Step.agda +++ b/prototyping/Properties/Step.agda @@ -1,11 +1,13 @@ module Properties.Step where open import Agda.Builtin.Equality using (_≡_; refl) +open import Agda.Builtin.Float using (primFloatPlus; primFloatMinus; primFloatTimes; primFloatDiv) open import FFI.Data.Maybe using (just; nothing) open import Luau.Heap using (Heap; _[_]; alloc; ok; function_is_end) -open import Luau.Syntax using (Block; Expr; nil; var; addr; function_is_end; block_is_end; _$_; local_←_; return; done; _∙_; name; fun; arg; number) -open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app₁ ; app₂ ; beta; function; block; return; done; local; subst) -open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; NilIsNotAFunction; NumberIsNotAFunction; UnboundVariable; SEGV; app₁; app₂; block; local; return) +open import Luau.Syntax using (Block; Expr; nil; var; addr; function_is_end; block_is_end; _$_; local_←_; return; done; _∙_; name; fun; arg; number; binexp; +) +open import Luau.OpSem using (_⊢_⟶ᴱ_⊣_; _⊢_⟶ᴮ_⊣_; app₁ ; app₂ ; beta; function; block; return; done; local; subst; binOpEval; evalBinOp; binOp₁; binOp₂) +open import Luau.RuntimeError using (RuntimeErrorᴱ; RuntimeErrorᴮ; TypeMismatch; UnboundVariable; SEGV; app₁; app₂; block; local; return; bin₁; bin₂) +open import Luau.RuntimeType using (function; number) open import Luau.Substitution using (_[_/_]ᴮ) open import Luau.Value using (nil; addr; val; number) open import Properties.Remember using (remember; _,_) @@ -35,8 +37,8 @@ stepᴱ H (M $ N) with stepᴱ H M stepᴱ H (M $ N) | step H′ M′ D = step H′ (M′ $ N) (app₁ D) stepᴱ H (_ $ N) | value V refl with stepᴱ H N stepᴱ H (_ $ N) | value V refl | step H′ N′ s = step H′ (val V $ N′) (app₂ s) -stepᴱ H (_ $ _) | value nil refl | value W refl = error NilIsNotAFunction -stepᴱ H (_ $ _) | value (number n) refl | value W refl = error (NumberIsNotAFunction n) +stepᴱ H (_ $ _) | value nil refl | value W refl = error (app₁ (TypeMismatch function nil λ())) +stepᴱ H (_ $ _) | value (number n) refl | value W refl = error (app₁ (TypeMismatch function (number n) λ())) stepᴱ H (_ $ _) | value (addr a) refl | value W refl with remember (H [ a ]) stepᴱ H (_ $ _) | value (addr a) refl | value W refl | (nothing , p) = error (app₁ (SEGV a p)) stepᴱ H (_ $ _) | value (addr a) refl | value W refl | (just(function F is B end) , p) = step H (block fun F is B [ W / name (arg F) ]ᴮ end) (beta p) @@ -49,6 +51,17 @@ stepᴱ H (block b is done end) | done refl = step H nil done stepᴱ H (block b is B end) | error E = error (block b E) stepᴱ H (function F is C end) with alloc H (function F is C end) stepᴱ H function F is C end | ok a H′ p = step H′ (addr a) (function p) +stepᴱ H (binexp x op y) with stepᴱ H x +stepᴱ H (binexp x op y) | value x′ refl with stepᴱ H y +stepᴱ H (binexp x op y) | value (number x′) refl | value (number y′) refl = step H (number (evalBinOp x′ op y′)) binOpEval +stepᴱ H (binexp x op y) | value (number x′) refl | step H′ y′ s = step H′ (binexp (number x′) op y′) (binOp₂ s) +stepᴱ H (binexp x op y) | value (number x′) refl | error E = error (bin₂ E) +stepᴱ H (binexp x op y) | value nil refl | _ = error (bin₁ (TypeMismatch number nil λ())) +stepᴱ H (binexp x op y) | _ | value nil refl = error (bin₂ (TypeMismatch number nil λ())) +stepᴱ H (binexp x op y) | value (addr a) refl | _ = error (bin₁ (TypeMismatch number (addr a) λ())) +stepᴱ H (binexp x op y) | _ | value (addr a) refl = error (bin₂ (TypeMismatch number (addr a) λ())) +stepᴱ H (binexp x op y) | step H′ x′ s = step H′ (binexp x′ op y) (binOp₁ s) +stepᴱ H (binexp x op y) | error E = error (bin₁ E) stepᴮ H (function F is C end ∙ B) with alloc H (function F is C end) stepᴮ H (function F is C end ∙ B) | ok a H′ p = step H′ (B [ addr a / fun F ]ᴮ) (function p) diff --git a/prototyping/Tests/Interpreter/binary_exps/in.lua b/prototyping/Tests/Interpreter/binary_exps/in.lua new file mode 100644 index 00000000..0750f9e6 --- /dev/null +++ b/prototyping/Tests/Interpreter/binary_exps/in.lua @@ -0,0 +1 @@ +return 1 + 2 - 2 * 2 / 2 diff --git a/prototyping/Tests/Interpreter/binary_exps/out.txt b/prototyping/Tests/Interpreter/binary_exps/out.txt new file mode 100644 index 00000000..d3827e75 --- /dev/null +++ b/prototyping/Tests/Interpreter/binary_exps/out.txt @@ -0,0 +1 @@ +1.0 From 0bc7c51afc3e4ce008176f291dc00e207a27571d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petri=20H=C3=A4kkinen?= Date: Wed, 23 Feb 2022 20:03:58 +0200 Subject: [PATCH 25/25] Lua API: add return types to table getters (#389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Petri Häkkinen --- VM/include/lua.h | 10 +++++----- VM/src/lapi.cpp | 20 +++++++++---------- tests/Conformance.test.cpp | 41 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/VM/include/lua.h b/VM/include/lua.h index c5dcef25..5f39cb3d 100644 --- a/VM/include/lua.h +++ b/VM/include/lua.h @@ -178,11 +178,11 @@ LUA_API int lua_pushthread(lua_State* L); /* ** get functions (Lua -> stack) */ -LUA_API void lua_gettable(lua_State* L, int idx); -LUA_API void lua_getfield(lua_State* L, int idx, const char* k); -LUA_API void lua_rawgetfield(lua_State* L, int idx, const char* k); -LUA_API void lua_rawget(lua_State* L, int idx); -LUA_API void lua_rawgeti(lua_State* L, int idx, int n); +LUA_API int lua_gettable(lua_State* L, int idx); +LUA_API int lua_getfield(lua_State* L, int idx, const char* k); +LUA_API int lua_rawgetfield(lua_State* L, int idx, const char* k); +LUA_API int lua_rawget(lua_State* L, int idx); +LUA_API int lua_rawgeti(lua_State* L, int idx, int n); LUA_API void lua_createtable(lua_State* L, int narr, int nrec); LUA_API void lua_setreadonly(lua_State* L, int idx, int enabled); diff --git a/VM/src/lapi.cpp b/VM/src/lapi.cpp index 29d5f397..39c76e08 100644 --- a/VM/src/lapi.cpp +++ b/VM/src/lapi.cpp @@ -659,16 +659,16 @@ int lua_pushthread(lua_State* L) ** get functions (Lua -> stack) */ -void lua_gettable(lua_State* L, int idx) +int lua_gettable(lua_State* L, int idx) { luaC_checkthreadsleep(L); StkId t = index2addr(L, idx); api_checkvalidindex(L, t); luaV_gettable(L, t, L->top - 1, L->top - 1); - return; + return ttype(L->top - 1); } -void lua_getfield(lua_State* L, int idx, const char* k) +int lua_getfield(lua_State* L, int idx, const char* k) { luaC_checkthreadsleep(L); StkId t = index2addr(L, idx); @@ -677,10 +677,10 @@ void lua_getfield(lua_State* L, int idx, const char* k) setsvalue(L, &key, luaS_new(L, k)); luaV_gettable(L, t, &key, L->top); api_incr_top(L); - return; + return ttype(L->top - 1); } -void lua_rawgetfield(lua_State* L, int idx, const char* k) +int lua_rawgetfield(lua_State* L, int idx, const char* k) { luaC_checkthreadsleep(L); StkId t = index2addr(L, idx); @@ -689,26 +689,26 @@ void lua_rawgetfield(lua_State* L, int idx, const char* k) setsvalue(L, &key, luaS_new(L, k)); setobj2s(L, L->top, luaH_getstr(hvalue(t), tsvalue(&key))); api_incr_top(L); - return; + return ttype(L->top - 1); } -void lua_rawget(lua_State* L, int idx) +int lua_rawget(lua_State* L, int idx) { luaC_checkthreadsleep(L); StkId t = index2addr(L, idx); api_check(L, ttistable(t)); setobj2s(L, L->top - 1, luaH_get(hvalue(t), L->top - 1)); - return; + return ttype(L->top - 1); } -void lua_rawgeti(lua_State* L, int idx, int n) +int lua_rawgeti(lua_State* L, int idx, int n) { luaC_checkthreadsleep(L); StkId t = index2addr(L, idx); api_check(L, ttistable(t)); setobj2s(L, L->top, luaH_getnum(hvalue(t), n)); api_incr_top(L); - return; + return ttype(L->top - 1); } void lua_createtable(lua_State* L, int narray, int nrec) diff --git a/tests/Conformance.test.cpp b/tests/Conformance.test.cpp index b09c1efb..23e90f4d 100644 --- a/tests/Conformance.test.cpp +++ b/tests/Conformance.test.cpp @@ -712,6 +712,47 @@ TEST_CASE("Reference") CHECK(dtorhits == 2); } +TEST_CASE("ApiTables") +{ + StateRef globalState(luaL_newstate(), lua_close); + lua_State* L = globalState.get(); + + lua_newtable(L); + lua_pushnumber(L, 123.0); + lua_setfield(L, -2, "key"); + lua_pushstring(L, "test"); + lua_rawseti(L, -2, 5); + + // lua_gettable + lua_pushstring(L, "key"); + CHECK(lua_gettable(L, -2) == LUA_TNUMBER); + CHECK(lua_tonumber(L, -1) == 123.0); + lua_pop(L, 1); + + // lua_getfield + CHECK(lua_getfield(L, -1, "key") == LUA_TNUMBER); + CHECK(lua_tonumber(L, -1) == 123.0); + lua_pop(L, 1); + + // lua_rawgetfield + CHECK(lua_rawgetfield(L, -1, "key") == LUA_TNUMBER); + CHECK(lua_tonumber(L, -1) == 123.0); + lua_pop(L, 1); + + // lua_rawget + lua_pushstring(L, "key"); + CHECK(lua_rawget(L, -2) == LUA_TNUMBER); + CHECK(lua_tonumber(L, -1) == 123.0); + lua_pop(L, 1); + + // lua_rawgeti + CHECK(lua_rawgeti(L, -1, 5) == LUA_TSTRING); + CHECK(strcmp(lua_tostring(L, -1), "test") == 0); + lua_pop(L, 1); + + lua_pop(L, 1); +} + TEST_CASE("ApiFunctionCalls") { StateRef globalState = runConformance("apicalls.lua");