merge

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Help and support
+  - name: Questions
     url: https://github.com/Roblox/luau/discussions
     about: Please use GitHub Discussions if you have questions or need help.

CLI/Repl.cpp

@@ -13,6 +13,17 @@
 
 #include <memory>
 
+#ifdef _WIN32
+#include <io.h>
+#include <fcntl.h>
+#endif
+
+enum class CompileFormat
+{
+    Default,
+    Binary
+};
+
 static int lua_loadstring(lua_State* L)
 {
     size_t l = 0;
@@ -204,6 +215,9 @@ extern "C"
 {
     const char* executeScript(const char* source)
     {
+        // static string for caching result (prevents dangling ptr on function exit)
+        static std::string result;
+
         // setup flags
         for (Luau::FValue<bool>* flag = Luau::FValue<bool>::list; flag; flag = flag->next)
             if (strncmp(flag->name, "Luau", 4) == 0)
@@ -221,11 +235,11 @@ extern "C"
 
         // run code + collect error
         std::string error = runCode(L, source);
+        result  = error;
         
-        // output error(s)
         if (error.length())
         {
-            return std::move(error.c_str());
+            return result.c_str();
         }
         return NULL;
     }
@@ -410,7 +424,7 @@ static void reportError(const char* name, const Luau::CompileError& error)
     report(name, error.getLocation(), "CompileError", error.what());
 }
 
-static bool compileFile(const char* name)
+static bool compileFile(const char* name, CompileFormat format)
 {
     std::optional<std::string> source = readFile(name);
     if (!source)
@@ -427,7 +441,15 @@ static bool compileFile(const char* name)
 
         Luau::compileOrThrow(bcb, *source);
 
-        printf("%s", bcb.dumpEverything().c_str());
+        switch (format)
+        {
+        case CompileFormat::Default:
+            printf("%s", bcb.dumpEverything().c_str());
+            break;
+        case CompileFormat::Binary:
+            fwrite(bcb.getBytecode().data(), 1, bcb.getBytecode().size(), stdout);
+            break;
+        }
 
         return true;
     }
@@ -452,7 +474,7 @@ static void displayHelp(const char* argv0)
     printf("\n");
     printf("Available modes:\n");
     printf("  omitted: compile and run input files one by one\n");
-    printf("  --compile: compile input files and output resulting bytecode\n");
+    printf("  --compile[=format]: compile input files and output resulting formatted bytecode (binary or text)\n");
     printf("\n");
     printf("Available options:\n");
     printf("  --profile[=N]: profile the code using N Hz sampling (default 10000) and output results to profile.out\n");
@@ -484,8 +506,19 @@ int main(int argc, char** argv)
         return 0;
     }
 
-    if (argc >= 2 && strcmp(argv[1], "--compile") == 0)
+
+    if (argc >= 2 && strncmp(argv[1], "--compile", strlen("--compile")) == 0)
     {
+        CompileFormat format = CompileFormat::Default;
+
+        if (strcmp(argv[1], "--compile=binary") == 0)
+            format = CompileFormat::Binary;
+
+#ifdef _WIN32
+        if (format == CompileFormat::Binary)
+            _setmode(_fileno(stdout), _O_BINARY);
+#endif
+
         int failed = 0;
 
         for (int i = 2; i < argc; ++i)
@@ -496,13 +529,15 @@ int main(int argc, char** argv)
             if (isDirectory(argv[i]))
             {
                 traverseDirectory(argv[i], [&](const std::string& name) {
-                    if (name.length() > 4 && name.rfind(".lua") == name.length() - 4)
+                    if (name.length() > 5 && name.rfind(".luau") == name.length() - 5)
-                        failed += !compileFile(name.c_str());
+                        failed += !compileFile(name.c_str(), format);
+                    else if (name.length() > 4 && name.rfind(".lua") == name.length() - 4)
+                        failed += !compileFile(name.c_str(), format);
                 });
             }
             else
             {
-                failed += !compileFile(argv[i]);
+                failed += !compileFile(argv[i], format);
             }
         }
 

Compiler/include/Luau/Compiler.h

@@ -36,6 +36,10 @@ struct CompileOptions
     // global builtin to construct vectors; disabled by default
     const char* vectorLib = nullptr;
     const char* vectorCtor = nullptr;
+
+    // array of globals that are mutable; disables the import optimization for fields accessed through them
+    // use NULL to end the array
+    const char** mutableGlobals = nullptr;
 };
 
 class CompileError : public std::exception

Compiler/src/Compiler.cpp

@@ -13,6 +13,7 @@
 LUAU_FASTFLAGVARIABLE(LuauPreloadClosures, false)
 LUAU_FASTFLAGVARIABLE(LuauPreloadClosuresFenv, false)
 LUAU_FASTFLAGVARIABLE(LuauPreloadClosuresUpval, false)
+LUAU_FASTFLAGVARIABLE(LuauGenericSpecialGlobals, false)
 LUAU_FASTFLAG(LuauIfElseExpressionBaseSupport)
 
 namespace Luau
@@ -1277,7 +1278,7 @@ struct Compiler
     {
         const Global* global = globals.find(expr->name);
 
-        return options.optimizationLevel >= 1 && (!global || (!global->written && !global->special));
+        return options.optimizationLevel >= 1 && (!global || (!global->written && !global->writable));
     }
 
     void compileExprIndexName(AstExprIndexName* expr, uint8_t target)
@@ -3447,7 +3448,7 @@ struct Compiler
 
     struct Global
     {
-        bool special = false;
+        bool writable = false;
         bool written = false;
     };
 
@@ -3505,7 +3506,7 @@ struct Compiler
             {
                 Global* g = globals.find(object->name);
 
-                return !g || (!g->special && !g->written) ? Builtin{object->name, expr->index} : Builtin();
+                return !g || (!g->writable && !g->written) ? Builtin{object->name, expr->index} : Builtin();
             }
             else
             {
@@ -3703,13 +3704,26 @@ void compileOrThrow(BytecodeBuilder& bytecode, AstStatBlock* root, const AstName
 
     Compiler compiler(bytecode, options);
 
-    // since access to some global objects may result in values that change over time, we block table imports
+    // since access to some global objects may result in values that change over time, we block imports from non-readonly tables
-    for (const char* global : kSpecialGlobals)
+    if (FFlag::LuauGenericSpecialGlobals)
     {
-        AstName name = names.get(global);
+        if (AstName name = names.get("_G"); name.value)
+            compiler.globals[name].writable = true;
 
-        if (name.value)
+        if (options.mutableGlobals)
-            compiler.globals[name].special = true;
+            for (const char** ptr = options.mutableGlobals; *ptr != NULL; ++ptr)
+            {
+                if (AstName name = names.get(*ptr); name.value)
+                    compiler.globals[name].writable = true;
+            }
+    }
+    else
+    {
+        for (const char* global : kSpecialGlobals)
+        {
+            if (AstName name = names.get(global); name.value)
+                compiler.globals[name].writable = true;
+        }
     }
 
     // this visitor traverses the AST to analyze mutability of locals/globals, filling Local::written and Global::written

README.md

@@ -31,7 +31,7 @@ cmake --build . --target Luau.Repl.CLI --config RelWithDebInfo
 cmake --build . --target Luau.Analyze.CLI --config RelWithDebInfo
 ```
 
-Alternatively, on Linus/macOS you can use make:
+Alternatively, on Linux/macOS you can use make:
 
 ```sh
 make config=release luau luau-analyze

VM/include/lua.h

@@ -346,6 +346,8 @@ struct lua_Debug
  * can only be changed when the VM is not running any code */
 struct lua_Callbacks
 {
+    void* userdata; /* arbitrary userdata pointer that is never overwritten by Luau */
+
     void (*interrupt)(lua_State* L, int gc);  /* gets called at safepoints (loop back edges, call/ret, gc) if set */
     void (*panic)(lua_State* L, int errcode); /* gets called when an unprotected error is raised (if longjmp is used) */
 

VM/src/lapi.cpp

@@ -703,6 +703,7 @@ void lua_setreadonly(lua_State* L, int objindex, bool value)
     const TValue* o = index2adr(L, objindex);
     api_check(L, ttistable(o));
     Table* t = hvalue(o);
+    api_check(L, t != hvalue(registry(L)));
     t->readonly = value;
     return;
 }

VM/src/lbuiltins.cpp

@@ -20,8 +20,9 @@
 // If types of the arguments mismatch, luauF_* needs to return -1 and the execution will fall back to the usual call path
 // If luauF_* succeeds, it needs to return *all* requested arguments, filling results with nil as appropriate.
 // On input, nparams refers to the actual number of arguments (0+), whereas nresults contains LUA_MULTRET for arbitrary returns or 0+ for a
-// fixed-length return Because of this, and the fact that "extra" returned values will be ignored, implementations below typically check that nresults
+// fixed-length return
-// is <= expected number, which covers the LUA_MULTRET case.
+// Because of this, and the fact that "extra" returned values will be ignored, implementations below typically check that nresults is <= expected
+// number, which covers the LUA_MULTRET case.
 
 static int luauF_assert(lua_State* L, StkId res, TValue* arg0, int nresults, StkId args, int nparams)
 {

VM/src/ltable.cpp

@@ -30,6 +30,7 @@ LUAU_FASTFLAGVARIABLE(LuauArrayBoundary, false)
 #define MAXBITS 26
 #define MAXSIZE (1 << MAXBITS)
 
+static_assert(offsetof(LuaNode, val) == 0, "Unexpected Node memory layout, pointer cast in gval2slot is incorrect");
 // TKey is bitpacked for memory efficiency so we need to validate bit counts for worst case
 static_assert(TKey{{NULL}, 0, LUA_TDEADKEY, 0}.tt == LUA_TDEADKEY, "not enough bits for tt");
 static_assert(TKey{{NULL}, 0, LUA_TNIL, MAXSIZE - 1}.next == MAXSIZE - 1, "not enough bits for next");

VM/src/ltable.h

@@ -9,7 +9,6 @@
 #define gval(n) (&(n)->val)
 #define gnext(n) ((n)->key.next)
 
-static_assert(offsetof(LuaNode, val) == 0, "Unexpected Node memory layout, pointer cast below is incorrect");
 #define gval2slot(t, v) int(cast_to(LuaNode*, static_cast<const TValue*>(v)) - t->node)
 
 LUAI_FUNC const TValue* luaH_getnum(Table* t, int key);

docs/_data/navigation.yml

@@ -3,6 +3,8 @@ main:
     url: /news
   - title: Getting Started
     url: /getting-started
+  - title: GitHub
+    url: https://github.com/Roblox/luau
 
 pages:
   - title: Getting Started
@@ -25,5 +27,7 @@ pages:
     url: /profile
   - title: Library
     url: /library
-  - title: Demo
+
-    url: /demo
+# Remove demo pages until solution is found
+#  - title: Demo
+#    url: /demo

docs/_pages/getting-started.md

@@ -8,7 +8,7 @@ To get started with Luau you need to use `luau` command line binary to run your
 
 ## Creating a script
 
-To create your own testing script, create a new file with `.lua` as the extension:
+To create your own testing script, create a new file with `.luau` as the extension:
 
 ```lua
 function ispositive(x)
@@ -26,7 +26,7 @@ print(isfoo("bar"))
 print(isfoo(1))
 ```
 
-You can now run the file using `luau test.lua` and analyze it using `luau-analyze test.lua`.
+You can now run the file using `luau test.luau` and analyze it using `luau-analyze test.luau`.
 
 Note that there are no warnings about calling ``ispositive()`` with a string, or calling ``isfoo()`` a number. This is because the type checking uses non-strict mode by default, which is lenient in how it infers types used by the program.
 
@@ -52,8 +52,8 @@ In this case, Luau will use the ``return x > 0`` statement to infer that ``ispos
 Based on Luau's type inference, the analysis tool will now flag the incorrect call to ``ispositive()``:
 
 ```
-$ luau-analyze test.lua
+$ luau-analyze test.luau
-test.lua(7,18): TypeError: Type 'string' could not be converted into 'number'
+test.luau(7,18): TypeError: Type 'string' could not be converted into 'number'
 ```
 
 ## Annotations
@@ -92,9 +92,9 @@ result = ispositive(1)
 Oops -- we're returning string values, but we forgot to update the function return type. Since we've told Luau that ``ispositive()`` returns a boolean (and that's how we're using it), the call site isn't flagged as an error. But because the annotation doesn't match our code, we get a warning in the function body itself:
 
 ```
-$ luau-analyze test.lua
+$ luau-analyze test.luau
-test.lua(5,9): TypeError: Type 'string' could not be converted into 'boolean'
+test.luau(5,9): TypeError: Type 'string' could not be converted into 'boolean'
-test.lua(7,9): TypeError: Type 'string' could not be converted into 'boolean'
+test.luau(7,9): TypeError: Type 'string' could not be converted into 'boolean'
 ```
 
 The fix is simple; just change the annotation to declare the return type as a string:
@@ -117,8 +117,8 @@ result = ispositive(1)
 Well, almost - since we declared ``result`` as a boolean, the call site is now flagged:
 
 ```
-$ luau-analyze test.lua
+$ luau-analyze test.luau
-test.lua(12,10): TypeError: Type 'string' could not be converted into 'boolean'
+test.luau(12,10): TypeError: Type 'string' could not be converted into 'boolean'
 ```
 
 If we update the type of the local variable, everything is good. Note that we could also just let Luau infer the type of ``result`` by changing it to the single line version ``local result = ispositive(1)``.

docs/_pages/typecheck.md

@@ -351,6 +351,30 @@ local onlyString: string = stringOrNumber -- ok
 local onlyNumber: number = stringOrNumber -- not ok
 ```
 
+## Typecasts
+
+Expressions may be typecast using `::`.  Typecasting is useful for specifying the type of an expression when the automatically inferred type is too generic.
+
+For example, consider the following table constructor where the intent is to store a table of names:
+```lua
+local myTable = {names = {}}
+table.insert(myTable.names, 42)         -- Inserting a number ought to cause a type error, but doesn't
+```
+
+In order to specify the type of the `names` table a typecast may be used: 
+
+```lua
+local myTable = {names = {} :: {string}}
+table.insert(myTable.names, 42)         -- not ok, invalid 'number' to 'string' conversion
+```
+
+A typecast itself is also type checked to ensure the conversion specified is a valid one:
+```lua
+local number = 1
+local value = number :: any             -- ok, 'number' is a subtype of 'any'
+local flag = number :: boolean          -- not ok, invalid 'number' to 'boolean' conversion
+```
+
 ## Roblox types
 
 Roblox supports a rich set of classes and data types, [documented here](https://developer.roblox.com/en-us/api-reference). All of them are readily available for the type checker to use by their name (e.g. `Part` or `RaycastResult`).
@@ -397,3 +421,10 @@ return module
 ```
 
 There are some caveats here though. For instance, the require path must be resolvable statically, otherwise Luau cannot accurately type check it.
+
+### Cyclic module dependencies
+
+Cyclic module dependencies can cause problems for the type checker.  In order to break a module dependency cycle a typecast of the module to `any` may be used:
+```lua
+local myModule = require(MyModule) :: any
+```

rfcs/function-bit32-countlz-countrz.md

@@ -0,0 +1,50 @@
+# bit32.countlz/countrz
+
+## Summary
+
+Add bit32.countlz (count left zeroes) and bit32.countrz (count right zeroes) to accelerate bit scanning
+
+## Motivation
+
+All CPUs have instructions to determine the position of first/last set bit in an integer. These instructions have a variety of uses, the popular ones being:
+
+- Fast implementation of integer logarithm (essentially allowing to compute `floor(log2(value))` quickly)
+- Scanning set bits in an integer, which allows efficient traversal of compact representation of bitmaps
+- Allocating bits out of a bitmap quickly
+
+Today it's possible to approximate `countlz` using `floor` and `log` but this approximation is relatively slow; approximating `countrz` is difficult without iterating through each bit.
+
+## Design
+
+`bit32` library will gain two new functions, `countlz` and `countrz`:
+
+```
+function bit32.countlz(n: number): number
+function bit32.countrz(n: number): number
+```
+
+`countlz` takes an integer number (converting the input number to a 32-bit unsigned integer as all other `bit32` functions do), and returns the number of consecutive left-most zero bits - that is, the number of most significant zero bits in a 32-bit number until the first 1. The result is in `[0, 32]` range.
+
+For example, when the input number is `0`, it's `32`. When the input number is `2^k`, the result is `31-k`.
+
+`countrz` takes an integer number (converting the input number to a 32-bit unsigned integer as all other `bit32` functions do), and returns the number of consecutive right-most zero bits - that is,
+the number of least significant zero bits in a 32-bit number until the first 1. The result is in `[0, 32]` range.
+
+For example, when the input number is `0`, it's `32`. When the input number is `2^k`, the result is `k`.
+
+> Non-normative: a proof of concept implementation shows that a polyfill for `countlz` takes ~34 ns per loop iteration when computing `countlz` for an increasing number sequence, whereas
+> a builtin implementation takes ~4 ns.
+
+## Drawbacks
+
+None known.
+
+## Alternatives
+
+These functions can be alternatively specified as "find the position of the most/least significant bit set" (e.g. "ffs"/"fls" for "find first set"/"find last set"). This formulation
+can be more immediately useful since the bit position is usually more important than the number of bits. However, the bit position is undefined when the input number is zero,
+returning a sentinel such as -1 seems non-idiomatic, and returning `nil` seems awkward for calling code. Counting functions don't have this problem.
+
+An early version of this proposal suggested `clz`/`ctz` (leading/trailing) as names; however, using a full verb is more consistent with other operations like shift/rotate, and left/right may be easier to understand intuitively compared to leading/trailing. left/right are used by C++20.
+
+Of the two functions, `countlz` is vastly more useful than `countrz`; we could implement just `countlz`, but having both is nice for symmetry.

tests/Compiler.test.cpp

@@ -13,6 +13,7 @@
 LUAU_FASTFLAG(LuauPreloadClosures)
 LUAU_FASTFLAG(LuauPreloadClosuresFenv)
 LUAU_FASTFLAG(LuauPreloadClosuresUpval)
+LUAU_FASTFLAG(LuauGenericSpecialGlobals)
 
 using namespace Luau;
 
@@ -3670,4 +3671,118 @@ RETURN R0 0
 )");
 }
 
+TEST_CASE("LuauGenericSpecialGlobals")
+{
+    const char* source = R"(
+print()
+Game.print()
+Workspace.print()
+_G.print()
+game.print()
+plugin.print()
+script.print()
+shared.print()
+workspace.print()
+)";
+
+    {
+        ScopedFastFlag genericSpecialGlobals{"LuauGenericSpecialGlobals", false};
+
+        // Check Roblox globals are here
+        CHECK_EQ("\n" + compileFunction0(source), R"(
+GETIMPORT R0 1
+CALL R0 0 0
+GETIMPORT R1 3
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 5
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 7
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 9
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 11
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 13
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 15
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 17
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+RETURN R0 0
+)");
+    }
+
+    ScopedFastFlag genericSpecialGlobals{"LuauGenericSpecialGlobals", true};
+
+    // Check Roblox globals are no longer here
+    CHECK_EQ("\n" + compileFunction0(source), R"(
+GETIMPORT R0 1
+CALL R0 0 0
+GETIMPORT R0 3
+CALL R0 0 0
+GETIMPORT R0 5
+CALL R0 0 0
+GETIMPORT R1 7
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R0 9
+CALL R0 0 0
+GETIMPORT R0 11
+CALL R0 0 0
+GETIMPORT R0 13
+CALL R0 0 0
+GETIMPORT R0 15
+CALL R0 0 0
+GETIMPORT R0 17
+CALL R0 0 0
+RETURN R0 0
+)");
+
+    // Check we can add them back
+    Luau::BytecodeBuilder bcb;
+    bcb.setDumpFlags(Luau::BytecodeBuilder::Dump_Code);
+    Luau::CompileOptions options;
+    const char* mutableGlobals[] = {"Game", "Workspace", "game", "plugin", "script", "shared", "workspace", NULL};
+    options.mutableGlobals = &mutableGlobals[0];
+    Luau::compileOrThrow(bcb, source, options);
+
+    CHECK_EQ("\n" + bcb.dumpFunction(0), R"(
+GETIMPORT R0 1
+CALL R0 0 0
+GETIMPORT R1 3
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 5
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 7
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 9
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 11
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 13
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 15
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+GETIMPORT R1 17
+GETTABLEKS R0 R1 K0
+CALL R0 0 0
+RETURN R0 0
+)");
+}
+
 TEST_SUITE_END();