Merge remote-tracking branch 'origin/master' into merge

Analysis/src/EmbeddedBuiltinDefinitions.cpp

@@ -143,7 +143,7 @@ declare coroutine: {
     create: <A..., R...>((A...) -> R...) -> thread,
     resume: <A..., R...>(thread, A...) -> (boolean, R...),
     running: () -> thread,
-    status: (thread) -> string,
+    status: (thread) -> "dead" | "running" | "normal" | "suspended",
     -- FIXME: This technically returns a function, but we can't represent this yet.
     wrap: <A..., R...>((A...) -> R...) -> any,
     yield: <A..., R...>(A...) -> R...,

VM/include/lua.h

@@ -148,6 +148,7 @@ LUA_API const char* lua_tostringatom(lua_State* L, int idx, int* atom);
 LUA_API const char* lua_namecallatom(lua_State* L, int* atom);
 LUA_API int lua_objlen(lua_State* L, int idx);
 LUA_API lua_CFunction lua_tocfunction(lua_State* L, int idx);
+LUA_API void* lua_tolightuserdata(lua_State* L, int idx);
 LUA_API void* lua_touserdata(lua_State* L, int idx);
 LUA_API void* lua_touserdatatagged(lua_State* L, int idx, int tag);
 LUA_API int lua_userdatatag(lua_State* L, int idx);

VM/src/lapi.cpp

@@ -478,18 +478,22 @@ lua_CFunction lua_tocfunction(lua_State* L, int idx)
     return (!iscfunction(o)) ? NULL : cast_to(lua_CFunction, clvalue(o)->c.f);
 }
 
+void* lua_tolightuserdata(lua_State* L, int idx)
+{
+    StkId o = index2addr(L, idx);
+    return (!ttislightuserdata(o)) ? NULL : pvalue(o);
+}
+
 void* lua_touserdata(lua_State* L, int idx)
 {
     StkId o = index2addr(L, idx);
-    switch (ttype(o))
-    {
-    case LUA_TUSERDATA:
+    // fast-path: check userdata first since it is most likely the expected result
+    if (ttisuserdata(o))
         return uvalue(o)->data;
-    case LUA_TLIGHTUSERDATA:
+    else if (ttislightuserdata(o))
         return pvalue(o);
-    default:
+    else
         return NULL;
-    }
 }
 
 void* lua_touserdatatagged(lua_State* L, int idx, int tag)
@@ -524,8 +528,9 @@ const void* lua_topointer(lua_State* L, int idx)
     case LUA_TTHREAD:
         return thvalue(o);
     case LUA_TUSERDATA:
+        return uvalue(o)->data;
     case LUA_TLIGHTUSERDATA:
-        return lua_touserdata(L, idx);
+        return pvalue(o);
     default:
         return NULL;
     }

VM/src/ldo.cpp

@@ -213,6 +213,14 @@ CallInfo* luaD_growCI(lua_State* L)
     return ++L->ci;
 }
 
+void luaD_checkCstack(lua_State *L)
+{
+    if (L->nCcalls == LUAI_MAXCCALLS)
+        luaG_runerror(L, "C stack overflow");
+    else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3)))
+        luaD_throw(L, LUA_ERRERR); /* error while handling stack error */
+}
+
 /*
 ** Call a function (C or Lua). The function to be called is at *func.
 ** The arguments are on the stack, right after the function.
@@ -222,12 +230,8 @@ CallInfo* luaD_growCI(lua_State* L)
 void luaD_call(lua_State* L, StkId func, int nResults)
 {
     if (++L->nCcalls >= LUAI_MAXCCALLS)
-    {
-        if (L->nCcalls == LUAI_MAXCCALLS)
-            luaG_runerror(L, "C stack overflow");
-        else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3)))
-            luaD_throw(L, LUA_ERRERR); /* error while handing stack error */
-    }
+        luaD_checkCstack(L);
+
     if (luau_precall(L, func, nResults) == PCRLUA)
     {                                        /* is a Lua function? */
         L->ci->flags |= LUA_CALLINFO_RETURN; /* luau_execute will stop after returning from the stack frame */

VM/src/ldo.h

@@ -49,6 +49,7 @@ LUAI_FUNC int luaD_pcall(lua_State* L, Pfunc func, void* u, ptrdiff_t oldtop, pt
 LUAI_FUNC void luaD_reallocCI(lua_State* L, int newsize);
 LUAI_FUNC void luaD_reallocstack(lua_State* L, int newsize);
 LUAI_FUNC void luaD_growstack(lua_State* L, int n);
+LUAI_FUNC void luaD_checkCstack(lua_State* L);
 
 LUAI_FUNC l_noret luaD_throw(lua_State* L, int errcode);
 LUAI_FUNC int luaD_rawrunprotected(lua_State* L, Pfunc f, void* ud);

VM/src/lvmexecute.cpp

@@ -181,7 +181,7 @@ LUAU_NOINLINE static void luau_callTM(lua_State* L, int nparams, int res)
     ++L->nCcalls;
 
     if (L->nCcalls >= LUAI_MAXCCALLS)
-        luaG_runerror(L, "C stack overflow");
+        luaD_checkCstack(L);
 
     luaD_checkstack(L, LUA_MINSTACK);
 

docs/_pages/performance.md

@@ -92,12 +92,22 @@ As a result, builtin calls are very fast in Luau - they are still slightly slowe
 
 ## Optimized table iteration
 
-Luau implements a fully generic iteration protocol; however, for iteration through tables it recognizes three common idioms (`for .. in ipairs(t)`, `for .. in pairs(t)` and `for .. in next, t`) and emits specialized bytecode that is carefully optimized using custom internal iterators.
+Luau implements a fully generic iteration protocol; however, for iteration through tables in addition to generalized iteration (`for .. in t`) it recognizes three common idioms (`for .. in ipairs(t)`, `for .. in pairs(t)` and `for .. in next, t`) and emits specialized bytecode that is carefully optimized using custom internal iterators.
 
-As a result, iteration through tables typically doesn't result in function calls for every iteration; the performance of iteration using `pairs` and `ipairs` is comparable, so it's recommended to pick the iteration style based on readability instead of performance.
+As a result, iteration through tables typically doesn't result in function calls for every iteration; the performance of iteration using generalized iteration, `pairs` and `ipairs` is comparable, so generalized iteration (without the use of `pairs`/`ipairs`) is recommended unless the code needs to be compatible with vanilla Lua or the specific semantics of `ipairs` (which stops at the first `nil` element) is required. Additionally, using generalized iteration avoids calling `pairs` when the loop starts which can be noticeable when the table is very short.
 
 Iterating through array-like tables using `for i=1,#t` tends to be slightly slower because of extra cost incurred when reading elements from the table.
 
+## Optimized table length
+
+Luau tables use a hybrid array/hash storage, like in Lua; in some sense "arrays" don't truly exist and are an internal optimization, but some operations, notably `#t` and functions that depend on it, like `table.insert`, are defined by the Luau/Lua language to allow internal optimizations. Luau takes advantage of that fact.
+
+Unlike Lua, Luau guarantees that the element at index `#t` is stored in the array part of the table. This can accelerate various table operations that use indices limited by `#t`, and this makes `#t` worst-case complexity O(logN), unlike Lua where the worst case complexity is O(N). This also accelerates computation of this value for small tables like `{ [1] = 1 }` since we never need to look at the hash part.
+
+The "default" implementation of `#t` in both Lua and Luau is a binary search. Luau uses a special branch-free (depending on the compiler...) implementation of the binary search which results in 50+% faster computation of table length when it needs to be computed from scratch.
+
+Additionally, Luau can cache the length of the table and adjust it following operations like `table.insert`/`table.remove`; this means that in practice, `#t` is almost always a constant time operation.
+
 ## Creating and modifying tables
 
 Luau implements several optimizations for table creation. When creating object-like tables, it's recommended to use table literals (`{ ... }`) and to specify all table fields in the literal in one go instead of assigning fields later; this triggers an optimization inspired by LuaJIT's "table templates" and results in higher performance when creating objects. When creating array-like tables, if the maximum size of the table is known up front, it's recommended to use `table.create` function which can create an empty table with preallocated storage, and optionally fill it with a given value.
@@ -112,7 +122,7 @@ v.z = 3
 return v
 ```
 
-When appending elements to tables, it's recommended to use `table.insert` (which is the fastest method to append an element to a table if the table size is not known). In cases when a table is filled sequentially, however, it's much more efficient to use a known index for insertion - together with preallocating tables using `table.create` this can result in much faster code, for example this is the fastest way to build a table of squares:
+When appending elements to tables, it's recommended to use `table.insert` (which is the fastest method to append an element to a table if the table size is not known). In cases when a table is filled sequentially, however, it can be more efficient to use a known index for insertion - together with preallocating tables using `table.create` this can result in much faster code, for example this is the fastest way to build a table of squares:
 
 ```lua
 local t = table.create(N)

tests/Conformance.test.cpp

@@ -1066,6 +1066,7 @@ TEST_CASE("UserdataApi")
     int lud;
     lua_pushlightuserdata(L, &lud);
 
+    CHECK(lua_tolightuserdata(L, -1) == &lud);
     CHECK(lua_touserdata(L, -1) == &lud);
     CHECK(lua_topointer(L, -1) == &lud);
 
@@ -1073,6 +1074,7 @@ TEST_CASE("UserdataApi")
     int* ud1 = (int*)lua_newuserdata(L, 4);
     *ud1 = 42;
 
+    CHECK(lua_tolightuserdata(L, -1) == nullptr);
     CHECK(lua_touserdata(L, -1) == ud1);
     CHECK(lua_topointer(L, -1) == ud1);
 

tests/conformance/errors.lua

@@ -167,6 +167,81 @@ if not limitedstack then
   end
 end
 
+-- C stack overflow
+if not limitedstack then
+  local count = 1
+  local cso = setmetatable({}, {
+    __index = function(self, i)
+      count = count + 1
+      return self[i]
+    end,
+    __newindex = function(self, i, v)
+      count = count + 1
+      self[i] = v
+    end,
+    __tostring = function(self)
+      count = count + 1
+      return tostring(self)
+    end
+  })
+
+  local ehline
+  local function ehassert(cond)
+    if not cond then
+      ehline = debug.info(2, "l")
+      error()
+    end
+  end
+
+  local userdata = newproxy(true)
+  getmetatable(userdata).__index = print
+  assert(debug.info(print, "s") == "[C]")
+
+  local s, e = xpcall(tostring, function(e)
+    ehassert(string.find(e, "C stack overflow"))
+    print("after __tostring C stack overflow", count) -- 198: 1 resume + 1 xpcall + 198 luaB_tostring calls (which runs our __tostring successfully 197 times, erroring on the last attempt)
+    ehassert(count > 1)
+
+    local ps, pe
+
+    -- __tostring overflow (lua_call)
+    count = 1
+    ps, pe = pcall(tostring, cso)
+    print("after __tostring overflow in handler", count) -- 23: xpcall error handler + pcall + 23 luaB_tostring calls
+    ehassert(not ps and string.find(pe, "error in error handling"))
+    ehassert(count > 1)
+
+    -- __index overflow (callTMres)
+    count = 1
+    ps, pe = pcall(function() return cso[cso] end)
+    print("after __index overflow in handler", count) -- 23: xpcall error handler + pcall + 23 __index calls
+    ehassert(not ps and string.find(pe, "error in error handling"))
+    ehassert(count > 1)
+
+    -- __newindex overflow (callTM)
+    count = 1
+    ps, pe = pcall(function() cso[cso] = "kohuke" end)
+    print("after __newindex overflow in handler", count) -- 23: xpcall error handler + pcall + 23 __newindex calls
+    ehassert(not ps and string.find(pe, "error in error handling"))
+    ehassert(count > 1)
+
+    -- test various C __index invocations on userdata
+    ehassert(pcall(function() return userdata[userdata] end)) -- LOP_GETTABLE
+    ehassert(pcall(function() return userdata[1] end)) -- LOP_GETTABLEN
+    ehassert(pcall(function() return userdata.StringConstant end)) -- LOP_GETTABLEKS (luau_callTM)
+
+    -- lua_resume test
+    local coro = coroutine.create(function() end)
+    ps, pe = coroutine.resume(coro)
+    ehassert(not ps and string.find(pe, "C stack overflow"))
+
+    return true
+  end, cso)
+
+  assert(not s)
+  assert(e == true, "error in xpcall eh, line " .. tostring(ehline))
+end
+
 --[[
 local i=1
 while stack[i] ~= l1 do