Prevent overflow in lua_newuserdatadtor (#639)

In case a large userdata size is passed to lua_newuserdatadtor it might overflow the size resulting in luaU_newudata actually allocating the object without a memory error. This will then result in overwriting part of the metatable pointer of the userdata.
This PR fixes this issue by checking for the overflow and in such cases pass a size value which will cause a memory error in luaU_newudata.

VM/src/lapi.cpp

@@ -1209,7 +1209,10 @@ void* lua_newuserdatadtor(lua_State* L, size_t sz, void (*dtor)(void*))
 {
     luaC_checkGC(L);
     luaC_checkthreadsleep(L);
-    Udata* u = luaU_newudata(L, sz + sizeof(dtor), UTAG_IDTOR);
+    size_t as = sz + sizeof(dtor);
+    if (as < sizeof(dtor))
+        as = SIZE_MAX; // Will cause a memory error in luaU_newudata.
+    Udata* u = luaU_newudata(L, as, UTAG_IDTOR);
     memcpy(&u->data + sz, &dtor, sizeof(dtor));
     setuvalue(L, L->top, u);
     api_incr_top(L);

tests/Conformance.test.cpp

@@ -716,6 +716,23 @@ TEST_CASE("Reference")
     CHECK(dtorhits == 2);
 }
 
+TEST_CASE("NewUserdataOverflow")
+{
+    StateRef globalState(luaL_newstate(), lua_close);
+    lua_State* L = globalState.get();
+
+    lua_pushcfunction(L, [](lua_State* L1) {
+        // The following userdata request might cause an overflow.
+        lua_newuserdatadtor(L1, SIZE_MAX, [](void* d){});
+        // The overflow might segfault in the following call.
+        lua_getmetatable(L1, -1);
+        return 0;
+    }, "PCall");
+
+    CHECK(lua_pcall(L, 0, 0, 0) == LUA_ERRRUN);
+    CHECK(strcmp(lua_tostring(L, -1), "memory allocation error: block too big") == 0);
+}
+
 TEST_CASE("ApiTables")
 {
     StateRef globalState(luaL_newstate(), lua_close);