From 99ab30cf04bb0e55daa5455662a1ab68696fbba1 Mon Sep 17 00:00:00 2001
From: Alex Orlenko <aorlenko@yelp.com>
Date: Tue, 28 Mar 2023 22:39:04 +0100
Subject: [PATCH] Do not call message handler in xpcall if allocation error

---
 VM/src/ldo.cpp                   |  4 ++--
 tests/Conformance.test.cpp       |  5 +++++
 tests/conformance/exceptions.lua | 12 ++++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/VM/src/ldo.cpp b/VM/src/ldo.cpp
index ff8105b8..a89a3d5e 100644
--- a/VM/src/ldo.cpp
+++ b/VM/src/ldo.cpp
@@ -550,8 +550,8 @@ int luaD_pcall(lua_State* L, Pfunc func, void* u, ptrdiff_t old_top, ptrdiff_t e
     int status = luaD_rawrunprotected(L, func, u);
     if (status != 0)
     {
-        // call user-defined error function (used in xpcall)
-        if (ef)
+        // call user-defined error function (used in xpcall) except if allocation error
+        if (ef && status != LUA_ERRMEM)
         {
             // if errfunc fails, we fail with "error in error handling"
             if (luaD_rawrunprotected(L, callerrfunc, restorestack(L, ef)) != 0)
diff --git a/tests/Conformance.test.cpp b/tests/Conformance.test.cpp
index c32f2870..88f5e517 100644
--- a/tests/Conformance.test.cpp
+++ b/tests/Conformance.test.cpp
@@ -1078,6 +1078,11 @@ TEST_CASE("ExceptionObject")
         ExceptionResult result = captureException(L, "large_allocation_error");
         CHECK(result.exceptionGenerated);
     }
+
+    {
+        ExceptionResult result = captureException(L, "large_allocation_error_without_handler");
+        CHECK(result.exceptionGenerated);
+    }
 }
 #endif
 
diff --git a/tests/conformance/exceptions.lua b/tests/conformance/exceptions.lua
index e07029ff..22a58496 100644
--- a/tests/conformance/exceptions.lua
+++ b/tests/conformance/exceptions.lua
@@ -32,4 +32,16 @@ function large_allocation_error()
     table.create(1000000)
 end
 
+function large_allocation_error_without_handler()
+    -- Create a table that will require more memory than the test's memory
+    -- allocator will allow.
+    local msgh_called = false
+    local ok, err = xpcall(table.create, function() msgh_called = true end, 1000000)
+    -- Check that message handler was not called for memory allocation error
+    if not ok and not msgh_called then
+        -- Propagate error
+        error(err)
+    end
+end
+
 return('OK')