diff --git a/VM/src/ldo.cpp b/VM/src/ldo.cpp index c133a59e..c7904dde 100644 --- a/VM/src/ldo.cpp +++ b/VM/src/ldo.cpp @@ -213,6 +213,14 @@ CallInfo* luaD_growCI(lua_State* L) return ++L->ci; } +void luaD_checkCstack(lua_State *L) +{ + if (L->nCcalls == LUAI_MAXCCALLS) + luaG_runerror(L, "C stack overflow"); + else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3))) + luaD_throw(L, LUA_ERRERR); /* error while handling stack error */ +} + /* ** Call a function (C or Lua). The function to be called is at *func. ** The arguments are on the stack, right after the function. @@ -222,12 +230,8 @@ CallInfo* luaD_growCI(lua_State* L) void luaD_call(lua_State* L, StkId func, int nResults) { if (++L->nCcalls >= LUAI_MAXCCALLS) - { - if (L->nCcalls == LUAI_MAXCCALLS) - luaG_runerror(L, "C stack overflow"); - else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3))) - luaD_throw(L, LUA_ERRERR); /* error while handing stack error */ - } + luaD_checkCstack(L); + if (luau_precall(L, func, nResults) == PCRLUA) { /* is a Lua function? */ L->ci->flags |= LUA_CALLINFO_RETURN; /* luau_execute will stop after returning from the stack frame */ diff --git a/VM/src/ldo.h b/VM/src/ldo.h index 6e16e6f1..5e9472bf 100644 --- a/VM/src/ldo.h +++ b/VM/src/ldo.h @@ -49,6 +49,7 @@ LUAI_FUNC int luaD_pcall(lua_State* L, Pfunc func, void* u, ptrdiff_t oldtop, pt LUAI_FUNC void luaD_reallocCI(lua_State* L, int newsize); LUAI_FUNC void luaD_reallocstack(lua_State* L, int newsize); LUAI_FUNC void luaD_growstack(lua_State* L, int n); +LUAI_FUNC void luaD_checkCstack(lua_State* L); LUAI_FUNC l_noret luaD_throw(lua_State* L, int errcode); LUAI_FUNC int luaD_rawrunprotected(lua_State* L, Pfunc f, void* ud); diff --git a/VM/src/lvmexecute.cpp b/VM/src/lvmexecute.cpp index 9e2eb268..3c505fe5 100644 --- a/VM/src/lvmexecute.cpp +++ b/VM/src/lvmexecute.cpp @@ -181,7 +181,7 @@ LUAU_NOINLINE static void luau_callTM(lua_State* L, int nparams, int res) ++L->nCcalls; if (L->nCcalls >= LUAI_MAXCCALLS) - luaG_runerror(L, "C stack overflow"); + luaD_checkCstack(L); luaD_checkstack(L, LUA_MINSTACK); diff --git a/tests/conformance/errors.lua b/tests/conformance/errors.lua index 297cf011..d8dc9bd2 100644 --- a/tests/conformance/errors.lua +++ b/tests/conformance/errors.lua @@ -167,6 +167,81 @@ if not limitedstack then end end +-- C stack overflow +if not limitedstack then + local count = 1 + local cso = setmetatable({}, { + __index = function(self, i) + count = count + 1 + return self[i] + end, + __newindex = function(self, i, v) + count = count + 1 + self[i] = v + end, + __tostring = function(self) + count = count + 1 + return tostring(self) + end + }) + + local ehline + local function ehassert(cond) + if not cond then + ehline = debug.info(2, "l") + error() + end + end + + local userdata = newproxy(true) + getmetatable(userdata).__index = print + assert(debug.info(print, "s") == "[C]") + + local s, e = xpcall(tostring, function(e) + ehassert(string.find(e, "C stack overflow")) + print("after __tostring C stack overflow", count) -- 198: 1 resume + 1 xpcall + 198 luaB_tostring calls (which runs our __tostring successfully 197 times, erroring on the last attempt) + ehassert(count > 1) + + local ps, pe + + -- __tostring overflow (lua_call) + count = 1 + ps, pe = pcall(tostring, cso) + print("after __tostring overflow in handler", count) -- 23: xpcall error handler + pcall + 23 luaB_tostring calls + ehassert(not ps and string.find(pe, "error in error handling")) + ehassert(count > 1) + + -- __index overflow (callTMres) + count = 1 + ps, pe = pcall(function() return cso[cso] end) + print("after __index overflow in handler", count) -- 23: xpcall error handler + pcall + 23 __index calls + ehassert(not ps and string.find(pe, "error in error handling")) + ehassert(count > 1) + + -- __newindex overflow (callTM) + count = 1 + ps, pe = pcall(function() cso[cso] = "kohuke" end) + print("after __newindex overflow in handler", count) -- 23: xpcall error handler + pcall + 23 __newindex calls + ehassert(not ps and string.find(pe, "error in error handling")) + ehassert(count > 1) + + -- test various C __index invocations on userdata + ehassert(pcall(function() return userdata[userdata] end)) -- LOP_GETTABLE + ehassert(pcall(function() return userdata[1] end)) -- LOP_GETTABLEN + ehassert(pcall(function() return userdata.StringConstant end)) -- LOP_GETTABLEKS (luau_callTM) + + -- lua_resume test + local coro = coroutine.create(function() end) + ps, pe = coroutine.resume(coro) + ehassert(not ps and string.find(pe, "C stack overflow")) + + return true + end, cso) + + assert(not s) + assert(e == true, "error in xpcall eh, line " .. tostring(ehline)) +end + --[[ local i=1 while stack[i] ~= l1 do