From 5386b3bd6d6750709d751f67432ad71636123b7d Mon Sep 17 00:00:00 2001 From: XmiliaH Date: Tue, 16 Aug 2022 23:14:44 +0200 Subject: [PATCH] Change upvalue size from 48 to 32 bytes --- VM/src/lfunc.cpp | 61 +++++----------- VM/src/lfunc.h | 1 - VM/src/lgc.cpp | 111 +++++++++++++++++------------ VM/src/lgc.h | 1 - VM/src/lgcdebug.cpp | 19 +++-- VM/src/lobject.h | 10 +-- VM/src/lstate.cpp | 6 +- VM/src/lstate.h | 3 +- bench/gc/test_GC_upvalues_dead.lua | 17 +++++ bench/gc/test_GC_upvalues_gray.lua | 23 ++++++ 10 files changed, 139 insertions(+), 113 deletions(-) create mode 100644 bench/gc/test_GC_upvalues_dead.lua create mode 100644 bench/gc/test_GC_upvalues_gray.lua diff --git a/VM/src/lfunc.cpp b/VM/src/lfunc.cpp index dfde6dcb..07f13342 100644 --- a/VM/src/lfunc.cpp +++ b/VM/src/lfunc.cpp @@ -69,13 +69,14 @@ UpVal* luaF_findupval(lua_State* L, StkId level) global_State* g = L->global; UpVal** pp = &L->openupval; UpVal* p; + + LUAU_ASSERT(!isblack(obj2gco(L)) || !keepinvariant(L->global)); while (*pp != NULL && (p = *pp)->v >= level) { LUAU_ASSERT(p->v != &p->u.value); + LUAU_ASSERT(!isdead(g, obj2gco(p))); if (p->v == level) { // found a corresponding upvalue? - if (isdead(g, obj2gco(p))) // is it dead? - changewhite(obj2gco(p)); // resurrect it return p; } @@ -89,40 +90,14 @@ UpVal* luaF_findupval(lua_State* L, StkId level) uv->v = level; // current value lives in the stack // chain the upvalue in the threads open upvalue list at the proper position - UpVal* next = *pp; - uv->u.l.threadnext = next; - uv->u.l.threadprev = pp; - if (next) - next->u.l.threadprev = &uv->u.l.threadnext; - + uv->u.l.thread = L; + uv->u.l.threadnext = *pp; *pp = uv; - - // double link the upvalue in the global open upvalue list - uv->u.l.prev = &g->uvhead; - uv->u.l.next = g->uvhead.u.l.next; - uv->u.l.next->u.l.prev = uv; - g->uvhead.u.l.next = uv; - LUAU_ASSERT(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); return uv; } -void luaF_unlinkupval(UpVal* uv) -{ - // unlink upvalue from the global open upvalue list - LUAU_ASSERT(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); - uv->u.l.next->u.l.prev = uv->u.l.prev; - uv->u.l.prev->u.l.next = uv->u.l.next; - - // unlink upvalue from the thread open upvalue list - *uv->u.l.threadprev = uv->u.l.threadnext; - - if (UpVal* next = uv->u.l.threadnext) - next->u.l.threadprev = uv->u.l.threadprev; -} void luaF_freeupval(lua_State* L, UpVal* uv, lua_Page* page) { - if (uv->v != &uv->u.value) // is it open? - luaF_unlinkupval(uv); // remove from open list luaM_freegco(L, uv, sizeof(UpVal), uv->memcat, page); // free upvalue } @@ -130,26 +105,22 @@ void luaF_close(lua_State* L, StkId level) { global_State* g = L->global; UpVal* uv; + + LUAU_ASSERT(!isblack(obj2gco(L)) || !keepinvariant(L->global)); while (L->openupval != NULL && (uv = L->openupval)->v >= level) { GCObject* o = obj2gco(uv); - LUAU_ASSERT(!isblack(o) && uv->v != &uv->u.value); + LUAU_ASSERT(uv->v != &uv->u.value); + LUAU_ASSERT(L == uv->u.l.thread); + LUAU_ASSERT(!isdead(g, o)); - // by removing the upvalue from global/thread open upvalue lists, L->openupval will be pointing to the next upvalue - luaF_unlinkupval(uv); + // by removing the upvalue from thread open upvalue lists, L->openupval will be pointing to the next upvalue + L->openupval = uv->u.l.threadnext; - if (isdead(g, o)) - { - // close the upvalue without copying the dead data so that luaF_freeupval will not unlink again - uv->v = &uv->u.value; - } - else - { - setobj(L, &uv->u.value, uv->v); - uv->v = &uv->u.value; - // GC state of a new closed upvalue has to be initialized - luaC_initupval(L, uv); - } + setobj(L, &uv->u.value, uv->v); + uv->v = &uv->u.value; + // black upvalue might point to white object in white or gray thread + luaC_barrier(L, uv, uv->v); } } diff --git a/VM/src/lfunc.h b/VM/src/lfunc.h index a260d00a..9c3899a9 100644 --- a/VM/src/lfunc.h +++ b/VM/src/lfunc.h @@ -14,6 +14,5 @@ LUAI_FUNC UpVal* luaF_findupval(lua_State* L, StkId level); LUAI_FUNC void luaF_close(lua_State* L, StkId level); LUAI_FUNC void luaF_freeproto(lua_State* L, Proto* f, struct lua_Page* page); LUAI_FUNC void luaF_freeclosure(lua_State* L, Closure* c, struct lua_Page* page); -LUAI_FUNC void luaF_unlinkupval(UpVal* uv); LUAI_FUNC void luaF_freeupval(lua_State* L, UpVal* uv, struct lua_Page* page); LUAI_FUNC const LocVar* luaF_getlocal(const Proto* func, int local_number, int pc); diff --git a/VM/src/lgc.cpp b/VM/src/lgc.cpp index f7a851f4..8b0bc301 100644 --- a/VM/src/lgc.cpp +++ b/VM/src/lgc.cpp @@ -150,8 +150,18 @@ static void reallymarkobject(global_State* g, GCObject* o) { UpVal* uv = gco2uv(o); markvalue(g, uv->v); - if (uv->v == &uv->u.value) // closed? - gray2black(o); // open upvalues are never black + if (uv->v != &uv->u.value) { + lua_State* t = uv->u.l.thread; + if (iswhite(obj2gco(t)) && !t->gclistprev) { + t->gclist = g->whitethreads; + if (g->whitethreads) { + gco2th(g->whitethreads)->gclistprev = &t->gclist; + } + t->gclistprev = &g->whitethreads; + g->whitethreads = obj2gco(t); + } + } + gray2black(o); return; } case LUA_TFUNCTION: @@ -168,6 +178,10 @@ static void reallymarkobject(global_State* g, GCObject* o) } case LUA_TTHREAD: { + if (gco2th(o)->gclistprev) { + *gco2th(o)->gclistprev = gco2th(o)->gclist; + gco2th(o)->gclistprev = NULL; + } gco2th(o)->gclist = g->gray; g->gray = o; break; @@ -305,6 +319,19 @@ static void traversestack(global_State* g, lua_State* l, bool clearstack) } } +static size_t markupvals(global_State* g, UpVal* uv) { + size_t work = 0; + while(uv) { + work += sizeof(UpVal); + if (iswhite(obj2gco(uv))) { + white2gray(obj2gco(uv)); + gray2black(obj2gco(uv)); + } + uv = uv->u.l.threadnext; + } + return work; +} + /* ** traverse one gray object, turning it to black. ** Returns `quantity' traversed. @@ -334,6 +361,7 @@ static size_t propagatemark(global_State* g) case LUA_TTHREAD: { lua_State* th = gco2th(o); + size_t work = 0; g->gray = th->gclist; LUAU_ASSERT(!luaC_threadsleeping(th)); @@ -344,6 +372,7 @@ static size_t propagatemark(global_State* g) if (!active && g->gcstate == GCSpropagate) { traversestack(g, th, /* clearstack= */ true); + work += markupvals(g, th->openupval); l_setbit(th->stackstate, THREAD_SLEEPINGBIT); } @@ -355,9 +384,13 @@ static size_t propagatemark(global_State* g) black2gray(o); traversestack(g, th, /* clearstack= */ false); + + if (g->gcstate == GCSatomic) { + work += markupvals(g, th->openupval); + } } - return sizeof(lua_State) + sizeof(TValue) * th->stacksize + sizeof(CallInfo) * th->size_ci; + return sizeof(lua_State) + sizeof(TValue) * th->stacksize + sizeof(CallInfo) * th->size_ci + work; } case LUA_TPROTO: { @@ -534,20 +567,6 @@ static void shrinkbuffersfull(lua_State* L) static bool deletegco(void* context, lua_Page* page, GCObject* gco) { - // we are in the process of deleting everything - // threads with open upvalues will attempt to close them all on removal - // but those upvalues might point to stack values that were already deleted - if (gco->gch.tt == LUA_TTHREAD) - { - lua_State* th = gco2th(gco); - - while (UpVal* uv = th->openupval) - { - luaF_unlinkupval(uv); - // close the upvalue without copying the dead data so that luaF_freeupval will not unlink again - uv->v = &uv->u.value; - } - } lua_State* L = (lua_State*)context; freeobj(L, gco, page); @@ -584,6 +603,7 @@ static void markroot(lua_State* L) g->gray = NULL; g->grayagain = NULL; g->weak = NULL; + g->whitethreads = NULL; markobject(g, g->mainthread); // make global table be traversed before main stack markobject(g, g->mainthread->gt); @@ -592,15 +612,27 @@ static void markroot(lua_State* L) g->gcstate = GCSpropagate; } -static size_t remarkupvals(global_State* g) -{ +static size_t clearthreads(lua_State* L, GCObject* t) { size_t work = 0; - for (UpVal* uv = g->uvhead.u.l.next; uv != &g->uvhead; uv = uv->u.l.next) - { - work += sizeof(UpVal); - LUAU_ASSERT(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); - if (isgray(obj2gco(uv))) - markvalue(g, uv->v); + lua_State* s; + UpVal* uv; + UpVal* n; + + while(t) { + s = gco2th(t); + uv = s->openupval; + work += sizeof(UpVal*); + while (uv) { + LUAU_ASSERT(uv->u.l.thread == s); + work += sizeof(UpVal); + n = uv->u.l.threadnext; + if (isblack(obj2gco(uv))) { + setobj(L, &uv->u.value, uv->v); + uv->v = &uv->u.value; + } + uv = n; + } + t = s->gclist; } return work; } @@ -616,8 +648,6 @@ static size_t atomic(lua_State* L) double currts = lua_clock(); #endif - // remark occasional upvalues of (maybe) dead threads - work += remarkupvals(g); // traverse objects caught by write barrier and by 'remarkupvals' work += propagateall(g); @@ -650,6 +680,9 @@ static size_t atomic(lua_State* L) work += cleartable(L, g->weak); g->weak = NULL; + work += clearthreads(L, g->whitethreads); + g->whitethreads = NULL; + #ifdef LUAI_GCMETRICS g->gcmetrics.currcycle.atomictimeclear += recordGcDeltaTime(currts); #endif @@ -953,6 +986,10 @@ void luaC_fullgc(lua_State* L) g->gray = NULL; g->grayagain = NULL; g->weak = NULL; + while (g->whitethreads) { + gco2th(g->whitethreads)->gclistprev = NULL; + g->whitethreads = gco2th(g->whitethreads)->gclist; + } g->gcstate = GCSsweep; } LUAU_ASSERT(g->gcstate == GCSsweep); @@ -1057,26 +1094,6 @@ void luaC_initobj(lua_State* L, GCObject* o, uint8_t tt) o->gch.memcat = L->activememcat; } -void luaC_initupval(lua_State* L, UpVal* uv) -{ - global_State* g = L->global; - GCObject* o = obj2gco(uv); - - if (isgray(o)) - { - if (keepinvariant(g)) - { - gray2black(o); // closed upvalues need barrier - luaC_barrier(L, uv, uv->v); - } - else - { // sweep phase: sweep it (turning it into white) - makewhite(g, o); - LUAU_ASSERT(g->gcstate != GCSpause); - } - } -} - // measure the allocation rate in bytes/sec // returns -1 if allocation rate cannot be measured int64_t luaC_allocationrate(lua_State* L) diff --git a/VM/src/lgc.h b/VM/src/lgc.h index 7b03a25d..85b47212 100644 --- a/VM/src/lgc.h +++ b/VM/src/lgc.h @@ -136,7 +136,6 @@ LUAI_FUNC void luaC_freeall(lua_State* L); LUAI_FUNC size_t luaC_step(lua_State* L, bool assist); LUAI_FUNC void luaC_fullgc(lua_State* L); LUAI_FUNC void luaC_initobj(lua_State* L, GCObject* o, uint8_t tt); -LUAI_FUNC void luaC_initupval(lua_State* L, UpVal* uv); LUAI_FUNC void luaC_barrierupval(lua_State* L, GCObject* v); LUAI_FUNC void luaC_barrierf(lua_State* L, GCObject* o, GCObject* v); LUAI_FUNC void luaC_barriertable(lua_State* L, Table* t, GCObject* v); diff --git a/VM/src/lgcdebug.cpp b/VM/src/lgcdebug.cpp index bc997d44..46bf1fa2 100644 --- a/VM/src/lgcdebug.cpp +++ b/VM/src/lgcdebug.cpp @@ -207,6 +207,17 @@ static void validategraylist(global_State* g, GCObject* o) } } +static void validatewhitethreadlist(global_State* g, GCObject** p) { + GCObject* o; + while ((o = *p) != NULL) + { + LUAU_ASSERT(iswhite(o)); + LUAU_ASSERT(o->gch.tt == LUA_TTHREAD); + LUAU_ASSERT(gco2th(o)->gclistprev == p); + o = gco2th(o)->gclist; + } +} + static bool validategco(void* context, lua_Page* page, GCObject* gco) { lua_State* L = (lua_State*)context; @@ -230,17 +241,11 @@ void luaC_validate(lua_State* L) validategraylist(g, g->weak); validategraylist(g, g->gray); validategraylist(g, g->grayagain); + validatewhitethreadlist(g, &g->whitethreads); validategco(L, NULL, obj2gco(g->mainthread)); luaM_visitgco(L, L, validategco); - - for (UpVal* uv = g->uvhead.u.l.next; uv != &g->uvhead; uv = uv->u.l.next) - { - LUAU_ASSERT(uv->tt == LUA_TUPVAL); - LUAU_ASSERT(uv->v != &uv->u.value); - LUAU_ASSERT(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); - } } inline bool safejson(char ch) diff --git a/VM/src/lobject.h b/VM/src/lobject.h index 2097e335..190212a9 100644 --- a/VM/src/lobject.h +++ b/VM/src/lobject.h @@ -323,14 +323,10 @@ typedef struct UpVal TValue value; // the value (when closed) struct { - // global double linked list (when open) - struct UpVal* prev; - struct UpVal* next; - - // thread double linked list (when open) + // State into which stack this upvalue points at + lua_State* thread; + // Link to next upvalue in sorted order struct UpVal* threadnext; - // note: this is the location of a pointer to this upvalue in the previous element that can be either an UpVal or a lua_State - struct UpVal** threadprev; } l; } u; } UpVal; diff --git a/VM/src/lstate.cpp b/VM/src/lstate.cpp index 4489f840..b908b7e3 100644 --- a/VM/src/lstate.cpp +++ b/VM/src/lstate.cpp @@ -70,6 +70,7 @@ static void preinit_state(lua_State* L, global_State* g) L->stacksize = 0; L->gt = NULL; L->openupval = NULL; + L->gclistprev = NULL; L->size_ci = 0; L->nCcalls = L->baseCcalls = 0; L->status = 0; @@ -119,8 +120,6 @@ lua_State* luaE_newthread(lua_State* L) void luaE_freethread(lua_State* L, lua_State* L1, lua_Page* page) { - luaF_close(L1, L1->stack); // close all upvalues for this thread - LUAU_ASSERT(L1->openupval == NULL); global_State* g = L->global; if (g->cb.userthread) g->cb.userthread(NULL, L1); @@ -175,8 +174,6 @@ lua_State* lua_newstate(lua_Alloc f, void* ud) g->frealloc = f; g->ud = ud; g->mainthread = L; - g->uvhead.u.l.prev = &g->uvhead; - g->uvhead.u.l.next = &g->uvhead; g->GCthreshold = 0; // mark it as unfinished state g->registryfree = 0; g->errorjmp = NULL; @@ -194,6 +191,7 @@ lua_State* lua_newstate(lua_Alloc f, void* ud) g->gray = NULL; g->grayagain = NULL; g->weak = NULL; + g->whitethreads = NULL; g->strbufgc = NULL; g->totalbytes = sizeof(LG); g->gcgoal = LUAI_GCGOAL; diff --git a/VM/src/lstate.h b/VM/src/lstate.h index 72a09713..b3f2b380 100644 --- a/VM/src/lstate.h +++ b/VM/src/lstate.h @@ -166,6 +166,7 @@ typedef struct global_State GCObject* gray; // list of gray objects GCObject* grayagain; // list of objects to be traversed atomically GCObject* weak; // list of weak tables (to be cleared) + GCObject* whitethreads; // list of white threads which might have a black upvalue which need closing TString* strbufgc; // list of all string buffer objects @@ -185,7 +186,6 @@ typedef struct global_State struct lua_State* mainthread; - UpVal uvhead; // head of double-linked list of all open upvalues struct Table* mt[LUA_T_COUNT]; // metatables for basic types TString* ttname[LUA_T_COUNT]; // names for basic types TString* tmname[TM_N]; // array with tag-method names @@ -252,6 +252,7 @@ struct lua_State Table* gt; // table of globals UpVal* openupval; // list of open upvalues in this stack GCObject* gclist; + GCObject** gclistprev; // Only used when on white list. TString* namecall; // when invoked from Luau using NAMECALL, what method do we need to invoke? diff --git a/bench/gc/test_GC_upvalues_dead.lua b/bench/gc/test_GC_upvalues_dead.lua new file mode 100644 index 00000000..7849170f --- /dev/null +++ b/bench/gc/test_GC_upvalues_dead.lua @@ -0,0 +1,17 @@ +local bench = script and require(script.Parent.bench_support) or require("bench_support") + +function test() + + local ts0 = os.clock() + for i=1,1000000 do + local up = 1 + local function f() + up = 2 + end + end + local ts1 = os.clock() + + return ts1-ts0 +end + +bench.runCode(test, "Upvalues: dead") \ No newline at end of file diff --git a/bench/gc/test_GC_upvalues_gray.lua b/bench/gc/test_GC_upvalues_gray.lua new file mode 100644 index 00000000..0ec057d8 --- /dev/null +++ b/bench/gc/test_GC_upvalues_gray.lua @@ -0,0 +1,23 @@ +local bench = script and require(script.Parent.bench_support) or require("bench_support") + +function test() + + local function makeup() + local up = 1 + local function f() + up = 2 + end + coroutine.yield() + end + + local ts0 = os.clock() + for i=1,100000 do + local co = coroutine.create(makeup) + coroutine.resume(co) + end + local ts1 = os.clock() + + return ts1-ts0 +end + +bench.runCode(test, "Upvalues: gray") \ No newline at end of file