From bb283f4581ac07a4dd75325c4343d7f28b1ff376 Mon Sep 17 00:00:00 2001
From: Andy Belle-Isle <drumsetmonkey@gmail.com>
Date: Sun, 25 Aug 2019 16:39:02 -0400
Subject: Fixed LuaJIT files
---
lib/LuaJIT/src/lj_gc.c | 854 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 854 insertions(+)
create mode 100644 lib/LuaJIT/src/lj_gc.c
(limited to 'lib/LuaJIT/src/lj_gc.c')
diff --git a/lib/LuaJIT/src/lj_gc.c b/lib/LuaJIT/src/lj_gc.c
new file mode 100644
index 0000000..2aaf5b2
--- /dev/null
+++ b/lib/LuaJIT/src/lj_gc.c
@@ -0,0 +1,854 @@
+/*
+** Garbage collector.
+** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
+**
+** Major portions taken verbatim or adapted from the Lua interpreter.
+** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
+*/
+
+#define lj_gc_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+#include "lj_gc.h"
+#include "lj_err.h"
+#include "lj_buf.h"
+#include "lj_str.h"
+#include "lj_tab.h"
+#include "lj_func.h"
+#include "lj_udata.h"
+#include "lj_meta.h"
+#include "lj_state.h"
+#include "lj_frame.h"
+#if LJ_HASFFI
+#include "lj_ctype.h"
+#include "lj_cdata.h"
+#endif
+#include "lj_trace.h"
+#include "lj_vm.h"
+
+#define GCSTEPSIZE 1024u
+#define GCSWEEPMAX 40
+#define GCSWEEPCOST 10
+#define GCFINALIZECOST 100
+
+/* Macros to set GCobj colors and flags. */
+#define white2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_WHITES)
+#define gray2black(x) ((x)->gch.marked |= LJ_GC_BLACK)
+#define isfinalized(u) ((u)->marked & LJ_GC_FINALIZED)
+
+/* -- Mark phase ---------------------------------------------------------- */
+
+/* Mark a TValue (if needed). */
+#define gc_marktv(g, tv) \
+ { lua_assert(!tvisgcv(tv) || (~itype(tv) == gcval(tv)->gch.gct)); \
+ if (tviswhite(tv)) gc_mark(g, gcV(tv)); }
+
+/* Mark a GCobj (if needed). */
+#define gc_markobj(g, o) \
+ { if (iswhite(obj2gco(o))) gc_mark(g, obj2gco(o)); }
+
+/* Mark a string object. */
+#define gc_mark_str(s) ((s)->marked &= (uint8_t)~LJ_GC_WHITES)
+
+/* Mark a white GCobj. */
+static void gc_mark(global_State *g, GCobj *o)
+{
+ int gct = o->gch.gct;
+ lua_assert(iswhite(o) && !isdead(g, o));
+ white2gray(o);
+ if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) {
+ GCtab *mt = tabref(gco2ud(o)->metatable);
+ gray2black(o); /* Userdata are never gray. */
+ if (mt) gc_markobj(g, mt);
+ gc_markobj(g, tabref(gco2ud(o)->env));
+ } else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) {
+ GCupval *uv = gco2uv(o);
+ gc_marktv(g, uvval(uv));
+ if (uv->closed)
+ gray2black(o); /* Closed upvalues are never gray. */
+ } else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) {
+ lua_assert(gct == ~LJ_TFUNC || gct == ~LJ_TTAB ||
+ gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO || gct == ~LJ_TTRACE);
+ setgcrefr(o->gch.gclist, g->gc.gray);
+ setgcref(g->gc.gray, o);
+ }
+}
+
+/* Mark GC roots. */
+static void gc_mark_gcroot(global_State *g)
+{
+ ptrdiff_t i;
+ for (i = 0; i < GCROOT_MAX; i++)
+ if (gcref(g->gcroot[i]) != NULL)
+ gc_markobj(g, gcref(g->gcroot[i]));
+}
+
+/* Start a GC cycle and mark the root set. */
+static void gc_mark_start(global_State *g)
+{
+ setgcrefnull(g->gc.gray);
+ setgcrefnull(g->gc.grayagain);
+ setgcrefnull(g->gc.weak);
+ gc_markobj(g, mainthread(g));
+ gc_markobj(g, tabref(mainthread(g)->env));
+ gc_marktv(g, &g->registrytv);
+ gc_mark_gcroot(g);
+ g->gc.state = GCSpropagate;
+}
+
+/* Mark open upvalues. */
+static void gc_mark_uv(global_State *g)
+{
+ GCupval *uv;
+ for (uv = uvnext(&g->uvhead); uv != &g->uvhead; uv = uvnext(uv)) {
+ lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv);
+ if (isgray(obj2gco(uv)))
+ gc_marktv(g, uvval(uv));
+ }
+}
+
+/* Mark userdata in mmudata list. */
+static void gc_mark_mmudata(global_State *g)
+{
+ GCobj *root = gcref(g->gc.mmudata);
+ GCobj *u = root;
+ if (u) {
+ do {
+ u = gcnext(u);
+ makewhite(g, u); /* Could be from previous GC. */
+ gc_mark(g, u);
+ } while (u != root);
+ }
+}
+
+/* Separate userdata objects to be finalized to mmudata list. */
+size_t lj_gc_separateudata(global_State *g, int all)
+{
+ size_t m = 0;
+ GCRef *p = &mainthread(g)->nextgc;
+ GCobj *o;
+ while ((o = gcref(*p)) != NULL) {
+ if (!(iswhite(o) || all) || isfinalized(gco2ud(o))) {
+ p = &o->gch.nextgc; /* Nothing to do. */
+ } else if (!lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc)) {
+ markfinalized(o); /* Done, as there's no __gc metamethod. */
+ p = &o->gch.nextgc;
+ } else { /* Otherwise move userdata to be finalized to mmudata list. */
+ m += sizeudata(gco2ud(o));
+ markfinalized(o);
+ *p = o->gch.nextgc;
+ if (gcref(g->gc.mmudata)) { /* Link to end of mmudata list. */
+ GCobj *root = gcref(g->gc.mmudata);
+ setgcrefr(o->gch.nextgc, root->gch.nextgc);
+ setgcref(root->gch.nextgc, o);
+ setgcref(g->gc.mmudata, o);
+ } else { /* Create circular list. */
+ setgcref(o->gch.nextgc, o);
+ setgcref(g->gc.mmudata, o);
+ }
+ }
+ }
+ return m;
+}
+
+/* -- Propagation phase --------------------------------------------------- */
+
+/* Traverse a table. */
+static int gc_traverse_tab(global_State *g, GCtab *t)
+{
+ int weak = 0;
+ cTValue *mode;
+ GCtab *mt = tabref(t->metatable);
+ if (mt)
+ gc_markobj(g, mt);
+ mode = lj_meta_fastg(g, mt, MM_mode);
+ if (mode && tvisstr(mode)) { /* Valid __mode field? */
+ const char *modestr = strVdata(mode);
+ int c;
+ while ((c = *modestr++)) {
+ if (c == 'k') weak |= LJ_GC_WEAKKEY;
+ else if (c == 'v') weak |= LJ_GC_WEAKVAL;
+ }
+ if (weak) { /* Weak tables are cleared in the atomic phase. */
+#if LJ_HASFFI
+ CTState *cts = ctype_ctsG(g);
+ if (cts && cts->finalizer == t) {
+ weak = (int)(~0u & ~LJ_GC_WEAKVAL);
+ } else
+#endif
+ {
+ t->marked = (uint8_t)((t->marked & ~LJ_GC_WEAK) | weak);
+ setgcrefr(t->gclist, g->gc.weak);
+ setgcref(g->gc.weak, obj2gco(t));
+ }
+ }
+ }
+ if (weak == LJ_GC_WEAK) /* Nothing to mark if both keys/values are weak. */
+ return 1;
+ if (!(weak & LJ_GC_WEAKVAL)) { /* Mark array part. */
+ MSize i, asize = t->asize;
+ for (i = 0; i < asize; i++)
+ gc_marktv(g, arrayslot(t, i));
+ }
+ if (t->hmask > 0) { /* Mark hash part. */
+ Node *node = noderef(t->node);
+ MSize i, hmask = t->hmask;
+ for (i = 0; i <= hmask; i++) {
+ Node *n = &node[i];
+ if (!tvisnil(&n->val)) { /* Mark non-empty slot. */
+ lua_assert(!tvisnil(&n->key));
+ if (!(weak & LJ_GC_WEAKKEY)) gc_marktv(g, &n->key);
+ if (!(weak & LJ_GC_WEAKVAL)) gc_marktv(g, &n->val);
+ }
+ }
+ }
+ return weak;
+}
+
+/* Traverse a function. */
+static void gc_traverse_func(global_State *g, GCfunc *fn)
+{
+ gc_markobj(g, tabref(fn->c.env));
+ if (isluafunc(fn)) {
+ uint32_t i;
+ lua_assert(fn->l.nupvalues <= funcproto(fn)->sizeuv);
+ gc_markobj(g, funcproto(fn));
+ for (i = 0; i < fn->l.nupvalues; i++) /* Mark Lua function upvalues. */
+ gc_markobj(g, &gcref(fn->l.uvptr[i])->uv);
+ } else {
+ uint32_t i;
+ for (i = 0; i < fn->c.nupvalues; i++) /* Mark C function upvalues. */
+ gc_marktv(g, &fn->c.upvalue[i]);
+ }
+}
+
+#if LJ_HASJIT
+/* Mark a trace. */
+static void gc_marktrace(global_State *g, TraceNo traceno)
+{
+ GCobj *o = obj2gco(traceref(G2J(g), traceno));
+ lua_assert(traceno != G2J(g)->cur.traceno);
+ if (iswhite(o)) {
+ white2gray(o);
+ setgcrefr(o->gch.gclist, g->gc.gray);
+ setgcref(g->gc.gray, o);
+ }
+}
+
+/* Traverse a trace. */
+static void gc_traverse_trace(global_State *g, GCtrace *T)
+{
+ IRRef ref;
+ if (T->traceno == 0) return;
+ for (ref = T->nk; ref < REF_TRUE; ref++) {
+ IRIns *ir = &T->ir[ref];
+ if (ir->o == IR_KGC)
+ gc_markobj(g, ir_kgc(ir));
+ if (irt_is64(ir->t) && ir->o != IR_KNULL)
+ ref++;
+ }
+ if (T->link) gc_marktrace(g, T->link);
+ if (T->nextroot) gc_marktrace(g, T->nextroot);
+ if (T->nextside) gc_marktrace(g, T->nextside);
+ gc_markobj(g, gcref(T->startpt));
+}
+
+/* The current trace is a GC root while not anchored in the prototype (yet). */
+#define gc_traverse_curtrace(g) gc_traverse_trace(g, &G2J(g)->cur)
+#else
+#define gc_traverse_curtrace(g) UNUSED(g)
+#endif
+
+/* Traverse a prototype. */
+static void gc_traverse_proto(global_State *g, GCproto *pt)
+{
+ ptrdiff_t i;
+ gc_mark_str(proto_chunkname(pt));
+ for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) /* Mark collectable consts. */
+ gc_markobj(g, proto_kgc(pt, i));
+#if LJ_HASJIT
+ if (pt->trace) gc_marktrace(g, pt->trace);
+#endif
+}
+
+/* Traverse the frame structure of a stack. */
+static MSize gc_traverse_frames(global_State *g, lua_State *th)
+{
+ TValue *frame, *top = th->top-1, *bot = tvref(th->stack);
+ /* Note: extra vararg frame not skipped, marks function twice (harmless). */
+ for (frame = th->base-1; frame > bot+LJ_FR2; frame = frame_prev(frame)) {
+ GCfunc *fn = frame_func(frame);
+ TValue *ftop = frame;
+ if (isluafunc(fn)) ftop += funcproto(fn)->framesize;
+ if (ftop > top) top = ftop;
+ if (!LJ_FR2) gc_markobj(g, fn); /* Need to mark hidden function (or L). */
+ }
+ top++; /* Correct bias of -1 (frame == base-1). */
+ if (top > tvref(th->maxstack)) top = tvref(th->maxstack);
+ return (MSize)(top - bot); /* Return minimum needed stack size. */
+}
+
+/* Traverse a thread object. */
+static void gc_traverse_thread(global_State *g, lua_State *th)
+{
+ TValue *o, *top = th->top;
+ for (o = tvref(th->stack)+1+LJ_FR2; o < top; o++)
+ gc_marktv(g, o);
+ if (g->gc.state == GCSatomic) {
+ top = tvref(th->stack) + th->stacksize;
+ for (; o < top; o++) /* Clear unmarked slots. */
+ setnilV(o);
+ }
+ gc_markobj(g, tabref(th->env));
+ lj_state_shrinkstack(th, gc_traverse_frames(g, th));
+}
+
+/* Propagate one gray object. Traverse it and turn it black. */
+static size_t propagatemark(global_State *g)
+{
+ GCobj *o = gcref(g->gc.gray);
+ int gct = o->gch.gct;
+ lua_assert(isgray(o));
+ gray2black(o);
+ setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */
+ if (LJ_LIKELY(gct == ~LJ_TTAB)) {
+ GCtab *t = gco2tab(o);
+ if (gc_traverse_tab(g, t) > 0)
+ black2gray(o); /* Keep weak tables gray. */
+ return sizeof(GCtab) + sizeof(TValue) * t->asize +
+ (t->hmask ? sizeof(Node) * (t->hmask + 1) : 0);
+ } else if (LJ_LIKELY(gct == ~LJ_TFUNC)) {
+ GCfunc *fn = gco2func(o);
+ gc_traverse_func(g, fn);
+ return isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) :
+ sizeCfunc((MSize)fn->c.nupvalues);
+ } else if (LJ_LIKELY(gct == ~LJ_TPROTO)) {
+ GCproto *pt = gco2pt(o);
+ gc_traverse_proto(g, pt);
+ return pt->sizept;
+ } else if (LJ_LIKELY(gct == ~LJ_TTHREAD)) {
+ lua_State *th = gco2th(o);
+ setgcrefr(th->gclist, g->gc.grayagain);
+ setgcref(g->gc.grayagain, o);
+ black2gray(o); /* Threads are never black. */
+ gc_traverse_thread(g, th);
+ return sizeof(lua_State) + sizeof(TValue) * th->stacksize;
+ } else {
+#if LJ_HASJIT
+ GCtrace *T = gco2trace(o);
+ gc_traverse_trace(g, T);
+ return ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
+ T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry);
+#else
+ lua_assert(0);
+ return 0;
+#endif
+ }
+}
+
+/* Propagate all gray objects. */
+static size_t gc_propagate_gray(global_State *g)
+{
+ size_t m = 0;
+ while (gcref(g->gc.gray) != NULL)
+ m += propagatemark(g);
+ return m;
+}
+
+/* -- Sweep phase --------------------------------------------------------- */
+
+/* Type of GC free functions. */
+typedef void (LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o);
+
+/* GC free functions for LJ_TSTR .. LJ_TUDATA. ORDER LJ_T */
+static const GCFreeFunc gc_freefunc[] = {
+ (GCFreeFunc)lj_str_free,
+ (GCFreeFunc)lj_func_freeuv,
+ (GCFreeFunc)lj_state_free,
+ (GCFreeFunc)lj_func_freeproto,
+ (GCFreeFunc)lj_func_free,
+#if LJ_HASJIT
+ (GCFreeFunc)lj_trace_free,
+#else
+ (GCFreeFunc)0,
+#endif
+#if LJ_HASFFI
+ (GCFreeFunc)lj_cdata_free,
+#else
+ (GCFreeFunc)0,
+#endif
+ (GCFreeFunc)lj_tab_free,
+ (GCFreeFunc)lj_udata_free
+};
+
+/* Full sweep of a GC list. */
+#define gc_fullsweep(g, p) gc_sweep(g, (p), ~(uint32_t)0)
+
+/* Partial sweep of a GC list. */
+static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
+{
+ /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */
+ int ow = otherwhite(g);
+ GCobj *o;
+ while ((o = gcref(*p)) != NULL && lim-- > 0) {
+ if (o->gch.gct == ~LJ_TTHREAD) /* Need to sweep open upvalues, too. */
+ gc_fullsweep(g, &gco2th(o)->openupval);
+ if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* Black or current white? */
+ lua_assert(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED));
+ makewhite(g, o); /* Value is alive, change to the current white. */
+ p = &o->gch.nextgc;
+ } else { /* Otherwise value is dead, free it. */
+ lua_assert(isdead(g, o) || ow == LJ_GC_SFIXED);
+ setgcrefr(*p, o->gch.nextgc);
+ if (o == gcref(g->gc.root))
+ setgcrefr(g->gc.root, o->gch.nextgc); /* Adjust list anchor. */
+ gc_freefunc[o->gch.gct - ~LJ_TSTR](g, o);
+ }
+ }
+ return p;
+}
+
+/* Check whether we can clear a key or a value slot from a table. */
+static int gc_mayclear(cTValue *o, int val)
+{
+ if (tvisgcv(o)) { /* Only collectable objects can be weak references. */
+ if (tvisstr(o)) { /* But strings cannot be used as weak references. */
+ gc_mark_str(strV(o)); /* And need to be marked. */
+ return 0;
+ }
+ if (iswhite(gcV(o)))
+ return 1; /* Object is about to be collected. */
+ if (tvisudata(o) && val && isfinalized(udataV(o)))
+ return 1; /* Finalized userdata is dropped only from values. */
+ }
+ return 0; /* Cannot clear. */
+}
+
+/* Clear collected entries from weak tables. */
+static void gc_clearweak(GCobj *o)
+{
+ while (o) {
+ GCtab *t = gco2tab(o);
+ lua_assert((t->marked & LJ_GC_WEAK));
+ if ((t->marked & LJ_GC_WEAKVAL)) {
+ MSize i, asize = t->asize;
+ for (i = 0; i < asize; i++) {
+ /* Clear array slot when value is about to be collected. */
+ TValue *tv = arrayslot(t, i);
+ if (gc_mayclear(tv, 1))
+ setnilV(tv);
+ }
+ }
+ if (t->hmask > 0) {
+ Node *node = noderef(t->node);
+ MSize i, hmask = t->hmask;
+ for (i = 0; i <= hmask; i++) {
+ Node *n = &node[i];
+ /* Clear hash slot when key or value is about to be collected. */
+ if (!tvisnil(&n->val) && (gc_mayclear(&n->key, 0) ||
+ gc_mayclear(&n->val, 1)))
+ setnilV(&n->val);
+ }
+ }
+ o = gcref(t->gclist);
+ }
+}
+
+/* Call a userdata or cdata finalizer. */
+static void gc_call_finalizer(global_State *g, lua_State *L,
+ cTValue *mo, GCobj *o)
+{
+ /* Save and restore lots of state around the __gc callback. */
+ uint8_t oldh = hook_save(g);
+ GCSize oldt = g->gc.threshold;
+ int errcode;
+ TValue *top;
+ lj_trace_abort(g);
+ hook_entergc(g); /* Disable hooks and new traces during __gc. */
+ g->gc.threshold = LJ_MAX_MEM; /* Prevent GC steps. */
+ top = L->top;
+ copyTV(L, top++, mo);
+ if (LJ_FR2) setnilV(top++);
+ setgcV(L, top, o, ~o->gch.gct);
+ L->top = top+1;
+ errcode = lj_vm_pcall(L, top, 1+0, -1); /* Stack: |mo|o| -> | */
+ hook_restore(g, oldh);
+ g->gc.threshold = oldt; /* Restore GC threshold. */
+ if (errcode)
+ lj_err_throw(L, errcode); /* Propagate errors. */
+}
+
+/* Finalize one userdata or cdata object from the mmudata list. */
+static void gc_finalize(lua_State *L)
+{
+ global_State *g = G(L);
+ GCobj *o = gcnext(gcref(g->gc.mmudata));
+ cTValue *mo;
+ lua_assert(tvref(g->jit_base) == NULL); /* Must not be called on trace. */
+ /* Unchain from list of userdata to be finalized. */
+ if (o == gcref(g->gc.mmudata))
+ setgcrefnull(g->gc.mmudata);
+ else
+ setgcrefr(gcref(g->gc.mmudata)->gch.nextgc, o->gch.nextgc);
+#if LJ_HASFFI
+ if (o->gch.gct == ~LJ_TCDATA) {
+ TValue tmp, *tv;
+ /* Add cdata back to the GC list and make it white. */
+ setgcrefr(o->gch.nextgc, g->gc.root);
+ setgcref(g->gc.root, o);
+ makewhite(g, o);
+ o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN;
+ /* Resolve finalizer. */
+ setcdataV(L, &tmp, gco2cd(o));
+ tv = lj_tab_set(L, ctype_ctsG(g)->finalizer, &tmp);
+ if (!tvisnil(tv)) {
+ g->gc.nocdatafin = 0;
+ copyTV(L, &tmp, tv);
+ setnilV(tv); /* Clear entry in finalizer table. */
+ gc_call_finalizer(g, L, &tmp, o);
+ }
+ return;
+ }
+#endif
+ /* Add userdata back to the main userdata list and make it white. */
+ setgcrefr(o->gch.nextgc, mainthread(g)->nextgc);
+ setgcref(mainthread(g)->nextgc, o);
+ makewhite(g, o);
+ /* Resolve the __gc metamethod. */
+ mo = lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc);
+ if (mo)
+ gc_call_finalizer(g, L, mo, o);
+}
+
+/* Finalize all userdata objects from mmudata list. */
+void lj_gc_finalize_udata(lua_State *L)
+{
+ while (gcref(G(L)->gc.mmudata) != NULL)
+ gc_finalize(L);
+}
+
+#if LJ_HASFFI
+/* Finalize all cdata objects from finalizer table. */
+void lj_gc_finalize_cdata(lua_State *L)
+{
+ global_State *g = G(L);
+ CTState *cts = ctype_ctsG(g);
+ if (cts) {
+ GCtab *t = cts->finalizer;
+ Node *node = noderef(t->node);
+ ptrdiff_t i;
+ setgcrefnull(t->metatable); /* Mark finalizer table as disabled. */
+ for (i = (ptrdiff_t)t->hmask; i >= 0; i--)
+ if (!tvisnil(&node[i].val) && tviscdata(&node[i].key)) {
+ GCobj *o = gcV(&node[i].key);
+ TValue tmp;
+ makewhite(g, o);
+ o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN;
+ copyTV(L, &tmp, &node[i].val);
+ setnilV(&node[i].val);
+ gc_call_finalizer(g, L, &tmp, o);
+ }
+ }
+}
+#endif
+
+/* Free all remaining GC objects. */
+void lj_gc_freeall(global_State *g)
+{
+ MSize i, strmask;
+ /* Free everything, except super-fixed objects (the main thread). */
+ g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED;
+ gc_fullsweep(g, &g->gc.root);
+ strmask = g->strmask;
+ for (i = 0; i <= strmask; i++) /* Free all string hash chains. */
+ gc_fullsweep(g, &g->strhash[i]);
+}
+
+/* -- Collector ----------------------------------------------------------- */
+
+/* Atomic part of the GC cycle, transitioning from mark to sweep phase. */
+static void atomic(global_State *g, lua_State *L)
+{
+ size_t udsize;
+
+ gc_mark_uv(g); /* Need to remark open upvalues (the thread may be dead). */
+ gc_propagate_gray(g); /* Propagate any left-overs. */
+
+ setgcrefr(g->gc.gray, g->gc.weak); /* Empty the list of weak tables. */
+ setgcrefnull(g->gc.weak);
+ lua_assert(!iswhite(obj2gco(mainthread(g))));
+ gc_markobj(g, L); /* Mark running thread. */
+ gc_traverse_curtrace(g); /* Traverse current trace. */
+ gc_mark_gcroot(g); /* Mark GC roots (again). */
+ gc_propagate_gray(g); /* Propagate all of the above. */
+
+ setgcrefr(g->gc.gray, g->gc.grayagain); /* Empty the 2nd chance list. */
+ setgcrefnull(g->gc.grayagain);
+ gc_propagate_gray(g); /* Propagate it. */
+
+ udsize = lj_gc_separateudata(g, 0); /* Separate userdata to be finalized. */
+ gc_mark_mmudata(g); /* Mark them. */
+ udsize += gc_propagate_gray(g); /* And propagate the marks. */
+
+ /* All marking done, clear weak tables. */
+ gc_clearweak(gcref(g->gc.weak));
+
+ lj_buf_shrink(L, &g->tmpbuf); /* Shrink temp buffer. */
+
+ /* Prepare for sweep phase. */
+ g->gc.currentwhite = (uint8_t)otherwhite(g); /* Flip current white. */
+ g->strempty.marked = g->gc.currentwhite;
+ setmref(g->gc.sweep, &g->gc.root);
+ g->gc.estimate = g->gc.total - (GCSize)udsize; /* Initial estimate. */
+}
+
+/* GC state machine. Returns a cost estimate for each step performed. */
+static size_t gc_onestep(lua_State *L)
+{
+ global_State *g = G(L);
+ switch (g->gc.state) {
+ case GCSpause:
+ gc_mark_start(g); /* Start a new GC cycle by marking all GC roots. */
+ return 0;
+ case GCSpropagate:
+ if (gcref(g->gc.gray) != NULL)
+ return propagatemark(g); /* Propagate one gray object. */
+ g->gc.state = GCSatomic; /* End of mark phase. */
+ return 0;
+ case GCSatomic:
+ if (tvref(g->jit_base)) /* Don't run atomic phase on trace. */
+ return LJ_MAX_MEM;
+ atomic(g, L);
+ g->gc.state = GCSsweepstring; /* Start of sweep phase. */
+ g->gc.sweepstr = 0;
+ return 0;
+ case GCSsweepstring: {
+ GCSize old = g->gc.total;
+ gc_fullsweep(g, &g->strhash[g->gc.sweepstr++]); /* Sweep one chain. */
+ if (g->gc.sweepstr > g->strmask)
+ g->gc.state = GCSsweep; /* All string hash chains sweeped. */
+ lua_assert(old >= g->gc.total);
+ g->gc.estimate -= old - g->gc.total;
+ return GCSWEEPCOST;
+ }
+ case GCSsweep: {
+ GCSize old = g->gc.total;
+ setmref(g->gc.sweep, gc_sweep(g, mref(g->gc.sweep, GCRef), GCSWEEPMAX));
+ lua_assert(old >= g->gc.total);
+ g->gc.estimate -= old - g->gc.total;
+ if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) {
+ if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB*2-1)
+ lj_str_resize(L, g->strmask >> 1); /* Shrink string table. */
+ if (gcref(g->gc.mmudata)) { /* Need any finalizations? */
+ g->gc.state = GCSfinalize;
+#if LJ_HASFFI
+ g->gc.nocdatafin = 1;
+#endif
+ } else { /* Otherwise skip this phase to help the JIT. */
+ g->gc.state = GCSpause; /* End of GC cycle. */
+ g->gc.debt = 0;
+ }
+ }
+ return GCSWEEPMAX*GCSWEEPCOST;
+ }
+ case GCSfinalize:
+ if (gcref(g->gc.mmudata) != NULL) {
+ if (tvref(g->jit_base)) /* Don't call finalizers on trace. */
+ return LJ_MAX_MEM;
+ gc_finalize(L); /* Finalize one userdata object. */
+ if (g->gc.estimate > GCFINALIZECOST)
+ g->gc.estimate -= GCFINALIZECOST;
+ return GCFINALIZECOST;
+ }
+#if LJ_HASFFI
+ if (!g->gc.nocdatafin) lj_tab_rehash(L, ctype_ctsG(g)->finalizer);
+#endif
+ g->gc.state = GCSpause; /* End of GC cycle. */
+ g->gc.debt = 0;
+ return 0;
+ default:
+ lua_assert(0);
+ return 0;
+ }
+}
+
+/* Perform a limited amount of incremental GC steps. */
+int LJ_FASTCALL lj_gc_step(lua_State *L)
+{
+ global_State *g = G(L);
+ GCSize lim;
+ int32_t ostate = g->vmstate;
+ setvmstate(g, GC);
+ lim = (GCSTEPSIZE/100) * g->gc.stepmul;
+ if (lim == 0)
+ lim = LJ_MAX_MEM;
+ if (g->gc.total > g->gc.threshold)
+ g->gc.debt += g->gc.total - g->gc.threshold;
+ do {
+ lim -= (GCSize)gc_onestep(L);
+ if (g->gc.state == GCSpause) {
+ g->gc.threshold = (g->gc.estimate/100) * g->gc.pause;
+ g->vmstate = ostate;
+ return 1; /* Finished a GC cycle. */
+ }
+ } while (sizeof(lim) == 8 ? ((int64_t)lim > 0) : ((int32_t)lim > 0));
+ if (g->gc.debt < GCSTEPSIZE) {
+ g->gc.threshold = g->gc.total + GCSTEPSIZE;
+ g->vmstate = ostate;
+ return -1;
+ } else {
+ g->gc.debt -= GCSTEPSIZE;
+ g->gc.threshold = g->gc.total;
+ g->vmstate = ostate;
+ return 0;
+ }
+}
+
+/* Ditto, but fix the stack top first. */
+void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L)
+{
+ if (curr_funcisL(L)) L->top = curr_topL(L);
+ lj_gc_step(L);
+}
+
+#if LJ_HASJIT
+/* Perform multiple GC steps. Called from JIT-compiled code. */
+int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps)
+{
+ lua_State *L = gco2th(gcref(g->cur_L));
+ L->base = tvref(G(L)->jit_base);
+ L->top = curr_topL(L);
+ while (steps-- > 0 && lj_gc_step(L) == 0)
+ ;
+ /* Return 1 to force a trace exit. */
+ return (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize);
+}
+#endif
+
+/* Perform a full GC cycle. */
+void lj_gc_fullgc(lua_State *L)
+{
+ global_State *g = G(L);
+ int32_t ostate = g->vmstate;
+ setvmstate(g, GC);
+ if (g->gc.state <= GCSatomic) { /* Caught somewhere in the middle. */
+ setmref(g->gc.sweep, &g->gc.root); /* Sweep everything (preserving it). */
+ setgcrefnull(g->gc.gray); /* Reset lists from partial propagation. */
+ setgcrefnull(g->gc.grayagain);
+ setgcrefnull(g->gc.weak);
+ g->gc.state = GCSsweepstring; /* Fast forward to the sweep phase. */
+ g->gc.sweepstr = 0;
+ }
+ while (g->gc.state == GCSsweepstring || g->gc.state == GCSsweep)
+ gc_onestep(L); /* Finish sweep. */
+ lua_assert(g->gc.state == GCSfinalize || g->gc.state == GCSpause);
+ /* Now perform a full GC. */
+ g->gc.state = GCSpause;
+ do { gc_onestep(L); } while (g->gc.state != GCSpause);
+ g->gc.threshold = (g->gc.estimate/100) * g->gc.pause;
+ g->vmstate = ostate;
+}
+
+/* -- Write barriers ------------------------------------------------------ */
+
+/* Move the GC propagation frontier forward. */
+void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v)
+{
+ lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o));
+ lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
+ lua_assert(o->gch.gct != ~LJ_TTAB);
+ /* Preserve invariant during propagation. Otherwise it doesn't matter. */
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
+ gc_mark(g, v); /* Move frontier forward. */
+ else
+ makewhite(g, o); /* Make it white to avoid the following barrier. */
+}
+
+/* Specialized barrier for closed upvalue. Pass &uv->tv. */
+void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv)
+{
+#define TV2MARKED(x) \
+ (*((uint8_t *)(x) - offsetof(GCupval, tv) + offsetof(GCupval, marked)))
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
+ gc_mark(g, gcV(tv));
+ else
+ TV2MARKED(tv) = (TV2MARKED(tv) & (uint8_t)~LJ_GC_COLORS) | curwhite(g);
+#undef TV2MARKED
+}
+
+/* Close upvalue. Also needs a write barrier. */
+void lj_gc_closeuv(global_State *g, GCupval *uv)
+{
+ GCobj *o = obj2gco(uv);
+ /* Copy stack slot to upvalue itself and point to the copy. */
+ copyTV(mainthread(g), &uv->tv, uvval(uv));
+ setmref(uv->v, &uv->tv);
+ uv->closed = 1;
+ setgcrefr(o->gch.nextgc, g->gc.root);
+ setgcref(g->gc.root, o);
+ if (isgray(o)) { /* A closed upvalue is never gray, so fix this. */
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) {
+ gray2black(o); /* Make it black and preserve invariant. */
+ if (tviswhite(&uv->tv))
+ lj_gc_barrierf(g, o, gcV(&uv->tv));
+ } else {
+ makewhite(g, o); /* Make it white, i.e. sweep the upvalue. */
+ lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
+ }
+ }
+}
+
+#if LJ_HASJIT
+/* Mark a trace if it's saved during the propagation phase. */
+void lj_gc_barriertrace(global_State *g, uint32_t traceno)
+{
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
+ gc_marktrace(g, traceno);
+}
+#endif
+
+/* -- Allocator ----------------------------------------------------------- */
+
+/* Call pluggable memory allocator to allocate or resize a fragment. */
+void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz)
+{
+ global_State *g = G(L);
+ lua_assert((osz == 0) == (p == NULL));
+ p = g->allocf(g->allocd, p, osz, nsz);
+ if (p == NULL && nsz > 0)
+ lj_err_mem(L);
+ lua_assert((nsz == 0) == (p == NULL));
+ lua_assert(checkptrGC(p));
+ g->gc.total = (g->gc.total - osz) + nsz;
+ return p;
+}
+
+/* Allocate new GC object and link it to the root set. */
+void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size)
+{
+ global_State *g = G(L);
+ GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size);
+ if (o == NULL)
+ lj_err_mem(L);
+ lua_assert(checkptrGC(o));
+ g->gc.total += size;
+ setgcrefr(o->gch.nextgc, g->gc.root);
+ setgcref(g->gc.root, o);
+ newwhite(g, o);
+ return o;
+}
+
+/* Resize growable vector. */
+void *lj_mem_grow(lua_State *L, void *p, MSize *szp, MSize lim, MSize esz)
+{
+ MSize sz = (*szp) << 1;
+ if (sz < LJ_MIN_VECSZ)
+ sz = LJ_MIN_VECSZ;
+ if (sz > lim)
+ sz = lim;
+ p = lj_mem_realloc(L, p, (*szp)*esz, sz*esz);
+ *szp = sz;
+ return p;
+}
+
--
cgit v1.2.3