* This code synchronizes with interp_mark_stack () using compiler memory barriers.
*/
-static StackFragment*
-stack_frag_new (int size)
+static FrameDataFragment*
+frame_data_frag_new (int size)
{
- StackFragment *frag = (StackFragment*)g_malloc (size);
+ FrameDataFragment *frag = (FrameDataFragment*)g_malloc (size);
frag->pos = (guint8*)&frag->data;
frag->end = (guint8*)frag + size;
}
static void
-frame_stack_init (FrameStack *stack, int size)
+frame_data_frag_free (FrameDataFragment *frag)
{
- StackFragment *frag;
+ while (frag) {
+ FrameDataFragment *next = frag->next;
+ g_free (frag);
+ frag = next;
+ }
+}
+
+static void
+frame_data_allocator_init (FrameDataAllocator *stack, int size)
+{
+ FrameDataFragment *frag;
- frag = stack_frag_new (size);
+ frag = frame_data_frag_new (size);
stack->first = stack->current = frag;
- mono_compiler_barrier ();
- stack->inited = 1;
+ stack->infos_capacity = 4;
+ stack->infos = g_malloc (stack->infos_capacity * sizeof (FrameDataInfo));
}
-static StackFragment*
-add_frag (FrameStack *stack, int size)
+static void
+frame_data_allocator_free (FrameDataAllocator *stack)
{
- StackFragment *new_frag;
+ /* Assert to catch leaks */
+ g_assert_checked (stack->current == stack->first && stack->current->pos == (guint8*)&stack->current->data);
+ frame_data_frag_free (stack->first);
+}
+
+static FrameDataFragment*
+frame_data_allocator_add_frag (FrameDataAllocator *stack, int size)
+{
+ FrameDataFragment *new_frag;
// FIXME:
int frag_size = 4096;
- if (size + sizeof (StackFragment) > frag_size)
- frag_size = size + sizeof (StackFragment);
- new_frag = stack_frag_new (frag_size);
+ if (size + sizeof (FrameDataFragment) > frag_size)
+ frag_size = size + sizeof (FrameDataFragment);
+ new_frag = frame_data_frag_new (frag_size);
mono_compiler_barrier ();
stack->current->next = new_frag;
stack->current = new_frag;
return new_frag;
}
-static void
-free_frag (StackFragment *frag)
-{
- while (frag) {
- StackFragment *next = frag->next;
- g_free (frag);
- frag = next;
- }
-}
-
-static MONO_ALWAYS_INLINE gpointer
-frame_stack_alloc_ovf (FrameStack *stack, int size, StackFragment **out_frag)
+static gpointer
+frame_data_allocator_alloc (FrameDataAllocator *stack, InterpFrame *frame, int size)
{
- StackFragment *current = stack->current;
+ FrameDataFragment *current = stack->current;
gpointer res;
- if (current->next && current->next->pos + size <= current->next->end) {
- current = stack->current = current->next;
- current->pos = (guint8*)¤t->data;
- } else {
- StackFragment *tmp = current->next;
- /* avoid linking to be freed fragments, so the GC can't trip over it */
- current->next = NULL;
- mono_compiler_barrier ();
- free_frag (tmp);
+ int infos_len = stack->infos_len;
- current = add_frag (stack, size);
+ if (!infos_len || (infos_len > 0 && stack->infos [infos_len - 1].frame != frame)) {
+ /* First allocation by this frame. Save the markers for restore */
+ if (infos_len == stack->infos_capacity) {
+ stack->infos_capacity = infos_len * 2;
+ stack->infos = g_realloc (stack->infos, stack->infos_capacity * sizeof (FrameDataInfo));
+ }
+ stack->infos [infos_len].frame = frame;
+ stack->infos [infos_len].frag = current;
+ stack->infos [infos_len].pos = current->pos;
+ stack->infos_len++;
}
- g_assert (current->pos + size <= current->end);
- res = (gpointer)current->pos;
- current->pos += size;
-
- mono_compiler_barrier ();
-
- if (out_frag)
- *out_frag = current;
- return res;
-}
-
-static MONO_ALWAYS_INLINE gpointer
-frame_stack_alloc (FrameStack *stack, int size, StackFragment **out_frag)
-{
- StackFragment *current = stack->current;
- gpointer res;
if (G_LIKELY (current->pos + size <= current->end)) {
res = current->pos;
current->pos += size;
- mono_compiler_barrier ();
-
- if (out_frag)
- *out_frag = current;
- return res;
} else {
- return frame_stack_alloc_ovf (stack, size, out_frag);
- }
-}
+ if (current->next && current->next->pos + size <= current->next->end) {
+ current = stack->current = current->next;
+ current->pos = (guint8*)¤t->data;
+ } else {
+ FrameDataFragment *tmp = current->next;
+ /* avoid linking to be freed fragments, so the GC can't trip over it */
+ current->next = NULL;
+ mono_compiler_barrier ();
+ frame_data_frag_free (tmp);
-static MONO_ALWAYS_INLINE void
-frame_stack_pop (FrameStack *stack, StackFragment *frag, gpointer pos)
-{
- g_assert_checked ((guint8*)pos >= (guint8*)&frag->data && (guint8*)pos <= (guint8*)frag->end);
- stack->current = frag;
- mono_compiler_barrier ();
- stack->current->pos = (guint8*)pos;
+ current = frame_data_allocator_add_frag (stack, size);
+ }
+ g_assert (current->pos + size <= current->end);
+ res = (gpointer)current->pos;
+ current->pos += size;
+ }
mono_compiler_barrier ();
- //memset (stack->current->pos, 0, stack->current->end - stack->current->pos);
+ return res;
}
static void
-frame_stack_free (FrameStack *stack)
+frame_data_allocator_pop (FrameDataAllocator *stack, InterpFrame *frame)
{
- stack->inited = 0;
- mono_compiler_barrier ();
- free_frag (stack->first);
+ int infos_len = stack->infos_len;
+
+ if (infos_len > 0 && stack->infos [infos_len - 1].frame == frame) {
+ infos_len--;
+ stack->current = stack->infos [infos_len].frag;
+ stack->current->pos = stack->infos [infos_len].pos;
+ stack->infos_len = infos_len;
+ }
}
/*
}
/*
- * alloc_data_stack:
- *
- * Allocate stack space for a frame.
- */
-static MONO_ALWAYS_INLINE void
-alloc_stack_data (ThreadContext *ctx, InterpFrame *frame, int size)
-{
- StackFragment *frag;
- gpointer res = frame_stack_alloc (&ctx->data_stack, size, &frag);
-
- frame->stack = (stackval*)res;
- frame->data_frag = frag;
-}
-
-static gpointer
-alloc_extra_stack_data (ThreadContext *ctx, int size)
-{
- StackFragment *frag;
-
- return frame_stack_alloc (&ctx->data_stack, size, &frag);
-}
-
-/*
- * pop_frame:
- *
- * Pop FRAME and its child frames from the frame stack.
- * FRAME stays valid until the next alloc_frame () call.
- */
-static void
-pop_frame (ThreadContext *context, InterpFrame *frame)
-{
- if (frame->stack)
- frame_stack_pop (&context->data_stack, frame->data_frag, frame->stack);
-}
-
-/*
* List of classes whose methods will be executed by transitioning to JITted code.
* Used for testing.
*/
ThreadContext *context = (ThreadContext *) mono_native_tls_get_value (thread_context_id);
if (context == NULL) {
context = g_new0 (ThreadContext, 1);
- /*
- * Initialize data stack.
- * There are multiple advantages to this being separate from the frame stack.
- * Frame stack can be alloca.
- * Frame stack can be perfectly fit (if heap).
- * Frame stack can skip GC tracking.
- */
- frame_stack_init (&context->data_stack, 8192);
+ context->stack_start = (guchar*)mono_valloc (0, INTERP_STACK_SIZE, MONO_MMAP_READ | MONO_MMAP_WRITE, MONO_MEM_ACCOUNT_INTERP_STACK);
+ context->stack_pointer = context->stack_start;
+
+ frame_data_allocator_init (&context->data_stack, 8192);
+ /* Make sure all data is initialized before publishing the context */
+ mono_compiler_barrier ();
set_context (context);
}
return context;
{
ThreadContext *context = (ThreadContext*)ctx;
- frame_stack_free (&context->data_stack);
+ mono_vfree (context->stack_start, INTERP_STACK_SIZE, MONO_MEM_ACCOUNT_INTERP_STACK);
+ /* Prevent interp_mark_stack from trying to scan the data_stack, before freeing it */
+ context->stack_start = NULL;
+ mono_compiler_barrier ();
+ frame_data_allocator_free (&context->data_stack);
g_free (context);
}
MonoException *ex = do_transform_method (frame, context);
if (ex) {
*out_ex = ex;
+ /*
+ * Initialize the stack base pointer here, in the uncommon branch, so we don't
+ * need to check for it everytime when exitting a frame.
+ */
+ frame->stack = (stackval*)context->stack_pointer;
return slow;
}
}
- if (!clause_args || clause_args->base_frame)
- alloc_stack_data (context, frame, frame->imethod->alloca_size);
+ if (!clause_args || clause_args->base_frame) {
+ frame->stack = (stackval*)context->stack_pointer;
+ context->stack_pointer += frame->imethod->alloca_size;
+ /* Make sure the stack pointer is bumped before we store any references on the stack */
+ mono_compiler_barrier ();
+ }
return slow;
}
THROW_EX (ex, ip);
EXCEPTION_CHECKPOINT;
}
- const gboolean realloc_frame = new_method->alloca_size > frame->imethod->alloca_size;
- frame->imethod = new_method;
/*
- * We allocate the stack frame from scratch and store the arguments in the
- * locals again since it's possible for the caller stack frame to be smaller
+ * It's possible for the caller stack frame to be smaller
* than the callee stack frame (at the interp level)
*/
- if (realloc_frame) {
- alloc_stack_data (context, frame, frame->imethod->alloca_size);
- memset (frame->stack, 0, frame->imethod->alloca_size);
- locals = (guchar*)frame->stack;
- }
+ context->stack_pointer = (guchar*)frame->stack + new_method->alloca_size;
+ frame->imethod = new_method;
vt_sp = locals + frame->imethod->total_locals_size;
#if DEBUG_INTERP
vtalloc = vt_sp;
g_assert_checked (sp == (stackval*)(locals + frame->imethod->total_locals_size + frame->imethod->vt_stack_size));
goto exit_frame;
}
+ MINT_IN_CASE(MINT_RET_LOCALLOC)
+ --sp;
+ if (frame->parent) {
+ frame->parent->state.sp [0] = *sp;
+ frame->parent->state.sp++;
+ } else {
+ // FIXME This can only happen in a few wrappers. Add separate opcode for it
+ *frame->retval = *sp;
+ }
+ frame_data_allocator_pop (&context->data_stack, frame);
+ g_assert_checked (sp == (stackval*)(locals + frame->imethod->total_locals_size + frame->imethod->vt_stack_size));
+ goto exit_frame;
+ MINT_IN_CASE(MINT_RET_VOID_LOCALLOC)
+ frame_data_allocator_pop (&context->data_stack, frame);
+ g_assert_checked (sp == (stackval*)(locals + frame->imethod->total_locals_size + frame->imethod->vt_stack_size));
+ goto exit_frame;
+ MINT_IN_CASE(MINT_RET_VT_LOCALLOC) {
+ gpointer dest_vt;
+ int const i32 = READ32 (ip + 1);
+ --sp;
+ if (frame->parent) {
+ dest_vt = frame->parent->state.vt_sp;
+ /* Push the valuetype in the parent frame */
+ frame->parent->state.sp [0].data.p = dest_vt;
+ frame->parent->state.sp++;
+ frame->parent->state.vt_sp += ALIGN_TO (i32, MINT_VT_ALIGNMENT);
+ } else {
+ dest_vt = frame->retval->data.p;
+ }
+ memcpy (dest_vt, sp->data.p, i32);
+ frame_data_allocator_pop (&context->data_stack, frame);
+ g_assert_checked (sp == (stackval*)(locals + frame->imethod->total_locals_size + frame->imethod->vt_stack_size));
+ goto exit_frame;
+ }
#ifdef ENABLE_EXPERIMENT_TIERED
#define BACK_BRANCH_PROFILE(offset) do { \
ip += short_offset ? (short)*(ip + 1) : (gint32)READ32 (ip + 1);
const guint16 *endfinally_ip = ip;
GSList *old_list = finally_ips;
- MonoMethod *method = frame->imethod->method;
#if DEBUG_INTERP
if (tracing)
g_print ("* Handle finally IL_%04x\n", endfinally_ip - frame->imethod->code);
#endif
- // FIXME Null check for frame->imethod follows deref.
- if (frame->imethod == NULL || (method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL)
- || (method->iflags & (METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL | METHOD_IMPL_ATTRIBUTE_RUNTIME)))
- goto exit_frame;
-
finally_ips = g_slist_prepend (finally_ips, (void *)endfinally_ip);
for (int i = frame->imethod->num_clauses - 1; i >= 0; i--) {
mono_method_signature_internal (frame->imethod->method)->pinvoke);
if (sp > frame->stack)
g_warning_d ("retobj: more values on stack: %d", sp - frame->stack);
+ frame_data_allocator_pop (&context->data_stack, frame);
goto exit_frame;
MINT_IN_CASE(MINT_MONO_SGEN_THREAD_INFO)
sp->data.p = mono_tls_get_sgen_thread_info ();
}
ip += 4;
+ frame_data_allocator_pop (&context->data_stack, frame);
goto exit_frame;
}
MINT_IN_CASE(MINT_PROF_COVERAGE_STORE) {
goto abort_label;
int len = sp [-1].data.i;
- sp [-1].data.p = alloc_extra_stack_data (context, ALIGN_TO (len, 8));
+ // FIXME we need a separate allocator for localloc sections
+ sp [-1].data.p = frame_data_allocator_alloc (&context->data_stack, frame, ALIGN_TO (len, MINT_VT_ALIGNMENT));
if (frame->imethod->init_locals)
memset (sp [-1].data.p, 0, len);
*/
goto exit_clause;
}
+ // Because we are resuming in another frame, bypassing a normal ret opcode,
+ // we need to make sure to reset the localloc stack
+ frame_data_allocator_pop (&context->data_stack, frame);
// fall through
exit_frame:
g_assert_checked (frame->imethod);
if (frame->parent && frame->parent->state.ip) {
/* Return to the main loop after a non-recursive interpreter call */
//printf ("R: %s -> %s %p\n", mono_method_get_full_name (frame->imethod->method), mono_method_get_full_name (frame->parent->imethod->method), frame->parent->state.ip);
- InterpFrame* const child_frame = frame;
-
+ g_assert_checked (frame->stack);
+ context->stack_pointer = (guchar*)frame->stack;
frame = frame->parent;
LOAD_INTERP_STATE (frame);
- pop_frame (context, child_frame);
-
CHECK_RESUME_STATE (context);
goto main_loop;
char *locals_base = (char*)clause_args->base_frame->stack;
memcpy (locals_base, locals, frame->imethod->locals_size);
- pop_frame (context, frame);
+ context->stack_pointer = (guchar*)frame->stack;
}
} else {
- pop_frame (context, frame);
+ context->stack_pointer = (guchar*)frame->stack;
}
DEBUG_LEAVE ();
return;
ThreadContext *context = (ThreadContext*)jit_tls->interp_context;
- if (!context || !context->data_stack.inited)
+ if (!context || !context->stack_start)
return;
- StackFragment *frag;
+ // FIXME: Scan the whole area with 1 call
+ for (gpointer *p = (gpointer*)context->stack_start; p < (gpointer*)context->stack_pointer; p++)
+ func (p, gc_data);
+
+ FrameDataFragment *frag;
for (frag = context->data_stack.first; frag; frag = frag->next) {
// FIXME: Scan the whole area with 1 call
for (gpointer *p = (gpointer*)&frag->data; p < (gpointer*)frag->pos; ++p)
projects / platform / upstream / dotnet / runtime.git / commitdiff