2 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
5 * Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
7 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
8 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 * Permission is hereby granted to use or copy this program
11 * for any purpose, provided the above notices are retained on all copies.
12 * Permission to modify the code and to distribute modified code is granted,
13 * provided the above notices are retained, and a notice that the code was
14 * modified is included with the above copyright notice.
17 #include "private/pthread_support.h"
20 * Support code originally for LinuxThreads, the clone()-based kernel
21 * thread package for Linux which is included in libc6.
23 * This code no doubt makes some assumptions beyond what is
24 * guaranteed by the pthread standard, though it now does
25 * very little of that. It now also supports NPTL, and many
26 * other Posix thread implementations. We are trying to merge
27 * all flavors of pthread support code into this file.
30 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
38 # if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
39 # if !defined(GC_RTEMS_PTHREADS)
40 # include <sys/mman.h>
42 # include <sys/time.h>
43 # include <sys/types.h>
44 # include <sys/stat.h>
49 # include "gc_inline.h"
51 #if defined(GC_DARWIN_THREADS)
52 # include "private/darwin_semaphore.h"
54 # include <semaphore.h>
55 #endif /* !GC_DARWIN_THREADS */
57 #if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
58 # include <sys/sysctl.h>
59 #endif /* GC_DARWIN_THREADS */
61 #if defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
62 # include <sys/param.h>
63 # include <sys/sysctl.h>
64 #endif /* GC_NETBSD_THREADS */
66 /* Allocator lock definitions. */
67 #if !defined(USE_SPIN_LOCK)
68 GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
72 GC_INNER unsigned long GC_lock_holder = NO_THREAD;
73 /* Used only for assertions. */
76 #if defined(GC_DGUX386_THREADS)
77 # include <sys/dg_sys_info.h>
78 # include <sys/_int_psem.h>
79 /* sem_t is an uint in DG/UX */
80 typedef unsigned int sem_t;
81 #endif /* GC_DGUX386_THREADS */
83 /* Undefine macros used to redirect pthread primitives. */
84 # undef pthread_create
85 # ifndef GC_NO_PTHREAD_SIGMASK
86 # undef pthread_sigmask
88 # ifndef GC_NO_PTHREAD_CANCEL
89 # undef pthread_cancel
91 # ifdef GC_HAVE_PTHREAD_EXIT
95 # undef pthread_detach
96 # if defined(GC_OSF1_THREADS) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
97 && !defined(_PTHREAD_USE_PTDNAM_)
98 /* Restore the original mangled names on Tru64 UNIX. */
99 # define pthread_create __pthread_create
100 # define pthread_join __pthread_join
101 # define pthread_detach __pthread_detach
102 # ifndef GC_NO_PTHREAD_CANCEL
103 # define pthread_cancel __pthread_cancel
105 # ifdef GC_HAVE_PTHREAD_EXIT
106 # define pthread_exit __pthread_exit
110 #ifdef GC_USE_LD_WRAP
111 # define WRAP_FUNC(f) __wrap_##f
112 # define REAL_FUNC(f) __real_##f
113 int REAL_FUNC(pthread_create)(pthread_t *,
114 GC_PTHREAD_CREATE_CONST pthread_attr_t *,
115 void *(*start_routine)(void *), void *);
116 int REAL_FUNC(pthread_join)(pthread_t, void **);
117 int REAL_FUNC(pthread_detach)(pthread_t);
118 # ifndef GC_NO_PTHREAD_SIGMASK
119 int REAL_FUNC(pthread_sigmask)(int, const sigset_t *, sigset_t *);
121 # ifndef GC_NO_PTHREAD_CANCEL
122 int REAL_FUNC(pthread_cancel)(pthread_t);
124 # ifdef GC_HAVE_PTHREAD_EXIT
125 void REAL_FUNC(pthread_exit)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
128 # ifdef GC_USE_DLOPEN_WRAP
130 # define WRAP_FUNC(f) f
131 # define REAL_FUNC(f) GC_real_##f
132 /* We define both GC_f and plain f to be the wrapped function. */
133 /* In that way plain calls work, as do calls from files that */
134 /* included gc.h, which redefined f to GC_f. */
135 /* FIXME: Needs work for DARWIN and True64 (OSF1) */
136 typedef int (* GC_pthread_create_t)(pthread_t *,
137 GC_PTHREAD_CREATE_CONST pthread_attr_t *,
138 void * (*)(void *), void *);
139 static GC_pthread_create_t REAL_FUNC(pthread_create);
140 # ifndef GC_NO_PTHREAD_SIGMASK
141 typedef int (* GC_pthread_sigmask_t)(int, const sigset_t *,
143 static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask);
145 typedef int (* GC_pthread_join_t)(pthread_t, void **);
146 static GC_pthread_join_t REAL_FUNC(pthread_join);
147 typedef int (* GC_pthread_detach_t)(pthread_t);
148 static GC_pthread_detach_t REAL_FUNC(pthread_detach);
149 # ifndef GC_NO_PTHREAD_CANCEL
150 typedef int (* GC_pthread_cancel_t)(pthread_t);
151 static GC_pthread_cancel_t REAL_FUNC(pthread_cancel);
153 # ifdef GC_HAVE_PTHREAD_EXIT
154 typedef void (* GC_pthread_exit_t)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
155 static GC_pthread_exit_t REAL_FUNC(pthread_exit);
158 # define WRAP_FUNC(f) GC_##f
159 # if !defined(GC_DGUX386_THREADS)
160 # define REAL_FUNC(f) f
161 # else /* GC_DGUX386_THREADS */
162 # define REAL_FUNC(f) __d10_##f
163 # endif /* GC_DGUX386_THREADS */
167 #if defined(GC_USE_LD_WRAP) || defined(GC_USE_DLOPEN_WRAP)
168 /* Define GC_ functions as aliases for the plain ones, which will */
169 /* be intercepted. This allows files which include gc.h, and hence */
170 /* generate references to the GC_ symbols, to see the right symbols. */
171 GC_API int GC_pthread_create(pthread_t * t,
172 GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
173 void * (* fn)(void *), void * arg)
175 return pthread_create(t, a, fn, arg);
178 # ifndef GC_NO_PTHREAD_SIGMASK
179 GC_API int GC_pthread_sigmask(int how, const sigset_t *mask,
182 return pthread_sigmask(how, mask, old);
184 # endif /* !GC_NO_PTHREAD_SIGMASK */
186 GC_API int GC_pthread_join(pthread_t t, void **res)
188 return pthread_join(t, res);
191 GC_API int GC_pthread_detach(pthread_t t)
193 return pthread_detach(t);
196 # ifndef GC_NO_PTHREAD_CANCEL
197 GC_API int GC_pthread_cancel(pthread_t t)
199 return pthread_cancel(t);
201 # endif /* !GC_NO_PTHREAD_CANCEL */
203 # ifdef GC_HAVE_PTHREAD_EXIT
204 GC_API GC_PTHREAD_EXIT_ATTRIBUTE void GC_pthread_exit(void *retval)
206 pthread_exit(retval);
209 #endif /* Linker-based interception. */
211 #ifdef GC_USE_DLOPEN_WRAP
212 STATIC GC_bool GC_syms_initialized = FALSE;
214 STATIC void GC_init_real_syms(void)
218 if (GC_syms_initialized) return;
220 dl_handle = RTLD_NEXT;
222 dl_handle = dlopen("libpthread.so.0", RTLD_LAZY);
223 if (NULL == dl_handle) {
224 dl_handle = dlopen("libpthread.so", RTLD_LAZY); /* without ".0" */
226 if (NULL == dl_handle) ABORT("Couldn't open libpthread");
228 REAL_FUNC(pthread_create) = (GC_pthread_create_t)(word)
229 dlsym(dl_handle, "pthread_create");
231 if (REAL_FUNC(pthread_create) == 0)
232 ABORT("pthread_create not found"
233 " (probably -lgc is specified after -lpthread)");
235 # ifndef GC_NO_PTHREAD_SIGMASK
236 REAL_FUNC(pthread_sigmask) = (GC_pthread_sigmask_t)(word)
237 dlsym(dl_handle, "pthread_sigmask");
239 REAL_FUNC(pthread_join) = (GC_pthread_join_t)(word)
240 dlsym(dl_handle, "pthread_join");
241 REAL_FUNC(pthread_detach) = (GC_pthread_detach_t)(word)
242 dlsym(dl_handle, "pthread_detach");
243 # ifndef GC_NO_PTHREAD_CANCEL
244 REAL_FUNC(pthread_cancel) = (GC_pthread_cancel_t)(word)
245 dlsym(dl_handle, "pthread_cancel");
247 # ifdef GC_HAVE_PTHREAD_EXIT
248 REAL_FUNC(pthread_exit) = (GC_pthread_exit_t)(word)
249 dlsym(dl_handle, "pthread_exit");
251 GC_syms_initialized = TRUE;
254 # define INIT_REAL_SYMS() if (EXPECT(GC_syms_initialized, TRUE)) {} \
255 else GC_init_real_syms()
256 # define ASSERT_SYMS_INITIALIZED() GC_ASSERT(GC_syms_initialized)
258 # define INIT_REAL_SYMS() (void)0
259 # define ASSERT_SYMS_INITIALIZED() GC_ASSERT(parallel_initialized)
262 static GC_bool parallel_initialized = FALSE;
264 #ifndef GC_ALWAYS_MULTITHREADED
265 GC_INNER GC_bool GC_need_to_lock = FALSE;
268 STATIC int GC_nprocs = 1;
269 /* Number of processors. We may not have */
270 /* access to all of them, but this is as good */
271 /* a guess as any ... */
273 #ifdef THREAD_LOCAL_ALLOC
274 /* We must explicitly mark ptrfree and gcj free lists, since the free */
275 /* list links wouldn't otherwise be found. We also set them in the */
276 /* normal free lists, since that involves touching less memory than */
277 /* if we scanned them normally. */
278 GC_INNER void GC_mark_thread_local_free_lists(void)
283 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
284 for (p = GC_threads[i]; 0 != p; p = p -> next) {
285 if (!(p -> flags & FINISHED))
286 GC_mark_thread_local_fls_for(&(p->tlfs));
291 # if defined(GC_ASSERTIONS)
292 /* Check that all thread-local free-lists are completely marked. */
293 /* Also check that thread-specific-data structures are marked. */
294 void GC_check_tls(void)
299 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
300 for (p = GC_threads[i]; 0 != p; p = p -> next) {
301 if (!(p -> flags & FINISHED))
302 GC_check_tls_for(&(p->tlfs));
305 # if defined(USE_CUSTOM_SPECIFIC)
306 if (GC_thread_key != 0)
307 GC_check_tsd_marks(GC_thread_key);
310 # endif /* GC_ASSERTIONS */
312 #endif /* THREAD_LOCAL_ALLOC */
317 # define MAX_MARKERS 16
320 static ptr_t marker_sp[MAX_MARKERS - 1] = {0};
322 static ptr_t marker_bsp[MAX_MARKERS - 1] = {0};
325 #if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
326 static mach_port_t marker_mach_threads[MAX_MARKERS - 1] = {0};
328 /* Used only by GC_suspend_thread_list(). */
329 GC_INNER GC_bool GC_is_mach_marker(thread_act_t thread)
332 for (i = 0; i < GC_markers_m1; i++) {
333 if (marker_mach_threads[i] == thread)
338 #endif /* GC_DARWIN_THREADS */
340 #ifdef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG /* NetBSD */
341 static void set_marker_thread_name(unsigned id)
343 int err = pthread_setname_np(pthread_self(), "GC-marker-%zu",
346 WARN("pthread_setname_np failed, errno = %" WARN_PRIdPTR "\n", err);
348 #elif defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) \
349 || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
350 static void set_marker_thread_name(unsigned id)
352 char name_buf[16]; /* pthread_setname_np may fail for longer names */
353 int len = sizeof("GC-marker-") - 1;
355 /* Compose the name manually as snprintf may be unavailable or */
356 /* "%u directive output may be truncated" warning may occur. */
357 BCOPY("GC-marker-", name_buf, len);
359 name_buf[len++] = (char)('0' + (id / 10) % 10);
360 name_buf[len] = (char)('0' + id % 10);
361 name_buf[len + 1] = '\0';
363 # ifdef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID /* iOS, OS X */
364 (void)pthread_setname_np(name_buf);
365 # else /* Linux, Solaris, etc. */
366 if (pthread_setname_np(pthread_self(), name_buf) != 0)
367 WARN("pthread_setname_np failed\n", 0);
371 # define set_marker_thread_name(id) (void)(id)
374 STATIC void * GC_mark_thread(void * id)
377 IF_CANCEL(int cancel_state;)
379 if ((word)id == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
380 DISABLE_CANCEL(cancel_state);
381 /* Mark threads are not cancellable; they */
382 /* should be invisible to client. */
383 set_marker_thread_name((unsigned)(word)id);
384 marker_sp[(word)id] = GC_approx_sp();
386 marker_bsp[(word)id] = GC_save_regs_in_stack();
388 # if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
389 marker_mach_threads[(word)id] = mach_thread_self();
392 /* Inform GC_start_mark_threads about completion of marker data init. */
393 GC_acquire_mark_lock();
394 if (0 == --GC_fl_builder_count) /* count may have a negative value */
395 GC_notify_all_builder();
397 for (;; ++my_mark_no) {
398 /* GC_mark_no is passed only to allow GC_help_marker to terminate */
399 /* promptly. This is important if it were called from the signal */
400 /* handler or from the GC lock acquisition code. Under Linux, it's */
401 /* not safe to call it from a signal handler, since it uses mutexes */
402 /* and condition variables. Since it is called only here, the */
403 /* argument is unnecessary. */
404 if (my_mark_no < GC_mark_no || my_mark_no > GC_mark_no + 2) {
405 /* resynchronize if we get far off, e.g. because GC_mark_no */
407 my_mark_no = GC_mark_no;
409 # ifdef DEBUG_THREADS
410 GC_log_printf("Starting mark helper for mark number %lu\n",
411 (unsigned long)my_mark_no);
413 GC_help_marker(my_mark_no);
417 STATIC pthread_t GC_mark_threads[MAX_MARKERS];
419 #ifdef CAN_HANDLE_FORK
420 static int available_markers_m1 = 0;
421 static pthread_cond_t mark_cv;
422 /* initialized by GC_start_mark_threads_inner */
424 # define available_markers_m1 GC_markers_m1
425 static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
428 GC_INNER void GC_start_mark_threads_inner(void)
432 # ifndef NO_MARKER_SPECIAL_SIGMASK
433 sigset_t set, oldset;
436 GC_ASSERT(I_DONT_HOLD_LOCK());
437 if (available_markers_m1 <= 0) return;
438 /* Skip if parallel markers disabled or already started. */
439 # ifdef CAN_HANDLE_FORK
440 if (GC_parallel) return;
442 /* Initialize mark_cv (for the first time), or cleanup its value */
443 /* after forking in the child process. All the marker threads in */
444 /* the parent process were blocked on this variable at fork, so */
445 /* pthread_cond_wait() malfunction (hang) is possible in the */
446 /* child process without such a cleanup. */
447 /* TODO: This is not portable, it is better to shortly unblock */
448 /* all marker threads in the parent process at fork. */
450 pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
451 BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
455 GC_ASSERT(GC_fl_builder_count == 0);
456 INIT_REAL_SYMS(); /* for pthread_create */
457 if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
458 if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
459 ABORT("pthread_attr_setdetachstate failed");
461 # ifdef DEFAULT_STACK_MAYBE_SMALL
462 /* Default stack size is usually too small: increase it. */
463 /* Otherwise marker threads or GC may run out of space. */
467 if (pthread_attr_getstacksize(&attr, &old_size) != 0)
468 ABORT("pthread_attr_getstacksize failed");
469 if (old_size < MIN_STACK_SIZE
470 && old_size != 0 /* stack size is known */) {
471 if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
472 ABORT("pthread_attr_setstacksize failed");
475 # endif /* DEFAULT_STACK_MAYBE_SMALL */
477 # ifndef NO_MARKER_SPECIAL_SIGMASK
478 /* Apply special signal mask to GC marker threads, and don't drop */
479 /* user defined signals by GC marker threads. */
480 if (sigfillset(&set) != 0)
481 ABORT("sigfillset failed");
483 # if !defined(GC_DARWIN_THREADS) && !defined(GC_OPENBSD_UTHREADS) \
485 /* These are used by GC to stop and restart the world. */
486 if (sigdelset(&set, GC_get_suspend_signal()) != 0
487 || sigdelset(&set, GC_get_thr_restart_signal()) != 0)
488 ABORT("sigdelset failed");
491 if (REAL_FUNC(pthread_sigmask)(SIG_BLOCK, &set, &oldset) < 0) {
492 WARN("pthread_sigmask set failed, no markers started,"
493 " errno = %" WARN_PRIdPTR "\n", errno);
495 (void)pthread_attr_destroy(&attr);
498 # endif /* !NO_MARKER_SPECIAL_SIGMASK */
500 # ifdef CAN_HANDLE_FORK
501 /* To have proper GC_parallel value in GC_help_marker. */
502 GC_markers_m1 = available_markers_m1;
504 for (i = 0; i < available_markers_m1; ++i) {
505 if (0 != REAL_FUNC(pthread_create)(GC_mark_threads + i, &attr,
506 GC_mark_thread, (void *)(word)i)) {
507 WARN("Marker thread creation failed, errno = %" WARN_PRIdPTR "\n",
509 /* Don't try to create other marker threads. */
515 # ifndef NO_MARKER_SPECIAL_SIGMASK
516 /* Restore previous signal mask. */
517 if (REAL_FUNC(pthread_sigmask)(SIG_SETMASK, &oldset, NULL) < 0) {
518 WARN("pthread_sigmask restore failed, errno = %" WARN_PRIdPTR "\n",
523 (void)pthread_attr_destroy(&attr);
524 GC_wait_for_markers_init();
525 GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
528 #endif /* PARALLEL_MARK */
530 GC_INNER GC_bool GC_thr_initialized = FALSE;
532 GC_INNER volatile GC_thread GC_threads[THREAD_TABLE_SZ] = {0};
534 void GC_push_thread_structures(void)
536 GC_ASSERT(I_HOLD_LOCK());
537 GC_PUSH_ALL_SYM(GC_threads);
538 # if defined(THREAD_LOCAL_ALLOC)
539 GC_PUSH_ALL_SYM(GC_thread_key);
544 STATIC int GC_count_threads(void)
548 GC_ASSERT(I_HOLD_LOCK());
549 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
550 GC_thread th = GC_threads[i];
552 if (!(th->flags & FINISHED))
559 #endif /* DEBUG_THREADS */
561 /* It may not be safe to allocate when we register the first thread. */
562 /* As "next" and "status" fields are unused, no need to push this. */
563 static struct GC_Thread_Rep first_thread;
565 /* Add a thread to GC_threads. We assume it wasn't already there. */
566 /* Caller holds allocation lock. */
567 STATIC GC_thread GC_new_thread(pthread_t id)
569 int hv = THREAD_TABLE_INDEX(id);
571 static GC_bool first_thread_used = FALSE;
573 # ifdef DEBUG_THREADS
574 GC_log_printf("Creating thread %p\n", (void *)id);
575 for (result = GC_threads[hv]; result != NULL; result = result->next)
576 if (!THREAD_EQUAL(result->id, id)) {
577 GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
581 GC_ASSERT(I_HOLD_LOCK());
582 if (!EXPECT(first_thread_used, TRUE)) {
583 result = &first_thread;
584 first_thread_used = TRUE;
585 GC_ASSERT(NULL == GC_threads[hv]);
586 # if defined(THREAD_SANITIZER) && defined(CPPCHECK)
587 GC_noop1(result->dummy[0]);
590 result = (struct GC_Thread_Rep *)
591 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
592 if (result == 0) return(0);
595 # ifdef USE_TKILL_ON_ANDROID
596 result -> kernel_id = gettid();
598 result -> next = GC_threads[hv];
599 GC_threads[hv] = result;
601 GC_nacl_gc_thread_self = result;
602 GC_nacl_initialize_gc_thread();
604 GC_ASSERT(result -> flags == 0 && result -> thread_blocked == 0);
605 if (EXPECT(result != &first_thread, TRUE))
610 /* Delete a thread from GC_threads. We assume it is there. */
611 /* (The code intentionally traps if it wasn't.) */
612 /* It is safe to delete the main thread. */
613 STATIC void GC_delete_thread(pthread_t id)
615 int hv = THREAD_TABLE_INDEX(id);
616 GC_thread p = GC_threads[hv];
617 GC_thread prev = NULL;
619 # ifdef DEBUG_THREADS
620 GC_log_printf("Deleting thread %p, n_threads = %d\n",
621 (void *)id, GC_count_threads());
625 GC_nacl_shutdown_gc_thread();
626 GC_nacl_gc_thread_self = NULL;
629 GC_ASSERT(I_HOLD_LOCK());
630 while (!THREAD_EQUAL(p -> id, id)) {
635 GC_threads[hv] = p -> next;
637 GC_ASSERT(prev != &first_thread);
638 prev -> next = p -> next;
641 if (p != &first_thread) {
642 # ifdef GC_DARWIN_THREADS
643 mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
649 /* If a thread has been joined, but we have not yet */
650 /* been notified, then there may be more than one thread */
651 /* in the table with the same pthread id. */
652 /* This is OK, but we need a way to delete a specific one. */
653 STATIC void GC_delete_gc_thread(GC_thread t)
655 pthread_t id = t -> id;
656 int hv = THREAD_TABLE_INDEX(id);
657 GC_thread p = GC_threads[hv];
658 GC_thread prev = NULL;
660 GC_ASSERT(I_HOLD_LOCK());
666 GC_threads[hv] = p -> next;
668 GC_ASSERT(prev != &first_thread);
669 prev -> next = p -> next;
672 # ifdef GC_DARWIN_THREADS
673 mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
677 # ifdef DEBUG_THREADS
678 GC_log_printf("Deleted thread %p, n_threads = %d\n",
679 (void *)id, GC_count_threads());
683 /* Return a GC_thread corresponding to a given pthread_t. */
684 /* Returns 0 if it's not there. */
685 /* Caller holds allocation lock or otherwise inhibits */
687 /* If there is more than one thread with the given id we */
688 /* return the most recent one. */
689 GC_INNER GC_thread GC_lookup_thread(pthread_t id)
691 GC_thread p = GC_threads[THREAD_TABLE_INDEX(id)];
693 while (p != 0 && !THREAD_EQUAL(p -> id, id)) p = p -> next;
697 /* Called by GC_finalize() (in case of an allocation failure observed). */
698 GC_INNER void GC_reset_finalizer_nested(void)
700 GC_thread me = GC_lookup_thread(pthread_self());
701 me->finalizer_nested = 0;
704 /* Checks and updates the thread-local level of finalizers recursion. */
705 /* Returns NULL if GC_invoke_finalizers() should not be called by the */
706 /* collector (to minimize the risk of a deep finalizers recursion), */
707 /* otherwise returns a pointer to the thread-local finalizer_nested. */
708 /* Called by GC_notify_or_invoke_finalizers() only (the lock is held). */
709 GC_INNER unsigned char *GC_check_finalizer_nested(void)
711 GC_thread me = GC_lookup_thread(pthread_self());
712 unsigned nesting_level = me->finalizer_nested;
714 /* We are inside another GC_invoke_finalizers(). */
715 /* Skip some implicitly-called GC_invoke_finalizers() */
716 /* depending on the nesting (recursion) level. */
717 if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
718 me->finalizer_skipped = 0;
720 me->finalizer_nested = (unsigned char)(nesting_level + 1);
721 return &me->finalizer_nested;
724 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
725 /* This is called from thread-local GC_malloc(). */
726 GC_bool GC_is_thread_tsd_valid(void *tsd)
732 me = GC_lookup_thread(pthread_self());
734 return (word)tsd >= (word)(&me->tlfs)
735 && (word)tsd < (word)(&me->tlfs) + sizeof(me->tlfs);
737 #endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
739 GC_API int GC_CALL GC_thread_is_registered(void)
741 pthread_t self = pthread_self();
746 me = GC_lookup_thread(self);
751 static pthread_t main_pthread_id;
752 static void *main_stack, *main_altstack;
753 static word main_stack_size, main_altstack_size;
755 GC_API void GC_CALL GC_register_altstack(void *stack, GC_word stack_size,
757 GC_word altstack_size)
760 pthread_t self = pthread_self();
764 me = GC_lookup_thread(self);
766 me->stack = (ptr_t)stack;
767 me->stack_size = stack_size;
768 me->altstack = (ptr_t)altstack;
769 me->altstack_size = altstack_size;
771 /* This happens if we are called before GC_thr_init. */
772 main_pthread_id = self;
774 main_stack_size = stack_size;
775 main_altstack = altstack;
776 main_altstack_size = altstack_size;
781 #ifdef CAN_HANDLE_FORK
783 /* Prevent TSan false positive about the race during items removal */
784 /* from GC_threads. (The race cannot happen since only one thread */
785 /* survives in the child.) */
786 # ifdef CAN_CALL_ATFORK
787 GC_ATTR_NO_SANITIZE_THREAD
789 static void store_to_threads_table(int hv, GC_thread me)
794 /* Remove all entries from the GC_threads table, except the */
795 /* one for the current thread. We need to do this in the child */
796 /* process after a fork(), since only the current thread */
797 /* survives in the child. */
798 STATIC void GC_remove_all_threads_but_me(void)
800 pthread_t self = pthread_self();
803 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
807 for (p = GC_threads[hv]; 0 != p; p = next) {
809 if (THREAD_EQUAL(p -> id, self)
810 && me == NULL) { /* ignore dead threads with the same id */
813 # ifdef GC_DARWIN_THREADS
814 /* Update thread Id after fork (it is OK to call */
815 /* GC_destroy_thread_local and GC_free_internal */
816 /* before update). */
817 me -> stop_info.mach_thread = mach_thread_self();
819 # ifdef USE_TKILL_ON_ANDROID
820 me -> kernel_id = gettid();
822 # if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
826 /* Some TLS implementations might be not fork-friendly, so */
827 /* we re-assign thread-local pointer to 'tlfs' for safety */
828 /* instead of the assertion check (again, it is OK to call */
829 /* GC_destroy_thread_local and GC_free_internal before). */
830 res = GC_setspecific(GC_thread_key, &me->tlfs);
831 if (COVERT_DATAFLOW(res) != 0)
832 ABORT("GC_setspecific failed (in child)");
836 # ifdef THREAD_LOCAL_ALLOC
837 if (!(p -> flags & FINISHED)) {
838 /* Cannot call GC_destroy_thread_local here. The free */
839 /* lists may be in an inconsistent state (as thread p may */
840 /* be updating one of the lists by GC_generic_malloc_many */
841 /* or GC_FAST_MALLOC_GRANS when fork is invoked). */
842 /* This should not be a problem because the lost elements */
843 /* of the free lists will be collected during GC. */
844 GC_remove_specific_after_fork(GC_thread_key, p -> id);
847 /* TODO: To avoid TSan hang (when updating GC_bytes_freed), */
848 /* we just skip explicit freeing of GC_threads entries. */
849 # if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
850 if (p != &first_thread) GC_INTERNAL_FREE(p);
854 store_to_threads_table(hv, me);
857 #endif /* CAN_HANDLE_FORK */
859 #ifdef USE_PROC_FOR_LIBRARIES
860 GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
865 GC_ASSERT(I_HOLD_LOCK());
866 # ifdef PARALLEL_MARK
867 for (i = 0; i < GC_markers_m1; ++i) {
868 if ((word)marker_sp[i] > (word)lo && (word)marker_sp[i] < (word)hi)
871 if ((word)marker_bsp[i] > (word)lo
872 && (word)marker_bsp[i] < (word)hi)
877 for (i = 0; i < THREAD_TABLE_SZ; i++) {
878 for (p = GC_threads[i]; p != 0; p = p -> next) {
879 if (0 != p -> stack_end) {
880 # ifdef STACK_GROWS_UP
881 if ((word)p->stack_end >= (word)lo
882 && (word)p->stack_end < (word)hi)
884 # else /* STACK_GROWS_DOWN */
885 if ((word)p->stack_end > (word)lo
886 && (word)p->stack_end <= (word)hi)
894 #endif /* USE_PROC_FOR_LIBRARIES */
897 /* Find the largest stack_base smaller than bound. May be used */
898 /* to find the boundary between a register stack and adjacent */
899 /* immediately preceding memory stack. */
900 GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound)
906 GC_ASSERT(I_HOLD_LOCK());
907 # ifdef PARALLEL_MARK
908 for (i = 0; i < GC_markers_m1; ++i) {
909 if ((word)marker_sp[i] > (word)result
910 && (word)marker_sp[i] < (word)bound)
911 result = marker_sp[i];
914 for (i = 0; i < THREAD_TABLE_SZ; i++) {
915 for (p = GC_threads[i]; p != 0; p = p -> next) {
916 if ((word)p->stack_end > (word)result
917 && (word)p->stack_end < (word)bound) {
918 result = p -> stack_end;
927 /* Also defined in os_dep.c. */
928 # define STAT_BUF_SIZE 4096
929 # define STAT_READ read
930 /* If read is wrapped, this may need to be redefined to call */
934 #ifdef GC_HPUX_THREADS
935 # define GC_get_nprocs() pthread_num_processors_np()
937 #elif defined(GC_OSF1_THREADS) || defined(GC_AIX_THREADS) \
938 || defined(GC_HAIKU_THREADS) || defined(GC_SOLARIS_THREADS) \
939 || defined(HURD) || defined(HOST_ANDROID) || defined(NACL)
940 GC_INLINE int GC_get_nprocs(void)
942 int nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
943 return nprocs > 0 ? nprocs : 1; /* ignore error silently */
946 #elif defined(GC_IRIX_THREADS)
947 GC_INLINE int GC_get_nprocs(void)
949 int nprocs = (int)sysconf(_SC_NPROC_ONLN);
950 return nprocs > 0 ? nprocs : 1; /* ignore error silently */
953 #elif defined(GC_LINUX_THREADS) /* && !HOST_ANDROID && !NACL */
954 /* Return the number of processors. */
955 STATIC int GC_get_nprocs(void)
957 /* Should be "return sysconf(_SC_NPROCESSORS_ONLN);" but that */
958 /* appears to be buggy in many cases. */
959 /* We look for lines "cpu<n>" in /proc/stat. */
960 char stat_buf[STAT_BUF_SIZE];
964 f = open("/proc/stat", O_RDONLY);
966 WARN("Couldn't read /proc/stat\n", 0);
967 return 1; /* assume an uniprocessor */
969 len = STAT_READ(f, stat_buf, STAT_BUF_SIZE);
973 /* Some old kernels only have a single "cpu nnnn ..." */
974 /* entry in /proc/stat. We identify those as */
977 for (i = 0; i < len - 100; ++i) {
978 if (stat_buf[i] == '\n' && stat_buf[i+1] == 'c'
979 && stat_buf[i+2] == 'p' && stat_buf[i+3] == 'u') {
980 int cpu_no = atoi(&stat_buf[i + 4]);
981 if (cpu_no >= result)
988 #elif defined(GC_DGUX386_THREADS)
989 /* Return the number of processors, or i <= 0 if it can't be determined. */
990 STATIC int GC_get_nprocs(void)
993 struct dg_sys_info_pm_info pm_sysinfo;
996 status = dg_sys_info((long int *) &pm_sysinfo,
997 DG_SYS_INFO_PM_INFO_TYPE, DG_SYS_INFO_PM_CURRENT_VERSION);
999 /* set -1 for error */
1003 numCpus = pm_sysinfo.idle_vp_count;
1007 #elif defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS) \
1008 || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
1009 STATIC int GC_get_nprocs(void)
1011 int mib[] = {CTL_HW,HW_NCPU};
1013 size_t len = sizeof(res);
1015 sysctl(mib, sizeof(mib)/sizeof(int), &res, &len, NULL, 0);
1020 /* E.g., GC_RTEMS_PTHREADS */
1021 # define GC_get_nprocs() 1 /* not implemented */
1022 #endif /* !GC_LINUX_THREADS && !GC_DARWIN_THREADS && ... */
1024 #if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
1025 /* Some buggy Linux/arm kernels show only non-sleeping CPUs in */
1026 /* /proc/stat (and /proc/cpuinfo), so another data system source is */
1027 /* tried first. Result <= 0 on error. */
1028 STATIC int GC_get_nprocs_present(void)
1034 f = open("/sys/devices/system/cpu/present", O_RDONLY);
1036 return -1; /* cannot open the file */
1038 len = STAT_READ(f, stat_buf, sizeof(stat_buf));
1041 /* Recognized file format: "0\n" or "0-<max_cpu_id>\n" */
1042 /* The file might probably contain a comma-separated list */
1043 /* but we do not need to handle it (just silently ignore). */
1044 if (len < 2 || stat_buf[0] != '0' || stat_buf[len - 1] != '\n') {
1045 return 0; /* read error or unrecognized content */
1046 } else if (len == 2) {
1047 return 1; /* an uniprocessor */
1048 } else if (stat_buf[1] != '-') {
1049 return 0; /* unrecognized content */
1052 stat_buf[len - 1] = '\0'; /* terminate the string */
1053 return atoi(&stat_buf[2]) + 1; /* skip "0-" and parse max_cpu_num */
1055 #endif /* ARM32 && GC_LINUX_THREADS && !NACL */
1057 /* We hold the GC lock. Wait until an in-progress GC has finished. */
1058 /* Repeatedly RELEASES GC LOCK in order to wait. */
1059 /* If wait_for_all is true, then we exit with the GC lock held and no */
1060 /* collection in progress; otherwise we just wait for the current GC */
1062 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
1065 # if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
1066 /* GC_lock_holder is accessed with the lock held, so there is no */
1067 /* data race actually (unlike what is reported by TSan). */
1068 GC_ASSERT(I_HOLD_LOCK());
1070 ASSERT_CANCEL_DISABLED();
1071 if (GC_incremental && GC_collection_in_progress()) {
1072 word old_gc_no = GC_gc_no;
1074 /* Make sure that no part of our stack is still on the mark stack, */
1075 /* since it's about to be unmapped. */
1076 while (GC_incremental && GC_collection_in_progress()
1077 && (wait_for_all || old_gc_no == GC_gc_no)) {
1079 GC_in_thread_creation = TRUE;
1080 GC_collect_a_little_inner(1);
1081 GC_in_thread_creation = FALSE;
1090 #ifdef CAN_HANDLE_FORK
1091 /* Procedures called before and after a fork. The goal here is to make */
1092 /* it safe to call GC_malloc() in a forked child. It's unclear that is */
1093 /* attainable, since the single UNIX spec seems to imply that one */
1094 /* should only call async-signal-safe functions, and we probably can't */
1095 /* quite guarantee that. But we give it our best shot. (That same */
1096 /* spec also implies that it's not safe to call the system malloc */
1097 /* between fork() and exec(). Thus we're doing no worse than it.) */
1099 IF_CANCEL(static int fork_cancel_state;)
1100 /* protected by allocation lock. */
1102 /* Called before a fork() */
1103 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1104 /* GC_lock_holder is updated safely (no data race actually). */
1105 GC_ATTR_NO_SANITIZE_THREAD
1107 static void fork_prepare_proc(void)
1109 /* Acquire all relevant locks, so that after releasing the locks */
1110 /* the child will see a consistent state in which monitor */
1111 /* invariants hold. Unfortunately, we can't acquire libc locks */
1112 /* we might need, and there seems to be no guarantee that libc */
1113 /* must install a suitable fork handler. */
1114 /* Wait for an ongoing GC to finish, since we can't finish it in */
1115 /* the (one remaining thread in) the child. */
1117 DISABLE_CANCEL(fork_cancel_state);
1118 /* Following waits may include cancellation points. */
1119 # if defined(PARALLEL_MARK)
1121 GC_wait_for_reclaim();
1123 GC_wait_for_gc_completion(TRUE);
1124 # if defined(PARALLEL_MARK)
1126 GC_acquire_mark_lock();
1128 GC_acquire_dirty_lock();
1131 /* Called in parent after a fork() (even if the latter failed). */
1132 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1133 GC_ATTR_NO_SANITIZE_THREAD
1135 static void fork_parent_proc(void)
1137 GC_release_dirty_lock();
1138 # if defined(PARALLEL_MARK)
1140 GC_release_mark_lock();
1142 RESTORE_CANCEL(fork_cancel_state);
1146 /* Called in child after a fork() */
1147 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1148 GC_ATTR_NO_SANITIZE_THREAD
1150 static void fork_child_proc(void)
1152 GC_release_dirty_lock();
1153 # if defined(PARALLEL_MARK)
1155 GC_release_mark_lock();
1157 /* Clean up the thread table, so that just our thread is left. */
1158 GC_remove_all_threads_but_me();
1159 # ifdef PARALLEL_MARK
1160 /* Turn off parallel marking in the child, since we are probably */
1161 /* just going to exec, and we would have to restart mark threads. */
1162 GC_parallel = FALSE;
1163 # endif /* PARALLEL_MARK */
1164 RESTORE_CANCEL(fork_cancel_state);
1166 /* Even though after a fork the child only inherits the single */
1167 /* thread that called the fork(), if another thread in the parent */
1168 /* was attempting to lock the mutex while being held in */
1169 /* fork_child_prepare(), the mutex will be left in an inconsistent */
1170 /* state in the child after the UNLOCK. This is the case, at */
1171 /* least, in Mac OS X and leads to an unusable GC in the child */
1172 /* which will block when attempting to perform any GC operation */
1173 /* that acquires the allocation mutex. */
1174 # ifdef USE_PTHREAD_LOCKS
1175 GC_ASSERT(I_DONT_HOLD_LOCK());
1176 /* Reinitialize the mutex. It should be safe since we are */
1177 /* running this in the child which only inherits a single thread. */
1178 /* mutex_destroy() may return EBUSY, which makes no sense, but */
1179 /* that is the reason for the need of the reinitialization. */
1180 (void)pthread_mutex_destroy(&GC_allocate_ml);
1181 /* TODO: Probably some targets might need the default mutex */
1182 /* attribute to be passed instead of NULL. */
1183 if (0 != pthread_mutex_init(&GC_allocate_ml, NULL))
1184 ABORT("pthread_mutex_init failed (in child)");
1188 /* Routines for fork handling by client (no-op if pthread_atfork works). */
1189 GC_API void GC_CALL GC_atfork_prepare(void)
1191 if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
1192 # if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB)
1193 if (GC_auto_incremental) {
1194 GC_ASSERT(0 == GC_handle_fork);
1195 ABORT("Unable to fork while mprotect_thread is running");
1198 if (GC_handle_fork <= 0)
1199 fork_prepare_proc();
1202 GC_API void GC_CALL GC_atfork_parent(void)
1204 if (GC_handle_fork <= 0)
1208 GC_API void GC_CALL GC_atfork_child(void)
1210 if (GC_handle_fork <= 0)
1213 #endif /* CAN_HANDLE_FORK */
1215 #ifdef INCLUDE_LINUX_THREAD_DESCR
1216 __thread int GC_dummy_thread_local;
1219 #ifdef PARALLEL_MARK
1220 static void setup_mark_lock(void);
1223 GC_INNER void GC_thr_init(void)
1225 GC_ASSERT(I_HOLD_LOCK());
1226 if (GC_thr_initialized) return;
1227 GC_thr_initialized = TRUE;
1229 GC_ASSERT((word)&GC_threads % sizeof(word) == 0);
1230 # ifdef CAN_HANDLE_FORK
1231 /* Prepare for forks if requested. */
1232 if (GC_handle_fork) {
1233 # ifdef CAN_CALL_ATFORK
1234 if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
1235 fork_child_proc) == 0) {
1236 /* Handlers successfully registered. */
1240 /* else */ if (GC_handle_fork != -1)
1241 ABORT("pthread_atfork failed");
1244 # ifdef INCLUDE_LINUX_THREAD_DESCR
1245 /* Explicitly register the region including the address */
1246 /* of a thread local variable. This should include thread */
1247 /* locals for the main thread, except for those allocated */
1248 /* in response to dlopen calls. */
1250 ptr_t thread_local_addr = (ptr_t)(&GC_dummy_thread_local);
1251 ptr_t main_thread_start, main_thread_end;
1252 if (!GC_enclosing_mapping(thread_local_addr, &main_thread_start,
1253 &main_thread_end)) {
1254 ABORT("Failed to find mapping for main thread thread locals");
1256 /* main_thread_start and main_thread_end are initialized. */
1257 GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
1261 /* Add the initial thread, so we can stop it. */
1263 pthread_t self = pthread_self();
1264 GC_thread t = GC_new_thread(self);
1267 ABORT("Failed to allocate memory for the initial thread");
1268 # ifdef GC_DARWIN_THREADS
1269 t -> stop_info.mach_thread = mach_thread_self();
1271 t -> stop_info.stack_ptr = GC_approx_sp();
1273 t -> flags = DETACHED | MAIN_THREAD;
1274 if (THREAD_EQUAL(self, main_pthread_id)) {
1275 t -> stack = (ptr_t)main_stack;
1276 t -> stack_size = main_stack_size;
1277 t -> altstack = (ptr_t)main_altstack;
1278 t -> altstack_size = main_altstack_size;
1282 # ifndef GC_DARWIN_THREADS
1286 /* Set GC_nprocs. */
1288 char * nprocs_string = GETENV("GC_NPROCS");
1290 if (nprocs_string != NULL) GC_nprocs = atoi(nprocs_string);
1293 # if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
1294 && (GC_nprocs = GC_get_nprocs_present()) <= 1
1295 /* Workaround for some Linux/arm kernels */
1299 GC_nprocs = GC_get_nprocs();
1301 if (GC_nprocs <= 0) {
1302 WARN("GC_get_nprocs() returned %" WARN_PRIdPTR "\n", GC_nprocs);
1303 GC_nprocs = 2; /* assume dual-core */
1304 # ifdef PARALLEL_MARK
1305 available_markers_m1 = 0; /* but use only one marker */
1308 # ifdef PARALLEL_MARK
1310 char * markers_string = GETENV("GC_MARKERS");
1313 if (markers_string != NULL) {
1314 markers = atoi(markers_string);
1315 if (markers <= 0 || markers > MAX_MARKERS) {
1316 WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
1317 "; using maximum threads\n", (signed_word)markers);
1318 markers = MAX_MARKERS;
1321 markers = GC_nprocs;
1322 # if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
1323 /* This is primarily for targets without getenv(). */
1324 if (markers < GC_MIN_MARKERS)
1325 markers = GC_MIN_MARKERS;
1327 if (markers > MAX_MARKERS)
1328 markers = MAX_MARKERS; /* silently limit the value */
1330 available_markers_m1 = markers - 1;
1334 GC_COND_LOG_PRINTF("Number of processors = %d\n", GC_nprocs);
1335 # ifdef PARALLEL_MARK
1336 if (available_markers_m1 <= 0) {
1337 /* Disable parallel marking. */
1338 GC_parallel = FALSE;
1340 "Single marker thread, turning off parallel marking\n");
1347 /* Perform all initializations, including those that */
1348 /* may require allocation. */
1349 /* Called without allocation lock. */
1350 /* Must be called before a second thread is created. */
1351 /* Did we say it's called without the allocation lock? */
1352 GC_INNER void GC_init_parallel(void)
1354 # if defined(THREAD_LOCAL_ALLOC)
1357 if (parallel_initialized) return;
1358 parallel_initialized = TRUE;
1360 /* GC_init() calls us back, so set flag first. */
1361 if (!GC_is_initialized) GC_init();
1362 /* Initialize thread local free lists if used. */
1363 # if defined(THREAD_LOCAL_ALLOC)
1365 GC_init_thread_local(&(GC_lookup_thread(pthread_self())->tlfs));
1370 #ifndef GC_NO_PTHREAD_SIGMASK
1371 GC_API int WRAP_FUNC(pthread_sigmask)(int how, const sigset_t *set,
1374 sigset_t fudged_set;
1377 if (set != NULL && (how == SIG_BLOCK || how == SIG_SETMASK)) {
1378 int sig_suspend = GC_get_suspend_signal();
1381 GC_ASSERT(sig_suspend >= 0);
1382 if (sigdelset(&fudged_set, sig_suspend) != 0)
1383 ABORT("sigdelset failed");
1386 return(REAL_FUNC(pthread_sigmask)(how, set, oset));
1388 #endif /* !GC_NO_PTHREAD_SIGMASK */
1390 /* Wrapper for functions that are likely to block for an appreciable */
1391 /* length of time. */
1393 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
1395 struct blocking_data * d = (struct blocking_data *) data;
1396 pthread_t self = pthread_self();
1398 # if defined(SPARC) || defined(IA64)
1399 ptr_t stack_ptr = GC_save_regs_in_stack();
1401 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1402 GC_bool topOfStackUnset = FALSE;
1407 me = GC_lookup_thread(self);
1408 GC_ASSERT(!(me -> thread_blocked));
1410 me -> stop_info.stack_ptr = stack_ptr;
1412 me -> stop_info.stack_ptr = GC_approx_sp();
1414 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1415 if (me -> topOfStack == NULL) {
1416 /* GC_do_blocking_inner is not called recursively, */
1417 /* so topOfStack should be computed now. */
1418 topOfStackUnset = TRUE;
1419 me -> topOfStack = GC_FindTopOfStack(0);
1423 me -> backing_store_ptr = stack_ptr;
1425 me -> thread_blocked = (unsigned char)TRUE;
1426 /* Save context here if we want to support precise stack marking */
1428 d -> client_data = (d -> fn)(d -> client_data);
1429 LOCK(); /* This will block if the world is stopped. */
1430 # if defined(CPPCHECK)
1431 GC_noop1((word)&me->thread_blocked);
1433 me -> thread_blocked = FALSE;
1434 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1435 if (topOfStackUnset)
1436 me -> topOfStack = NULL; /* make topOfStack unset again */
1441 GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
1442 const struct GC_stack_base *sb)
1444 GC_thread t = (GC_thread)gc_thread_handle;
1446 GC_ASSERT(sb -> mem_base != NULL);
1447 if (!EXPECT(GC_is_initialized, TRUE)) {
1448 GC_ASSERT(NULL == t);
1450 GC_ASSERT(I_HOLD_LOCK());
1451 if (NULL == t) /* current thread? */
1452 t = GC_lookup_thread(pthread_self());
1453 GC_ASSERT((t -> flags & FINISHED) == 0);
1454 GC_ASSERT(!(t -> thread_blocked)
1455 && NULL == t -> traced_stack_sect); /* for now */
1457 if ((t -> flags & MAIN_THREAD) == 0) {
1458 t -> stack_end = (ptr_t)sb->mem_base;
1460 t -> backing_store_end = (ptr_t)sb->reg_base;
1464 /* Otherwise alter the stack bottom of the primordial thread. */
1467 GC_stackbottom = (char*)sb->mem_base;
1469 GC_register_stackbottom = (ptr_t)sb->reg_base;
1473 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
1475 pthread_t self = pthread_self();
1480 me = GC_lookup_thread(self);
1481 /* The thread is assumed to be registered. */
1482 if ((me -> flags & MAIN_THREAD) == 0) {
1483 sb -> mem_base = me -> stack_end;
1485 sb -> reg_base = me -> backing_store_end;
1488 sb -> mem_base = GC_stackbottom;
1490 sb -> reg_base = GC_register_stackbottom;
1494 return (void *)me; /* gc_thread_handle */
1497 /* GC_call_with_gc_active() has the opposite to GC_do_blocking() */
1498 /* functionality. It might be called from a user function invoked by */
1499 /* GC_do_blocking() to temporarily back allow calling any GC function */
1500 /* and/or manipulating pointers to the garbage collected heap. */
1501 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
1504 struct GC_traced_stack_sect_s stacksect;
1505 pthread_t self = pthread_self();
1509 LOCK(); /* This will block if the world is stopped. */
1510 me = GC_lookup_thread(self);
1512 /* Adjust our stack bottom value (this could happen unless */
1513 /* GC_get_stack_base() was used which returned GC_SUCCESS). */
1514 if ((me -> flags & MAIN_THREAD) == 0) {
1515 GC_ASSERT(me -> stack_end != NULL);
1516 if ((word)me->stack_end HOTTER_THAN (word)(&stacksect))
1517 me -> stack_end = (ptr_t)(&stacksect);
1519 /* The original stack. */
1520 if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect))
1521 GC_stackbottom = (ptr_t)COVERT_DATAFLOW(&stacksect);
1524 if (!me->thread_blocked) {
1525 /* We are not inside GC_do_blocking() - do nothing more. */
1527 client_data = fn(client_data);
1528 /* Prevent treating the above as a tail call. */
1529 GC_noop1(COVERT_DATAFLOW(&stacksect));
1530 return client_data; /* result */
1533 /* Setup new "stack section". */
1534 stacksect.saved_stack_ptr = me -> stop_info.stack_ptr;
1536 /* This is the same as in GC_call_with_stack_base(). */
1537 stacksect.backing_store_end = GC_save_regs_in_stack();
1538 /* Unnecessarily flushes register stack, */
1539 /* but that probably doesn't hurt. */
1540 stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
1542 stacksect.prev = me -> traced_stack_sect;
1543 me -> thread_blocked = FALSE;
1544 me -> traced_stack_sect = &stacksect;
1547 client_data = fn(client_data);
1548 GC_ASSERT(me -> thread_blocked == FALSE);
1549 GC_ASSERT(me -> traced_stack_sect == &stacksect);
1551 /* Restore original "stack section". */
1552 # if defined(CPPCHECK)
1553 GC_noop1((word)me->traced_stack_sect);
1556 me -> traced_stack_sect = stacksect.prev;
1558 me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
1560 me -> thread_blocked = (unsigned char)TRUE;
1561 me -> stop_info.stack_ptr = stacksect.saved_stack_ptr;
1564 return client_data; /* result */
1567 STATIC void GC_unregister_my_thread_inner(GC_thread me)
1569 # ifdef DEBUG_THREADS
1571 "Unregistering thread %p, gc_thread = %p, n_threads = %d\n",
1572 (void *)me->id, (void *)me, GC_count_threads());
1574 GC_ASSERT(!(me -> flags & FINISHED));
1575 # if defined(THREAD_LOCAL_ALLOC)
1576 GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
1577 GC_destroy_thread_local(&(me->tlfs));
1579 # if defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)
1580 /* Handle DISABLED_GC flag which is set by the */
1581 /* intercepted pthread_cancel or pthread_exit. */
1582 if ((me -> flags & DISABLED_GC) != 0) {
1586 if (me -> flags & DETACHED) {
1587 GC_delete_thread(pthread_self());
1589 me -> flags |= FINISHED;
1591 # if defined(THREAD_LOCAL_ALLOC)
1592 /* It is required to call remove_specific defined in specific.c. */
1593 GC_remove_specific(GC_thread_key);
1597 GC_API int GC_CALL GC_unregister_my_thread(void)
1599 pthread_t self = pthread_self();
1601 IF_CANCEL(int cancel_state;)
1605 DISABLE_CANCEL(cancel_state);
1606 /* Wait for any GC that may be marking from our stack to */
1607 /* complete before we remove this thread. */
1608 GC_wait_for_gc_completion(FALSE);
1609 me = GC_lookup_thread(self);
1610 # ifdef DEBUG_THREADS
1612 "Called GC_unregister_my_thread on %p, gc_thread = %p\n",
1613 (void *)self, (void *)me);
1615 GC_ASSERT(THREAD_EQUAL(me->id, self));
1616 GC_unregister_my_thread_inner(me);
1617 RESTORE_CANCEL(cancel_state);
1622 /* Called at thread exit. */
1623 /* Never called for main thread. That's OK, since it */
1624 /* results in at most a tiny one-time leak. And */
1625 /* linuxthreads doesn't reclaim the main threads */
1626 /* resources or id anyway. */
1627 GC_INNER_PTHRSTART void GC_thread_exit_proc(void *arg)
1629 IF_CANCEL(int cancel_state;)
1632 # ifdef DEBUG_THREADS
1633 GC_log_printf("Called GC_thread_exit_proc on %p, gc_thread = %p\n",
1634 (void *)((GC_thread)arg)->id, arg);
1637 DISABLE_CANCEL(cancel_state);
1638 GC_wait_for_gc_completion(FALSE);
1639 GC_unregister_my_thread_inner((GC_thread)arg);
1640 RESTORE_CANCEL(cancel_state);
1644 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
1645 GC_API int WRAP_FUNC(pthread_join)(pthread_t thread, void **retval)
1651 ASSERT_SYMS_INITIALIZED();
1653 t = GC_lookup_thread(thread);
1654 /* This is guaranteed to be the intended one, since the thread id */
1655 /* can't have been recycled by pthreads. */
1657 result = REAL_FUNC(pthread_join)(thread, retval);
1658 # if defined(GC_FREEBSD_THREADS)
1659 /* On FreeBSD, the wrapped pthread_join() sometimes returns (what
1660 appears to be) a spurious EINTR which caused the test and real code
1661 to gratuitously fail. Having looked at system pthread library source
1662 code, I see how this return code may be generated. In one path of
1663 code, pthread_join() just returns the errno setting of the thread
1664 being joined. This does not match the POSIX specification or the
1665 local man pages thus I have taken the liberty to catch this one
1666 spurious return value properly conditionalized on GC_FREEBSD_THREADS. */
1667 if (result == EINTR) result = 0;
1671 /* Here the pthread thread id may have been recycled. */
1672 /* Delete the thread from GC_threads (unless it has been */
1673 /* registered again from the client thread key destructor). */
1674 if ((t -> flags & FINISHED) != 0)
1675 GC_delete_gc_thread(t);
1681 GC_API int WRAP_FUNC(pthread_detach)(pthread_t thread)
1687 ASSERT_SYMS_INITIALIZED();
1689 t = GC_lookup_thread(thread);
1691 result = REAL_FUNC(pthread_detach)(thread);
1694 t -> flags |= DETACHED;
1695 /* Here the pthread thread id may have been recycled. */
1696 if ((t -> flags & FINISHED) != 0) {
1697 GC_delete_gc_thread(t);
1703 #endif /* !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */
1705 #ifndef GC_NO_PTHREAD_CANCEL
1706 /* We should deal with the fact that apparently on Solaris and, */
1707 /* probably, on some Linux we can't collect while a thread is */
1708 /* exiting, since signals aren't handled properly. This currently */
1709 /* gives rise to deadlocks. The only workaround seen is to intercept */
1710 /* pthread_cancel() and pthread_exit(), and disable the collections */
1711 /* until the thread exit handler is called. That's ugly, because we */
1712 /* risk growing the heap unnecessarily. But it seems that we don't */
1713 /* really have an option in that the process is not in a fully */
1714 /* functional state while a thread is exiting. */
1715 GC_API int WRAP_FUNC(pthread_cancel)(pthread_t thread)
1725 t = GC_lookup_thread(thread);
1726 /* We test DISABLED_GC because pthread_exit could be called at */
1727 /* the same time. (If t is NULL then pthread_cancel should */
1728 /* return ESRCH.) */
1729 if (t != NULL && (t -> flags & DISABLED_GC) == 0) {
1730 t -> flags |= DISABLED_GC;
1735 return REAL_FUNC(pthread_cancel)(thread);
1737 #endif /* !GC_NO_PTHREAD_CANCEL */
1739 #ifdef GC_HAVE_PTHREAD_EXIT
1740 GC_API GC_PTHREAD_EXIT_ATTRIBUTE void WRAP_FUNC(pthread_exit)(void *retval)
1742 pthread_t self = pthread_self();
1748 me = GC_lookup_thread(self);
1749 /* We test DISABLED_GC because someone else could call */
1750 /* pthread_cancel at the same time. */
1751 if (me != 0 && (me -> flags & DISABLED_GC) == 0) {
1752 me -> flags |= DISABLED_GC;
1757 REAL_FUNC(pthread_exit)(retval);
1759 #endif /* GC_HAVE_PTHREAD_EXIT */
1761 GC_INNER GC_bool GC_in_thread_creation = FALSE;
1762 /* Protected by allocation lock. */
1764 GC_INLINE void GC_record_stack_base(GC_thread me,
1765 const struct GC_stack_base *sb)
1767 # ifndef GC_DARWIN_THREADS
1768 me -> stop_info.stack_ptr = (ptr_t)sb->mem_base;
1770 me -> stack_end = (ptr_t)sb->mem_base;
1771 if (me -> stack_end == NULL)
1772 ABORT("Bad stack base in GC_register_my_thread");
1774 me -> backing_store_end = (ptr_t)sb->reg_base;
1778 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
1779 pthread_t my_pthread)
1783 GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */
1784 me = GC_new_thread(my_pthread);
1785 GC_in_thread_creation = FALSE;
1787 ABORT("Failed to allocate memory for thread registering");
1788 # ifdef GC_DARWIN_THREADS
1789 me -> stop_info.mach_thread = mach_thread_self();
1791 GC_record_stack_base(me, sb);
1792 # ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
1793 /* Since this could be executed from a detached thread */
1794 /* destructor, our signals might already be blocked. */
1795 GC_unblock_gc_signals();
1800 GC_API void GC_CALL GC_allow_register_threads(void)
1802 /* Check GC is initialized and the current thread is registered. */
1803 GC_ASSERT(GC_lookup_thread(pthread_self()) != 0);
1807 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
1809 pthread_t self = pthread_self();
1813 if (GC_need_to_lock == FALSE)
1814 ABORT("Threads explicit registering is not previously enabled");
1817 me = GC_lookup_thread(self);
1819 me = GC_register_my_thread_inner(sb, self);
1820 # if defined(CPPCHECK)
1821 GC_noop1(me->flags);
1823 me -> flags |= DETACHED;
1824 /* Treat as detached, since we do not need to worry about */
1825 /* pointer results. */
1826 # if defined(THREAD_LOCAL_ALLOC)
1827 GC_init_thread_local(&(me->tlfs));
1831 } else if ((me -> flags & FINISHED) != 0) {
1832 /* This code is executed when a thread is registered from the */
1833 /* client thread key destructor. */
1834 # ifdef GC_DARWIN_THREADS
1835 /* Reinitialize mach_thread to avoid thread_suspend fail */
1836 /* with MACH_SEND_INVALID_DEST error. */
1837 me -> stop_info.mach_thread = mach_thread_self();
1839 GC_record_stack_base(me, sb);
1840 me -> flags &= ~FINISHED; /* but not DETACHED */
1841 # ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
1842 /* Since this could be executed from a thread destructor, */
1843 /* our signals might be blocked. */
1844 GC_unblock_gc_signals();
1846 # if defined(THREAD_LOCAL_ALLOC)
1847 GC_init_thread_local(&(me->tlfs));
1853 return GC_DUPLICATE;
1858 void *(*start_routine)(void *);
1861 sem_t registered; /* 1 ==> in our thread table, but */
1862 /* parent hasn't yet noticed. */
1865 /* Called from GC_inner_start_routine(). Defined in this file to */
1866 /* minimize the number of include files in pthread_start.c (because */
1867 /* sem_t and sem_post() are not used that file directly). */
1868 GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
1869 void *(**pstart)(void *),
1871 struct GC_stack_base *sb, void *arg)
1873 struct start_info * si = (struct start_info *)arg;
1874 pthread_t self = pthread_self();
1878 # ifdef DEBUG_THREADS
1879 GC_log_printf("Starting thread %p, pid = %ld, sp = %p\n",
1880 (void *)self, (long)getpid(), (void *)&arg);
1883 me = GC_register_my_thread_inner(sb, self);
1884 me -> flags = si -> flags;
1885 # if defined(THREAD_LOCAL_ALLOC)
1886 GC_init_thread_local(&(me->tlfs));
1889 *pstart = si -> start_routine;
1890 # ifdef DEBUG_THREADS
1891 GC_log_printf("start_routine = %p\n", (void *)(signed_word)(*pstart));
1893 *pstart_arg = si -> arg;
1894 sem_post(&(si -> registered)); /* Last action on si. */
1895 /* OK to deallocate. */
1899 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
1900 STATIC void * GC_start_routine(void * arg)
1902 # ifdef INCLUDE_LINUX_THREAD_DESCR
1903 struct GC_stack_base sb;
1905 # ifdef REDIRECT_MALLOC
1906 /* GC_get_stack_base may call pthread_getattr_np, which can */
1907 /* unfortunately call realloc, which may allocate from an */
1908 /* unregistered thread. This is unpleasant, since it might */
1909 /* force heap growth (or, even, heap overflow). */
1912 if (GC_get_stack_base(&sb) != GC_SUCCESS)
1913 ABORT("Failed to get thread stack base");
1914 # ifdef REDIRECT_MALLOC
1917 return GC_inner_start_routine(&sb, arg);
1919 return GC_call_with_stack_base(GC_inner_start_routine, arg);
1923 GC_API int WRAP_FUNC(pthread_create)(pthread_t *new_thread,
1924 GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
1925 void *(*start_routine)(void *), void *arg)
1930 struct start_info si;
1932 /* This is otherwise saved only in an area mmapped by the thread */
1933 /* library, which isn't visible to the collector. */
1935 /* We resist the temptation to muck with the stack size here, */
1936 /* even if the default is unreasonably small. That's the client's */
1937 /* responsibility. */
1940 if (!EXPECT(parallel_initialized, TRUE))
1942 if (sem_init(&si.registered, GC_SEM_INIT_PSHARED, 0) != 0)
1943 ABORT("sem_init failed");
1945 si.start_routine = start_routine;
1948 if (!EXPECT(GC_thr_initialized, TRUE))
1950 # ifdef GC_ASSERTIONS
1952 size_t stack_size = 0;
1954 if (pthread_attr_getstacksize(attr, &stack_size) != 0)
1955 ABORT("pthread_attr_getstacksize failed");
1957 if (0 == stack_size) {
1958 pthread_attr_t my_attr;
1960 if (pthread_attr_init(&my_attr) != 0)
1961 ABORT("pthread_attr_init failed");
1962 if (pthread_attr_getstacksize(&my_attr, &stack_size) != 0)
1963 ABORT("pthread_attr_getstacksize failed");
1964 (void)pthread_attr_destroy(&my_attr);
1966 /* On Solaris 10, with default attr initialization, */
1967 /* stack_size remains 0. Fudge it. */
1968 if (0 == stack_size) {
1970 WARN("Failed to get stack size for assertion checking\n", 0);
1972 stack_size = 1000000;
1974 GC_ASSERT(stack_size >= 65536);
1975 /* Our threads may need to do some work for the GC. */
1976 /* Ridiculously small threads won't work, and they */
1977 /* probably wouldn't work anyway. */
1981 detachstate = PTHREAD_CREATE_JOINABLE;
1983 pthread_attr_getdetachstate(attr, &detachstate);
1985 if (PTHREAD_CREATE_DETACHED == detachstate) my_flags |= DETACHED;
1986 si.flags = my_flags;
1988 # ifdef DEBUG_THREADS
1989 GC_log_printf("About to start new thread from thread %p\n",
1990 (void *)pthread_self());
1993 result = REAL_FUNC(pthread_create)(new_thread, attr, GC_start_routine,
1996 /* Wait until child has been added to the thread table. */
1997 /* This also ensures that we hold onto the stack-allocated si until */
1998 /* the child is done with it. */
2000 IF_CANCEL(int cancel_state;)
2002 # ifdef DEBUG_THREADS
2003 /* new_thread is non-NULL because pthread_create requires it. */
2004 GC_log_printf("Started thread %p\n", (void *)(*new_thread));
2006 DISABLE_CANCEL(cancel_state);
2007 /* pthread_create is not a cancellation point. */
2008 while (0 != sem_wait(&si.registered)) {
2009 # if defined(GC_HAIKU_THREADS)
2010 /* To workaround some bug in Haiku semaphores. */
2011 if (EACCES == errno) continue;
2013 if (EINTR != errno) ABORT("sem_wait failed");
2015 RESTORE_CANCEL(cancel_state);
2017 sem_destroy(&si.registered);
2020 #endif /* !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */
2022 #if defined(USE_SPIN_LOCK) || !defined(NO_PTHREAD_TRYLOCK)
2023 /* Spend a few cycles in a way that can't introduce contention with */
2024 /* other threads. */
2025 #define GC_PAUSE_SPIN_CYCLES 10
2026 STATIC void GC_pause(void)
2030 for (i = 0; i < GC_PAUSE_SPIN_CYCLES; ++i) {
2031 /* Something that's unlikely to be optimized away. */
2032 # if defined(AO_HAVE_compiler_barrier) \
2033 && !defined(BASE_ATOMIC_OPS_EMULATED)
2034 AO_compiler_barrier();
2043 # define SPIN_MAX 128 /* Maximum number of calls to GC_pause before */
2047 GC_INNER volatile GC_bool GC_collecting = FALSE;
2048 /* A hint that we're in the collector and */
2049 /* holding the allocation lock for an */
2050 /* extended period. */
2052 #if (!defined(USE_SPIN_LOCK) && !defined(NO_PTHREAD_TRYLOCK)) \
2053 || defined(PARALLEL_MARK)
2054 /* If we don't want to use the below spinlock implementation, either */
2055 /* because we don't have a GC_test_and_set implementation, or because */
2056 /* we don't want to risk sleeping, we can still try spinning on */
2057 /* pthread_mutex_trylock for a while. This appears to be very */
2058 /* beneficial in many cases. */
2059 /* I suspect that under high contention this is nearly always better */
2060 /* than the spin lock. But it's a bit slower on a uniprocessor. */
2061 /* Hence we still default to the spin lock. */
2062 /* This is also used to acquire the mark lock for the parallel */
2065 /* Here we use a strict exponential backoff scheme. I don't know */
2066 /* whether that's better or worse than the above. We eventually */
2067 /* yield by calling pthread_mutex_lock(); it never makes sense to */
2068 /* explicitly sleep. */
2070 /* #define LOCK_STATS */
2071 /* Note that LOCK_STATS requires AO_HAVE_test_and_set. */
2073 volatile AO_t GC_spin_count = 0;
2074 volatile AO_t GC_block_count = 0;
2075 volatile AO_t GC_unlocked_count = 0;
2078 STATIC void GC_generic_lock(pthread_mutex_t * lock)
2080 #ifndef NO_PTHREAD_TRYLOCK
2081 unsigned pause_length = 1;
2084 if (0 == pthread_mutex_trylock(lock)) {
2086 (void)AO_fetch_and_add1(&GC_unlocked_count);
2090 for (; pause_length <= SPIN_MAX; pause_length <<= 1) {
2091 for (i = 0; i < pause_length; ++i) {
2094 switch(pthread_mutex_trylock(lock)) {
2097 (void)AO_fetch_and_add1(&GC_spin_count);
2103 ABORT("Unexpected error from pthread_mutex_trylock");
2106 #endif /* !NO_PTHREAD_TRYLOCK */
2108 (void)AO_fetch_and_add1(&GC_block_count);
2110 pthread_mutex_lock(lock);
2113 #endif /* !USE_SPIN_LOCK || ... */
2115 #if defined(AO_HAVE_char_load) && !defined(BASE_ATOMIC_OPS_EMULATED)
2116 # define is_collecting() \
2117 ((GC_bool)AO_char_load((unsigned char *)&GC_collecting))
2119 /* GC_collecting is a hint, a potential data race between */
2120 /* GC_lock() and ENTER/EXIT_GC() is OK to ignore. */
2121 # define is_collecting() GC_collecting
2124 #if defined(USE_SPIN_LOCK)
2126 /* Reasonably fast spin locks. Basically the same implementation */
2127 /* as STL alloc.h. This isn't really the right way to do this. */
2128 /* but until the POSIX scheduling mess gets straightened out ... */
2130 GC_INNER volatile AO_TS_t GC_allocate_lock = AO_TS_INITIALIZER;
2132 # define low_spin_max 30 /* spin cycles if we suspect uniprocessor */
2133 # define high_spin_max SPIN_MAX /* spin cycles for multiprocessor */
2135 static volatile AO_t spin_max = low_spin_max;
2136 static volatile AO_t last_spins = 0;
2137 /* A potential data race between */
2138 /* threads invoking GC_lock which reads */
2139 /* and updates spin_max and last_spins */
2140 /* could be ignored because these */
2141 /* variables are hints only. */
2143 GC_INNER void GC_lock(void)
2145 unsigned my_spin_max;
2146 unsigned my_last_spins;
2149 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2152 my_spin_max = (unsigned)AO_load(&spin_max);
2153 my_last_spins = (unsigned)AO_load(&last_spins);
2154 for (i = 0; i < my_spin_max; i++) {
2155 if (is_collecting() || GC_nprocs == 1)
2157 if (i < my_last_spins/2) {
2161 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2164 * Spinning worked. Thus we're probably not being scheduled
2165 * against the other process with which we were contending.
2166 * Thus it makes sense to spin longer the next time.
2168 AO_store(&last_spins, (AO_t)i);
2169 AO_store(&spin_max, (AO_t)high_spin_max);
2173 /* We are probably being scheduled against the other process. Sleep. */
2174 AO_store(&spin_max, (AO_t)low_spin_max);
2177 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2180 # define SLEEP_THRESHOLD 12
2181 /* Under Linux very short sleeps tend to wait until */
2182 /* the current time quantum expires. On old Linux */
2183 /* kernels nanosleep (<= 2 ms) just spins. */
2184 /* (Under 2.4, this happens only for real-time */
2185 /* processes.) We want to minimize both behaviors */
2187 if (i < SLEEP_THRESHOLD) {
2193 /* Don't wait for more than about 15 ms, */
2194 /* even under extreme contention. */
2196 ts.tv_nsec = 1 << i;
2202 #elif defined(USE_PTHREAD_LOCKS)
2204 # ifndef NO_PTHREAD_TRYLOCK
2205 GC_INNER void GC_lock(void)
2207 if (1 == GC_nprocs || is_collecting()) {
2208 pthread_mutex_lock(&GC_allocate_ml);
2210 GC_generic_lock(&GC_allocate_ml);
2213 # elif defined(GC_ASSERTIONS)
2214 GC_INNER void GC_lock(void)
2216 pthread_mutex_lock(&GC_allocate_ml);
2220 #endif /* !USE_SPIN_LOCK && USE_PTHREAD_LOCKS */
2222 #ifdef PARALLEL_MARK
2224 # ifdef GC_ASSERTIONS
2225 STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
2226 # define SET_MARK_LOCK_HOLDER \
2227 (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
2228 # define UNSET_MARK_LOCK_HOLDER \
2230 GC_ASSERT(GC_mark_lock_holder \
2231 == NUMERIC_THREAD_ID(pthread_self())); \
2232 GC_mark_lock_holder = NO_THREAD; \
2235 # define SET_MARK_LOCK_HOLDER (void)0
2236 # define UNSET_MARK_LOCK_HOLDER (void)0
2237 # endif /* !GC_ASSERTIONS */
2239 #ifdef GLIBC_2_1_MUTEX_HACK
2240 /* Ugly workaround for a linux threads bug in the final versions */
2241 /* of glibc2.1. Pthread_mutex_trylock sets the mutex owner */
2242 /* field even when it fails to acquire the mutex. This causes */
2243 /* pthread_cond_wait to die. Remove for glibc2.2. */
2244 /* According to the man page, we should use */
2245 /* PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, but that isn't actually */
2247 static pthread_mutex_t mark_mutex =
2248 {0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, {0, 0}};
2250 static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
2253 static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
2255 #ifdef GLIBC_2_19_TSX_BUG
2256 /* Parse string like <major>[.<minor>[<tail>]] and return major value. */
2257 static int parse_version(int *pminor, const char *pverstr) {
2259 unsigned long value = strtoul(pverstr, &endp, 10);
2260 int major = (int)value;
2262 if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
2267 /* No minor part. */
2270 value = strtoul(endp + 1, &endp, 10);
2271 *pminor = (int)value;
2272 if (*pminor < 0 || (unsigned)(*pminor) != value) {
2278 #endif /* GLIBC_2_19_TSX_BUG */
2280 static void setup_mark_lock(void)
2282 # ifdef GLIBC_2_19_TSX_BUG
2283 pthread_mutexattr_t mattr;
2284 int glibc_minor = -1;
2285 int glibc_major = parse_version(&glibc_minor, gnu_get_libc_version());
2287 if (glibc_major > 2 || (glibc_major == 2 && glibc_minor >= 19)) {
2288 /* TODO: disable this workaround for glibc with fixed TSX */
2289 /* This disables lock elision to workaround a bug in glibc 2.19+ */
2290 if (0 != pthread_mutexattr_init(&mattr)) {
2291 ABORT("pthread_mutexattr_init failed");
2293 if (0 != pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL)) {
2294 ABORT("pthread_mutexattr_settype failed");
2296 if (0 != pthread_mutex_init(&mark_mutex, &mattr)) {
2297 ABORT("pthread_mutex_init failed");
2299 (void)pthread_mutexattr_destroy(&mattr);
2304 GC_INNER void GC_acquire_mark_lock(void)
2306 # if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
2307 GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
2309 GC_generic_lock(&mark_mutex);
2310 SET_MARK_LOCK_HOLDER;
2313 GC_INNER void GC_release_mark_lock(void)
2315 UNSET_MARK_LOCK_HOLDER;
2316 if (pthread_mutex_unlock(&mark_mutex) != 0) {
2317 ABORT("pthread_mutex_unlock failed");
2321 /* Collector must wait for a freelist builders for 2 reasons: */
2322 /* 1) Mark bits may still be getting examined without lock. */
2323 /* 2) Partial free lists referenced only by locals may not be scanned */
2324 /* correctly, e.g. if they contain "pointer-free" objects, since the */
2325 /* free-list link may be ignored. */
2326 STATIC void GC_wait_builder(void)
2328 ASSERT_CANCEL_DISABLED();
2329 UNSET_MARK_LOCK_HOLDER;
2330 if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
2331 ABORT("pthread_cond_wait failed");
2333 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2334 SET_MARK_LOCK_HOLDER;
2337 GC_INNER void GC_wait_for_reclaim(void)
2339 GC_acquire_mark_lock();
2340 while (GC_fl_builder_count > 0) {
2343 GC_release_mark_lock();
2346 GC_INNER void GC_notify_all_builder(void)
2348 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
2349 if (pthread_cond_broadcast(&builder_cv) != 0) {
2350 ABORT("pthread_cond_broadcast failed");
2354 GC_INNER void GC_wait_marker(void)
2356 ASSERT_CANCEL_DISABLED();
2357 GC_ASSERT(GC_parallel);
2358 UNSET_MARK_LOCK_HOLDER;
2359 if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
2360 ABORT("pthread_cond_wait failed");
2362 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2363 SET_MARK_LOCK_HOLDER;
2366 GC_INNER void GC_notify_all_marker(void)
2368 GC_ASSERT(GC_parallel);
2369 if (pthread_cond_broadcast(&mark_cv) != 0) {
2370 ABORT("pthread_cond_broadcast failed");
2374 #endif /* PARALLEL_MARK */
2376 #ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS
2377 /* Workaround "undefined reference" linkage errors on some targets. */
2379 extern void __pthread_register_cancel(void) __attribute__((__weak__));
2380 extern void __pthread_unregister_cancel(void) __attribute__((__weak__));
2383 void __pthread_register_cancel(void) {}
2384 void __pthread_unregister_cancel(void) {}
2387 #endif /* GC_PTHREADS */