2 * Copyright (c) 2008-2009 Apple Inc. All rights reserved.
4 * @APPLE_APACHE_LICENSE_HEADER_START@
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 * @APPLE_APACHE_LICENSE_HEADER_END@
23 // semaphores are too fundamental to use the dispatch_assume*() macros
25 #define DISPATCH_SEMAPHORE_VERIFY_KR(x) do { \
27 DISPATCH_CRASH("flawed group/semaphore logic"); \
32 #define DISPATCH_SEMAPHORE_VERIFY_RET(x) do { \
34 DISPATCH_CRASH("flawed group/semaphore logic"); \
39 struct dispatch_semaphore_vtable_s {
40 DISPATCH_VTABLE_HEADER(dispatch_semaphore_s);
43 static void _dispatch_semaphore_dispose(dispatch_semaphore_t dsema);
44 static size_t _dispatch_semaphore_debug(dispatch_semaphore_t dsema, char *buf, size_t bufsiz);
45 static long _dispatch_group_wake(dispatch_semaphore_t dsema);
47 const struct dispatch_semaphore_vtable_s _dispatch_semaphore_vtable = {
48 .do_type = DISPATCH_SEMAPHORE_TYPE,
49 .do_kind = "semaphore",
50 .do_dispose = _dispatch_semaphore_dispose,
51 .do_debug = _dispatch_semaphore_debug,
55 _dispatch_get_thread_semaphore(void)
57 dispatch_semaphore_t dsema;
59 dsema = fastpath(_dispatch_thread_getspecific(dispatch_sema4_key));
61 while (!(dsema = dispatch_semaphore_create(0))) {
65 _dispatch_thread_setspecific(dispatch_sema4_key, NULL);
70 _dispatch_put_thread_semaphore(dispatch_semaphore_t dsema)
72 dispatch_semaphore_t old_sema = _dispatch_thread_getspecific(dispatch_sema4_key);
73 _dispatch_thread_setspecific(dispatch_sema4_key, dsema);
75 dispatch_release(old_sema);
80 dispatch_group_create(void)
82 return (dispatch_group_t)dispatch_semaphore_create(LONG_MAX);
86 dispatch_semaphore_create(long value)
88 dispatch_semaphore_t dsema;
93 // If the internal value is negative, then the absolute of the value is
94 // equal to the number of waiting threads. Therefore it is bogus to
95 // initialize the semaphore with a negative value.
100 dsema = calloc(1, sizeof(struct dispatch_semaphore_s));
102 if (fastpath(dsema)) {
103 dsema->do_vtable = &_dispatch_semaphore_vtable;
104 dsema->do_next = DISPATCH_OBJECT_LISTLESS;
105 dsema->do_ref_cnt = 1;
106 dsema->do_xref_cnt = 1;
107 dsema->do_targetq = dispatch_get_global_queue(0, 0);
108 dsema->dsema_value = value;
109 dsema->dsema_orig = value;
111 ret = sem_init(&dsema->dsema_sem, 0, 0);
112 (void)dispatch_assume_zero(ret);
121 _dispatch_semaphore_create_port(semaphore_t *s4)
130 // lazily allocate the semaphore port
133 // 1) Switch to a doubly-linked FIFO in user-space.
134 // 2) User-space timers for the timeout.
135 // 3) Use the per-thread semaphore port.
137 while (dispatch_assume_zero(kr = semaphore_create(mach_task_self(), &tmp, SYNC_POLICY_FIFO, 0))) {
138 DISPATCH_VERIFY_MIG(kr);
142 if (!dispatch_atomic_cmpxchg(s4, 0, tmp)) {
143 kr = semaphore_destroy(mach_task_self(), tmp);
144 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
147 _dispatch_safe_fork = false;
153 _dispatch_semaphore_wait_slow(dispatch_semaphore_t dsema, dispatch_time_t timeout)
156 mach_timespec_t _timeout;
161 struct timespec _timeout;
167 // Mach semaphores appear to sometimes spuriously wake up. Therefore,
168 // we keep a parallel count of the number of times a Mach semaphore is
169 // signaled (6880961).
170 while ((orig = dsema->dsema_sent_ksignals)) {
171 if (dispatch_atomic_cmpxchg(&dsema->dsema_sent_ksignals, orig, orig - 1)) {
177 _dispatch_semaphore_create_port(&dsema->dsema_port);
180 // From xnu/osfmk/kern/sync_sema.c:
181 // wait_semaphore->count = -1; /* we don't keep an actual count */
183 // The code above does not match the documentation, and that fact is
184 // not surprising. The documented semantics are clumsy to use in any
185 // practical way. The above hack effectively tricks the rest of the
186 // Mach semaphore logic to behave like the libdispatch algorithm.
192 // timeout() already calculates relative time left
193 nsec = _dispatch_timeout(timeout);
194 _timeout.tv_sec = (typeof(_timeout.tv_sec))(nsec / NSEC_PER_SEC);
195 _timeout.tv_nsec = (typeof(_timeout.tv_nsec))(nsec % NSEC_PER_SEC);
196 kr = slowpath(semaphore_timedwait(dsema->dsema_port, _timeout));
197 } while (kr == KERN_ABORTED);
199 if (kr != KERN_OPERATION_TIMED_OUT) {
200 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
206 _timeout = _dispatch_timeout_ts(timeout);
207 ret = slowpath(sem_timedwait(&dsema->dsema_sem,
209 } while (ret == -1 && errno == EINTR);
211 if (!(ret == -1 && errno == ETIMEDOUT)) {
212 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
216 // Fall through and try to undo what the fast path did to dsema->dsema_value
217 case DISPATCH_TIME_NOW:
218 while ((orig = dsema->dsema_value) < 0) {
219 if (dispatch_atomic_cmpxchg(&dsema->dsema_value, orig, orig + 1)) {
221 return KERN_OPERATION_TIMED_OUT;
229 // Another thread called semaphore_signal().
230 // Fall through and drain the wakeup.
231 case DISPATCH_TIME_FOREVER:
234 kr = semaphore_wait(dsema->dsema_port);
235 } while (kr == KERN_ABORTED);
236 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
240 ret = sem_wait(&dsema->dsema_sem);
242 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
252 dispatch_group_enter(dispatch_group_t dg)
254 dispatch_semaphore_t dsema = (dispatch_semaphore_t)dg;
255 #if USE_APPLE_SEMAPHORE_OPTIMIZATIONS && defined(__OPTIMIZE__) && defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) && !defined(__llvm__)
257 // 1) Way too much about the optimizer of GCC.
258 // 2) There will never be more than LONG_MAX threads.
259 // Therefore: no overflow detection
269 : "+m" (dsema->dsema_value)
273 _dispatch_semaphore_wait_slow(dsema, DISPATCH_TIME_FOREVER);
275 dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
281 dispatch_semaphore_wait(dispatch_semaphore_t dsema, dispatch_time_t timeout)
283 #if USE_APPLE_SEMAPHORE_OPTIMIZATIONS && defined(__OPTIMIZE__) && defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) && !defined(__llvm__)
285 // 1) Way too much about the optimizer of GCC.
286 // 2) There will never be more than LONG_MAX threads.
287 // Therefore: no overflow detection
295 "xor %%eax, %%eax\n\t"
298 : "+m" (dsema->dsema_value)
303 if (dispatch_atomic_dec(&dsema->dsema_value) >= 0) {
307 return _dispatch_semaphore_wait_slow(dsema, timeout);
312 _dispatch_semaphore_signal_slow(dispatch_semaphore_t dsema)
320 _dispatch_semaphore_create_port(&dsema->dsema_port);
323 // Before dsema_sent_ksignals is incremented we can rely on the reference
324 // held by the waiter. However, once this value is incremented the waiter
325 // may return between the atomic increment and the semaphore_signal(),
326 // therefore an explicit reference must be held in order to safely access
327 // dsema after the atomic increment.
328 _dispatch_retain(dsema);
330 dispatch_atomic_inc(&dsema->dsema_sent_ksignals);
333 kr = semaphore_signal(dsema->dsema_port);
334 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
337 ret = sem_post(&dsema->dsema_sem);
338 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
341 _dispatch_release(dsema);
347 dispatch_group_leave(dispatch_group_t dg)
349 dispatch_semaphore_t dsema = (dispatch_semaphore_t)dg;
351 dispatch_semaphore_signal(dsema);
353 if (dsema->dsema_value == dsema->dsema_orig) {
354 _dispatch_group_wake(dsema);
360 dispatch_semaphore_signal(dispatch_semaphore_t dsema)
362 #if USE_APPLE_SEMAPHORE_OPTIMIZATIONS && defined(__OPTIMIZE__) && defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) && !defined(__llvm__)
363 // overflow detection
364 // this assumes way too much about the optimizer of GCC
373 "xor %%eax, %%eax\n\t"
378 : "+m" (dsema->dsema_value)
383 if (dispatch_atomic_inc(&dsema->dsema_value) > 0) {
387 return _dispatch_semaphore_signal_slow(dsema);
392 _dispatch_group_wake(dispatch_semaphore_t dsema)
394 struct dispatch_sema_notify_s *tmp, *head = dispatch_atomic_xchg(&dsema->dsema_notify_head, NULL);
395 long rval = dispatch_atomic_xchg(&dsema->dsema_group_waiters, 0);
404 // wake any "group" waiter or notify blocks
408 _dispatch_semaphore_create_port(&dsema->dsema_waiter_port);
410 kr = semaphore_signal(dsema->dsema_waiter_port);
411 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
416 ret = sem_post(&dsema->dsema_sem);
417 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
422 dispatch_async_f(head->dsn_queue, head->dsn_ctxt, head->dsn_func);
423 _dispatch_release(head->dsn_queue);
425 tmp = head->dsn_next;
426 } while (!tmp && !dispatch_atomic_cmpxchg(&dsema->dsema_notify_tail, head, NULL));
431 _dispatch_release(dsema);
438 _dispatch_group_wait_slow(dispatch_semaphore_t dsema, dispatch_time_t timeout)
441 mach_timespec_t _timeout;
446 struct timespec _timeout;
452 // check before we cause another signal to be sent by incrementing dsema->dsema_group_waiters
453 if (dsema->dsema_value == dsema->dsema_orig) {
454 return _dispatch_group_wake(dsema);
456 // Mach semaphores appear to sometimes spuriously wake up. Therefore,
457 // we keep a parallel count of the number of times a Mach semaphore is
458 // signaled (6880961).
459 dispatch_atomic_inc(&dsema->dsema_group_waiters);
460 // check the values again in case we need to wake any threads
461 if (dsema->dsema_value == dsema->dsema_orig) {
462 return _dispatch_group_wake(dsema);
466 _dispatch_semaphore_create_port(&dsema->dsema_waiter_port);
469 // From xnu/osfmk/kern/sync_sema.c:
470 // wait_semaphore->count = -1; /* we don't keep an actual count */
472 // The code above does not match the documentation, and that fact is
473 // not surprising. The documented semantics are clumsy to use in any
474 // practical way. The above hack effectively tricks the rest of the
475 // Mach semaphore logic to behave like the libdispatch algorithm.
481 nsec = _dispatch_timeout(timeout);
482 _timeout.tv_sec = (typeof(_timeout.tv_sec))(nsec / NSEC_PER_SEC);
483 _timeout.tv_nsec = (typeof(_timeout.tv_nsec))(nsec % NSEC_PER_SEC);
484 kr = slowpath(semaphore_timedwait(dsema->dsema_waiter_port, _timeout));
485 } while (kr == KERN_ABORTED);
486 if (kr != KERN_OPERATION_TIMED_OUT) {
487 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
493 _timeout = _dispatch_timeout_ts(timeout);
494 ret = slowpath(sem_timedwait(&dsema->dsema_sem,
496 } while (ret == -1 && errno == EINTR);
498 if (!(ret == -1 && errno == ETIMEDOUT)) {
499 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
503 // Fall through and try to undo the earlier change to dsema->dsema_group_waiters
504 case DISPATCH_TIME_NOW:
505 while ((orig = dsema->dsema_group_waiters)) {
506 if (dispatch_atomic_cmpxchg(&dsema->dsema_group_waiters, orig, orig - 1)) {
508 return KERN_OPERATION_TIMED_OUT;
516 // Another thread called semaphore_signal().
517 // Fall through and drain the wakeup.
518 case DISPATCH_TIME_FOREVER:
521 kr = semaphore_wait(dsema->dsema_waiter_port);
522 } while (kr == KERN_ABORTED);
523 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
527 ret = sem_wait(&dsema->dsema_sem);
528 } while (ret == -1 && errno == EINTR);
529 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
538 dispatch_group_wait(dispatch_group_t dg, dispatch_time_t timeout)
540 dispatch_semaphore_t dsema = (dispatch_semaphore_t)dg;
542 if (dsema->dsema_value == dsema->dsema_orig) {
547 return KERN_OPERATION_TIMED_OUT;
554 return _dispatch_group_wait_slow(dsema, timeout);
559 dispatch_group_notify(dispatch_group_t dg, dispatch_queue_t dq, dispatch_block_t db)
561 dispatch_group_notify_f(dg, dq, _dispatch_Block_copy(db), _dispatch_call_block_and_release);
566 dispatch_group_notify_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, void (*func)(void *))
568 dispatch_semaphore_t dsema = (dispatch_semaphore_t)dg;
569 struct dispatch_sema_notify_s *dsn, *prev;
571 // FIXME -- this should be updated to use the continuation cache
572 while (!(dsn = malloc(sizeof(*dsn)))) {
576 dsn->dsn_next = NULL;
578 dsn->dsn_ctxt = ctxt;
579 dsn->dsn_func = func;
580 _dispatch_retain(dq);
582 prev = dispatch_atomic_xchg(&dsema->dsema_notify_tail, dsn);
583 if (fastpath(prev)) {
584 prev->dsn_next = dsn;
586 _dispatch_retain(dg);
587 dsema->dsema_notify_head = dsn;
588 if (dsema->dsema_value == dsema->dsema_orig) {
589 _dispatch_group_wake(dsema);
595 _dispatch_semaphore_dispose(dispatch_semaphore_t dsema)
604 if (dsema->dsema_value < dsema->dsema_orig) {
605 DISPATCH_CLIENT_CRASH("Semaphore/group object deallocated while in use");
609 if (dsema->dsema_port) {
610 kr = semaphore_destroy(mach_task_self(), dsema->dsema_port);
611 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
613 if (dsema->dsema_waiter_port) {
614 kr = semaphore_destroy(mach_task_self(), dsema->dsema_waiter_port);
615 DISPATCH_SEMAPHORE_VERIFY_KR(kr);
619 ret = sem_destroy(&dsema->dsema_sem);
620 DISPATCH_SEMAPHORE_VERIFY_RET(ret);
623 _dispatch_dispose(dsema);
627 _dispatch_semaphore_debug(dispatch_semaphore_t dsema, char *buf, size_t bufsiz)
630 offset += snprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ", dx_kind(dsema), dsema);
631 offset += dispatch_object_debug_attr(dsema, &buf[offset], bufsiz - offset);
633 offset += snprintf(&buf[offset], bufsiz - offset, "port = 0x%u, ",
636 offset += snprintf(&buf[offset], bufsiz - offset,
637 "value = %ld, orig = %ld }", dsema->dsema_value, dsema->dsema_orig);
643 dispatch_group_async(dispatch_group_t dg, dispatch_queue_t dq, dispatch_block_t db)
645 dispatch_group_async_f(dg, dq, _dispatch_Block_copy(db), _dispatch_call_block_and_release);
651 dispatch_group_async_f(dispatch_group_t dg, dispatch_queue_t dq, void *ctxt, void (*func)(void *))
653 dispatch_continuation_t dc;
655 _dispatch_retain(dg);
656 dispatch_group_enter(dg);
658 dc = _dispatch_continuation_alloc_cacheonly() ?: _dispatch_continuation_alloc_from_heap();
660 dc->do_vtable = (void *)(DISPATCH_OBJ_ASYNC_BIT|DISPATCH_OBJ_GROUP_BIT);
665 _dispatch_queue_push(dq, dc);