+2003-05-25 Ulrich Drepper <drepper@redhat.com>
+
+ * sysdeps/unix/sysv/linux/kernel-features.h: Define
+ __ASSUME_FUTEX_REQUEUE for >= 2.5.70.
+
2003-05-22 Andreas Jaeger <aj@suse.de>
* sysdeps/unix/sysv/linux/x86_64/gettimeofday.S: Add CFI
2003-05-21 H.J. Lu <hongjiu.lu@intel.com>
- * math/test-fenv.c (feexcp_nomask_test): Fix comment
+ * math/test-fenv.c (feexcp_nomask_test): Fix comment.
(feexcp_mask_test): Likewise.
2003-05-21 Ulrich Drepper <drepper@redhat.com>
+2003-05-25 Ulrich Drepper <drepper@redhat.com>
+
+ * sysdeps/pthread/pthread_cond_broadcast.c: Try using FUTEX_REQUEUE
+ instead of FUTEX_WAIT.
+ * sysdeps/pthread/pthread_cond_signal.c: Likewise.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
+ * sysdeps/pthread/pthread_cond_timedwait.c: Remember mutex which was
+ used in condvar structure. Call __pthread_mutex_cond_lock instead
+ of __pthread_mutex_lock_internal.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
+ * sysdeps/pthread/pthread_cond_wait.c: Likewise.
+ (__condvar_cleanup): Always call __pthread_mutex_cond_lock.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
+ * sysdeps/unix/sysv/linux/Makefile (libpthread-sysdep_routines):
+ Add pthread_mutex_cond_lock.
+ * sysdeps/unix/sysv/linux/lowlevelcond.sym: Add dep_mutex.
+ * sysdeps/unix/sysv/linux/pthread_cond_mutex_lock.c: New file.
+ * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Define
+ lll_mutex_cond_lock.
+ * sysdeps/unix/sysv/linux/ia64/lowlevellock.h: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
+ * sysdeps/unix/sysv/linux/i386/bits/pthread_types.h (pthread_cond_t):
+ Add __mutex field.
+ * sysdeps/unix/sysv/linux/ia64/bits/pthread_types.h: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise.
+
+ * sysdeps/i386/tcb-offsets.sym: Define MUTEX_FUTEX.
+ * sysdeps/x86_64/tcb-offsets.sym: Likewise.
+
+ * pthreadP.h: Declare __pthread_mutex_cond_lock.
+ * pthread_mutex_lock.c: Define LLL_MUTEX_LOCK if not already defined.
+ Use it instead of lll_mutex_lock. If __pthread_mutex_lock is a
+ macro don't define aliases.
+
+ * cancellation.c: Remove __pthread_enable_asynccancel_2.
+ * pthreadP.h: Remove declaration of __pthread_enable_asynccancel_2.
+ * sysdeps/pthread/pthread_cond_timedwait.c: Use
+ __pthread_enable_asynccancel instead of __pthread_enable_asynccancel_2.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
+ * sysdeps/pthread/pthread_cond_wait.c: Likewise.
+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
+
2003-05-17 Ulrich Drepper <drepper@redhat.com>
* sem_open.c: Fix one endless loop. Implement correct semantics
extern int __pthread_mutex_trylock (pthread_mutex_t *_mutex);
extern int __pthread_mutex_lock (pthread_mutex_t *__mutex);
extern int __pthread_mutex_lock_internal (pthread_mutex_t *__mutex);
+extern int __pthread_mutex_cond_lock (pthread_mutex_t *__mutex)
+ attribute_hidden;
extern int __pthread_mutex_unlock (pthread_mutex_t *__mutex);
extern int __pthread_mutex_unlock_internal (pthread_mutex_t *__mutex);
extern int __pthread_mutexattr_init (pthread_mutexattr_t *attr);
extern void __pthread_exit (void *value);
extern int __pthread_setcanceltype (int type, int *oldtype);
extern int __pthread_enable_asynccancel (void) attribute_hidden;
-extern void __pthread_enable_asynccancel_2 (int *oldvalp)
- internal_function attribute_hidden;
extern void __pthread_disable_asynccancel (int oldtype)
internal_function attribute_hidden;
#include <lowlevellock.h>
+#ifndef LLL_MUTEX_LOCK
+# define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex)
+#endif
+
+
int
__pthread_mutex_lock (mutex)
pthread_mutex_t *mutex;
else
{
/* We have to get the mutex. */
- lll_mutex_lock (mutex->__data.__lock);
+ LLL_MUTEX_LOCK (mutex->__data.__lock);
/* Record the ownership. */
mutex->__data.__owner = id;
case PTHREAD_MUTEX_TIMED_NP:
case PTHREAD_MUTEX_ADAPTIVE_NP:
/* Normal mutex. */
- lll_mutex_lock (mutex->__data.__lock);
+ LLL_MUTEX_LOCK (mutex->__data.__lock);
/* Record the ownership. */
mutex->__data.__owner = id;
break;
return 0;
}
+#ifndef __pthread_mutex_lock
strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal)
+#endif
SYSINFO_OFFSET offsetof (tcbhead_t, sysinfo)
CLEANUP offsetof (struct pthread, cleanup)
CLEANUP_PREV offsetof (struct _pthread_cleanup_buffer, __prev)
+MUTEX_FUTEX offsetof (pthread_mutex_t, __data.__lock)
#include <pthreadP.h>
#include <shlib-compat.h>
+#include <kernel-features.h>
int
#endif
/* Wake everybody. */
- lll_futex_wake (futex, INT_MAX);
+ pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
+ if (__builtin_expect (lll_futex_requeue (futex, 1, MAX_INT,
+ &mut->__data.__lock) == -EINVAL,
+ 0))
+ {
+ /* The requeue functionality is not available. */
+#ifndef __ASSUME_FUTEX_REQUEUE
+ lll_futex_wake (futex, MAX_INT);
+#endif
+ }
/* That's all. */
return 0;
#include <pthreadP.h>
#include <shlib-compat.h>
+#include <kernel-features.h>
+
int
__pthread_cond_signal (cond)
pthread_cond_t *cond;
{
/* Make sure we are alone. */
- lll_mutex_lock(cond->__data.__lock);
+ lll_mutex_lock (cond->__data.__lock);
/* Are there any waiters to be woken? */
if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
#endif
/* Wake one. */
- lll_futex_wake (futex, 1);
+ int r = lll_futex_requeue (futex, 0, 1, &cond->__data.__lock);
+ if (__builtin_expect (r == -EINVAL, 0))
+ {
+ /* The requeue functionality is not available. */
+#ifndef __ASSUME_FUTEX_REQUEUE
+ lll_futex_wake (futex, 1);
+#endif
+ }
+ else if (r != 0)
+ {
+ /* We always have to make the syscall if requeue actually
+ moved a thread. */
+ lll_mutex_unlock_force (cond->__data.__lock);
+
+ return 0;
+ }
}
/* We are done. */
/* We have one new user of the condvar. */
++cond->__data.__total_seq;
+ /* Remember the mutex we are using here. If there is already a
+ different address store this is a bad user bug. */
+ cond->__data.__mutex = mutex;
+
/* Prepare structure passed to cancellation handler. */
cbuffer.cond = cond;
cbuffer.mutex = mutex;
lll_mutex_unlock (cond->__data.__lock);
/* Enable asynchronous cancellation. Required by the standard. */
- __pthread_enable_asynccancel_2 (&cbuffer.oldtype);
+ cbuffer.oldtype = __pthread_enable_asynccancel ();
/* Wait until woken by signal or broadcast. Note that we
truncate the 'val' value to 32 bits. */
__pthread_cleanup_pop (&buffer, 0);
/* Get the mutex before returning. */
- err = __pthread_mutex_lock_internal (mutex);
+ err = __pthread_mutex_cond_lock (mutex);
return err ?: result;
}
/* Get the mutex before returning unless asynchronous cancellation
is in effect. */
- if (!(cbuffer->oldtype & CANCELTYPE_BITMASK))
- __pthread_mutex_lock_internal (cbuffer->mutex);
+ __pthread_mutex_cond_lock (cbuffer->mutex);
}
/* We have one new user of the condvar. */
++cond->__data.__total_seq;
+ /* Remember the mutex we are using here. If there is already a
+ different address store this is a bad user bug. */
+ cond->__data.__mutex = mutex;
+
/* Prepare structure passed to cancellation handler. */
cbuffer.cond = cond;
cbuffer.mutex = mutex;
lll_mutex_unlock (cond->__data.__lock);
/* Enable asynchronous cancellation. Required by the standard. */
- __pthread_enable_asynccancel_2 (&cbuffer.oldtype);
+ cbuffer.oldtype = __pthread_enable_asynccancel ();
/* Wait until woken by signal or broadcast. Note that we
truncate the 'val' value to 32 bits. */
__pthread_cleanup_pop (&buffer, 0);
/* Get the mutex before returning. */
- return __pthread_mutex_lock_internal (mutex);
+ return __pthread_mutex_cond_lock (mutex);
}
versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
sysdep_routines += register-atfork unregister-atfork libc_pthread_init \
libc_multiple_threads
-libpthread-sysdep_routines += pt-fork
+libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock
gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym lowlevelbarrier.sym
endif
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
+ void *__mutex;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
long long int __align;
#include <sysdep.h>
#include <shlib-compat.h>
#include <lowlevelcond.h>
+#include <kernel-features.h>
#ifdef UP
# define LOCK
#define SYS_futex 240
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
+#define FUTEX_REQUEUE 3
+
+#define EINVAL 22
.text
__pthread_cond_broadcast:
pushl %ebx
+ pushl %esi
+ pushl %edi
- movl 8(%esp), %ebx
+ movl 16(%esp), %ebx
/* Get internal lock. */
movl $1, %eax
3: movl %ecx, (%ebx)
movl %eax, 4(%ebx)
+ /* Get the address of the mutex used. */
+ movl dep_mutex-wakeup_seq(%ebx), %edi
+
/* Unlock. */
LOCK
subl $1, cond_lock-wakeup_seq(%ebx)
jne 7f
/* Wake up all threads. */
-8: movl $FUTEX_WAKE, %ecx
+8: movl $FUTEX_REQUEUE, %ecx
movl $SYS_futex, %eax
- movl $0x7fffffff, %edx
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
ENTER_KERNEL
+#ifndef __ASSUME_FUTEX_REQUEUE
+ cmpl $-EINVAL, %eax
+ je 9f
+10:
+#endif
+
xorl %eax, %eax
+ popl %edi
+ popl %esi
popl %ebx
ret
jne 5f
6: xorl %eax, %eax
+ popl %edi
+ popl %esi
popl %ebx
ret
7: leal cond_lock-wakeup_seq(%ebx), %eax
call __lll_mutex_unlock_wake
jmp 8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9: /* The futex requeue functionality is not available. */
+ movl $0x7fffffff, %edx
+ movl $FUTEX_WAKE, %ecx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ jmp 10b
+#endif
.size __pthread_cond_broadcast, .-__pthread_cond_broadcast
versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
GLIBC_2_3_2)
#include <sysdep.h>
#include <shlib-compat.h>
#include <lowlevelcond.h>
+#include <kernel-features.h>
#ifdef UP
# define LOCK
#define SYS_futex 240
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
+#define FUTEX_REQUEUE 3
+
+#define EINVAL 22
.text
__pthread_cond_signal:
pushl %ebx
+ pushl %esi
+ pushl %edi
- movl 8(%esp), %ebx
+ movl 16(%esp), %edi
/* Get internal lock. */
movl $1, %eax
LOCK
#if cond_lock == 0
- xaddl %eax, (%ebx)
+ xaddl %eax, (%edi)
#else
- xaddl %eax, cond_lock(%ebx)
+ xaddl %eax, cond_lock(%edi)
#endif
testl %eax, %eax
jne 1f
-2: addl $wakeup_seq, %ebx
- movl total_seq+4-wakeup_seq(%ebx), %eax
- movl total_seq-wakeup_seq(%ebx), %ecx
+2: leal wakeup_seq(%edi), %ebx
+ movl total_seq+4(%edi), %eax
+ movl total_seq(%edi), %ecx
cmpl 4(%ebx), %eax
ja 3f
jb 4f
3: addl $1, (%ebx)
adcl $0, 4(%ebx)
- /* Wake up one thread. */
- movl $FUTEX_WAKE, %ecx
+ /* Wake up one thread by moving it to the internal lock futex. */
+ movl $FUTEX_REQUEUE, %ecx
movl $SYS_futex, %eax
- movl %ecx, %edx /* movl $1, %edx */
+ xorl %edx, %edx
+ movl $1, %esi
ENTER_KERNEL
+#ifndef __ASSUME_FUTEX_REQUEUE
+ cmpl $-EINVAL, %eax
+ je 7f
+#endif
+
+ /* If we moved a thread we in any case have to make the syscall. */
+ testl %eax, %eax
+ jne 5f
+
/* Unlock. */
4: LOCK
- subl $1, cond_lock-wakeup_seq(%ebx)
+ subl $1, (%edi)
jne 5f
6: xorl %eax, %eax
+ popl %edi
+ popl %esi
popl %ebx
ret
call __lll_mutex_lock_wait
jmp 2b
- /* Unlock in loop requires waekup. */
+ /* Unlock in loop requires wakeup. */
5:
- leal cond_lock-wakeup_seq(%ebx), %eax
+#if cond_lock == 0
+ movl %edi, %eax
+#else
+ leal cond_lock(%edi), %eax
+#endif
call __lll_mutex_unlock_wake
jmp 6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7: /* The futex requeue functionality is not available. */
+ movl $1, %edx
+ movl $FUTEX_WAKE, %ecx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ jmp 4b
+#endif
.size __pthread_cond_signal, .-__pthread_cond_signal
versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
GLIBC_2_3_2)
testl %eax, %eax
jne 1f
+ /* Store the reference to the mutex. If there is already a
+ different value in there this is a bad user bug. */
+2: movl 24(%esp), %eax
+ movl %eax, dep_mutex(%ebx)
+
/* Unlock the mutex. */
-2: pushl 24(%esp)
+ pushl %eax
.Lpush4:
call __pthread_mutex_unlock_internal
#endif
jne 3f
-4: leal 8(%esp), %eax
- call __pthread_enable_asynccancel_2
+4: call __pthread_enable_asynccancel
+ movl %eax, 8(%esp)
/* Get the current time. */
movl %ebx, %edx
movl %edx, %gs:CLEANUP
/* Trick ahead: (%esp) contains the address of the mutex. */
- call __pthread_mutex_lock_internal
+ call __pthread_mutex_cond_lock
addl $44, %esp
.Laddl:
movl $0x7fffffff, %edx
ENTER_KERNEL
- /* Lock the mutex unless asynchronous cancellation is in effect. */
- testl $2, 8(%esi)
- jne 3f
-
pushl (%esi)
- call __pthread_mutex_lock_internal
+ call __pthread_mutex_cond_lock
popl %eax
-3: popl %esi
+ popl %esi
popl %ebx
ret
.size __condvar_cleanup, .-__condvar_cleanup
testl %eax, %eax
jne 1f
+ /* Store the reference to the mutex. If there is already a
+ different value in there this is a bad user bug. */
+2: movl 20(%esp), %eax
+ movl %eax, dep_mutex(%ebx)
+
/* Unlock the mutex. */
-2: pushl 20(%esp)
+ pushl %eax
.Lpush4:
call __pthread_mutex_unlock_internal
#endif
jne 3f
-4: leal 8(%esp), %eax
- call __pthread_enable_asynccancel_2
+4: call __pthread_enable_asynccancel
+ movl %eax, 8(%esp)
movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */
movl %edi, %edx
movl %edx, %gs:CLEANUP
/* Trick ahead: (%esp) contains the address of the mutex. */
- call __pthread_mutex_lock_internal
+ call __pthread_mutex_cond_lock
addl $36, %esp
.Laddl:
: "memory"); })
+/* Special version of lll_mutex_lock which causes the unlock function to
+ always wakeup waiters. */
+#define lll_mutex_cond_lock(futex) \
+ (void) ({ int ignore1, ignore2; \
+ __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \
+ "testl %0, %0\n\t" \
+ "jne 1f\n\t" \
+ ".subsection 1\n" \
+ "1:\tleal %2, %%ecx\n\t" \
+ "call __lll_mutex_lock_wait\n\t" \
+ "jmp 2f\n\t" \
+ ".previous\n" \
+ "2:" \
+ : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
+ : "0" (2), "2" (futex) \
+ : "memory"); })
+
+
#define lll_mutex_timedlock(futex, timeout) \
({ int result, ignore1, ignore2; \
__asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t" \
#define SYS_futex 1230
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
+#define FUTEX_REQUEUE 3
/* Initializer for compatibility lock. */
#define LLL_MUTEX_LOCK_INITIALIZER (0)
"=r" (__o0), "=r" (__o1), "=r" (__o2), "=r" (__o3) \
: "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
"5" (__o2), "6" (__o3) \
- : lll_futex_clobbers); \
+ : "out4", lll_futex_clobbers); \
__r10 == -1 ? -__r8 : __r8; \
})
"=r" (__o0), "=r" (__o1), "=r" (__o2) \
: "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
"5" (__o2) \
- : "out3", lll_futex_clobbers); \
+ : "out3", "out4", lll_futex_clobbers); \
__r10 == -1 ? -__r8 : __r8; \
})
+
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \
+ ({ \
+ register long int __o0 asm ("out0") = (long int) (futex); \
+ register long int __o1 asm ("out1") = FUTEX_REQUEUE; \
+ register long int __o2 asm ("out2") = (long int) (nr_wake); \
+ register long int __o3 asm ("out3") = (long int) (nr_move); \
+ register long int __o4 asm ("out4") = (long int) (mutex); \
+ register long int __r8 asm ("r8"); \
+ register long int __r10 asm ("r10"); \
+ register long int __r15 asm ("r15") = SYS_futex; \
+ \
+ __asm __volatile ("break %7;;" \
+ : "=r" (__r8), "=r" (__r10), "=r" (__r15), \
+ "=r" (__o0), "=r" (__o1), "=r" (__o2), "r" (__o3), \
+ "=r" (__o4) \
+ : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
+ "5" (__o2), "6" (__o3), "7" (__o4) \
+ : lll_futex_clobbers); \
+ __r8; \
+ })
+
+
static inline int
__attribute__ ((always_inline))
__lll_mutex_trylock (int *futex)
#define lll_mutex_lock(futex) __lll_mutex_lock (&(futex))
+static inline void
+__attribute__ ((always_inline))
+__lll_mutex_cond_lock (int *futex)
+{
+ int val = atomic_exchange_and_add (futex, 2);
+
+ if (__builtin_expect (val != 0, 0))
+ __lll_lock_wait (futex, val);
+}
+#define lll_mutex_cond_lock(futex) __lll_mutex_cond_lock (&(futex))
+
+
extern int __lll_timedlock_wait (int *futex, int val, const struct timespec *)
attribute_hidden;
if (__builtin_expect (val > 1, 0))
lll_futex_wake (futex, 1);
}
-#define lll_mutex_unlock(futex) __lll_mutex_unlock(&(futex))
+#define lll_mutex_unlock(futex) \
+ __lll_mutex_unlock(&(futex))
+
+#define lll_mutex_unlock_force(futex) \
+ lll_futex_wake (&(futex), 1)
#define lll_mutex_islocked(futex) \
(futex != 0)
total_seq offsetof (pthread_cond_t, __data.__total_seq)
wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq)
woken_seq offsetof (pthread_cond_t, __data.__woken_seq)
+dep_mutex offsetof (pthread_cond_t, __data.__mutex)
--- /dev/null
+#include <pthreadP.h>
+
+#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex)
+#define __pthread_mutex_lock __pthread_mutex_cond_lock
+
+#include <nptl/pthread_mutex_lock.c>
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
+ void *__mutex;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
long int __align;
: "0" (1), "2" (futex) \
: "cx", "r11", "cc", "memory"); })
+
+#define lll_mutex_cond_lock(futex) \
+ (void) ({ int ignore1, ignore2; \
+ __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \
+ "testl %0, %0\n\t" \
+ "jne 1f\n\t" \
+ ".subsection 1\n" \
+ "1:\tleaq %2, %%rdi\n\t" \
+ "subq $128, %%rsp\n\t" \
+ "callq __lll_mutex_lock_wait\n\t" \
+ "addq $128, %%rsp\n\t" \
+ "jmp 2f\n\t" \
+ ".previous\n" \
+ "2:" \
+ : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
+ : "0" (2), "2" (futex) \
+ : "cx", "r11", "cc", "memory"); })
+
+
#define lll_mutex_timedlock(futex, timeout) \
({ int result, ignore1, ignore2, ignore3; \
__asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t" \
#include <sysdep.h>
#include <shlib-compat.h>
#include <lowlevelcond.h>
+#include <kernel-features.h>
#ifdef UP
# define LOCK
#define SYS_futex 202
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
+#define FUTEX_REQUEUE 3
+
+#define EINVAL 22
.text
woken up. */
movq %rcx, (%rdi)
+ /* Get the address of the mutex used. */
+ movq dep_mutex-wakeup_seq(%rdi), %r8
+
/* Unlock. */
LOCK
decl cond_lock-wakeup_seq(%rdi)
jne 7f
/* Wake up all threads. */
-8: movq $FUTEX_WAKE, %rsi
+8: movq $FUTEX_REQUEUE, %rsi
movq $SYS_futex, %rax
- movl $0x7fffffff, %edx
+ movl $1, %edx
+ movq $0x7fffffff, %r10
syscall
+#ifndef __ASSUME_FUTEX_REQUEUE
+ cmpq $-EINVAL, %eax
+ je 9f
+10:
+#endif
+
xorl %eax, %eax
retq
callq __lll_mutex_unlock_wake
subq $cond_lock-wakeup_seq, %rdi
jmp 8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9: /* The futex requeue functionality is not available. */
+ movq $0x7fffffff, %rdx
+ movq $FUTEX_WAKE, %rsi
+ movq $SYS_futex, %rax
+ syscall
+ jmp 10b
+#endif
.size __pthread_cond_broadcast, .-__pthread_cond_broadcast
versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
GLIBC_2_3_2)
#include <sysdep.h>
#include <shlib-compat.h>
#include <lowlevelcond.h>
+#include <kernel-features.h>
#ifdef UP
# define LOCK
#define SYS_futex 202
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
+#define FUTEX_REQUEUE 3
+
+#define EINVAL 22
.text
movl $1, %esi
LOCK
#if cond_lock == 0
- xaddl %esi, (%rdi)
+ xaddl %esi, (%r8)
#else
- xaddl %esi, cond_lock(%rdi)
+ xaddl %esi, cond_lock(%r8)
#endif
testl %esi, %esi
jne 1f
-2: addq $wakeup_seq, %rdi
- movq total_seq-wakeup_seq(%rdi), %rcx
+2: leaq wakeup_seq(%r8), %rdi
+ movq total_seq(%r8), %rcx
cmpq (%rdi), %rcx
jbe 4f
/* Wake up one thread. */
movq $FUTEX_WAKE, %rsi
movq $SYS_futex, %rax
- movq %rsi, %rdx /* movl $1, %edx */
+ xorq %rdx, %rdx
+ movq $1, %r10
syscall
+#ifndef __ASSUME_FUTEX_REQUEUE
+ cmpq $-EINVAL, %rax
+ je 7f
+#endif
+
+ /* If we moved a thread we in any case have to make the syscall. */
+ testq %rax, %rax
+ jne 5f
+
/* Unlock. */
4: LOCK
- decl cond_lock-wakeup_seq(%rdi)
+ decl cond_lock(%r8)
jne 5f
6: xorl %eax, %eax
jmp 2b
/* Unlock in loop requires waekup. */
-5: addq $cond_lock-wakeup_seq, %rdi
+5:
+#if cond_lock != 0
+ addq $cond_lock-wakeup_seq, %rdi
+#else
+ movq %r8, %rdi
+#endif
callq __lll_mutex_unlock_wake
jmp 6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7: /* The futex requeue functionality is not available. */
+ movq $1, %rdx
+ movq $FUTEX_WAKE, %esi
+ movq $SYS_futex, %rax
+ syscall
+ jmp 4b
.size __pthread_cond_signal, .-__pthread_cond_signal
versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
GLIBC_2_3_2)
jne 1f
/* Unlock the mutex. */
-2: movq 16(%rsp), %rdi
+2: movq %rdi, %rax
+ movq 16(%rsp), %rdi
+ movq %rdi, dep_mutex(%rax)
callq __pthread_mutex_unlock_internal
testl %eax, %eax
#endif
jne 3f
-4: movq %rsp, %rdi
- callq __pthread_enable_asynccancel_2
+4: callq __pthread_enable_asynccancel
+ movq %rax, (%rsp)
/* Get the current time. */
#ifdef __NR_clock_gettime
movq %rdx, %fs:CLEANUP
movq 16(%rsp), %rdi
- callq __pthread_mutex_lock_internal
+ callq __pthread_mutex_cond_lock
testq %rax, %rax
cmoveq %r14, %rax
movq $SYS_futex, %rax
syscall
- /* Lock the mutex unless asynchronous cancellation is in effect. */
- testq $2, (%r8)
- jne 3f
-
movq 16(%r8), %rdi
- callq __pthread_mutex_lock_internal
+ callq __pthread_mutex_cond_lock
3: retq
.size __condvar_cleanup, .-__condvar_cleanup
jne 1f
/* Unlock the mutex. */
-2: movq 16(%rsp), %rdi
+2: movq %rdi, %rax
+ movq 16(%rsp), %rdi
+ movq %rdi, dep_mutex(%rax)
callq __pthread_mutex_unlock_internal
testl %eax, %eax
#endif
jne 3f
-4: movq %rsp, %rdi
- callq __pthread_enable_asynccancel_2
+4: callq __pthread_enable_asynccancel
+ movq %rax, (%rsp)
movq 8(%rsp), %rdi
xorq %r10, %r10
movq %rdx, %fs:CLEANUP
movq 16(%rsp), %rdi
- callq __pthread_mutex_lock_internal
+ callq __pthread_mutex_cond_lock
14: addq $64, %rsp
.Laddq:
SELF offsetof (tcbhead_t, self)
CLEANUP offsetof (struct pthread, cleanup)
CLEANUP_PREV offsetof (struct _pthread_cleanup_buffer, __prev)
+MUTEX_FUTEX offsetof (pthread_mutex_t, __data.__lock)
#if __LINUX_KERNEL_VERSION >= 132405 && defined __i386__
# define __ASSUME_VSYSCALL 1
#endif
+
+/* The requeue futex functionality was introduced in 2.5.70. */
+#if __LINUX_KERNEL_VERSION >= 132422
+# define __ASSUME_FUTEX_REQUEUE 1
+#endif
projects / platform / upstream / glibc.git / commitdiff