private futexes are available.
* allocatestack.c (allocate_stack): Copy private_futex field from
current thread into the new stack.
* sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Use private
futexes if they are available.
* sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S: Likewise
* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Adjust so that change
in libc-lowlevellock.S allow using private futexes.
* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Likewise.
* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
FUTEX_PRIVATE_FLAG.
* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Use private futexes
if they are available.
* sysdeps/unix/sysv/linux/i386/pthread_once.S: Likewise.
* sysdeps/x86_64/tcb-offsets.sym: Add PRIVATE_FUTEX.
* sysdeps/i386/tcb-offsets.sym: Likewise.
* sysdeps/x86_64/tls.h (tcbhead_t): Add private_futex field.
* sysdeps/i386/tls.h (tcbhead_t): Likewise.
+2007-05-23 Ulrich Drepper <drepper@redhat.com>
+
+ * init.c (__pthread_initialize_minimal_internal): Check whether
+ private futexes are available.
+ * allocatestack.c (allocate_stack): Copy private_futex field from
+ current thread into the new stack.
+ * sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Use private
+ futexes if they are available.
+ * sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S: Likewise
+ * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Adjust so that change
+ in libc-lowlevellock.S allow using private futexes.
+ * sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
+ FUTEX_PRIVATE_FLAG.
+ * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Use private futexes
+ if they are available.
+ * sysdeps/unix/sysv/linux/i386/pthread_once.S: Likewise.
+ * sysdeps/x86_64/tcb-offsets.sym: Add PRIVATE_FUTEX.
+ * sysdeps/i386/tcb-offsets.sym: Likewise.
+ * sysdeps/x86_64/tls.h (tcbhead_t): Add private_futex field.
+ * sysdeps/i386/tls.h (tcbhead_t): Likewise.
+
2007-05-21 Ulrich Drepper <drepper@redhat.com>
* sysdeps/pthread/pthread-functions.h (struct pthread_functions):
__pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
#endif
+#ifndef __ASSUME_PRIVATE_FUTEX
+ /* The thread must know when private futexes are supported. */
+ pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
+ header.private_futex);
+#endif
+
#ifdef NEED_DL_SYSINFO
/* Copy the sysinfo value from the parent. */
THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
__pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
#endif
+#ifndef __ASSUME_PRIVATE_FUTEX
+ /* The thread must know when private futexes are supported. */
+ pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
+ header.private_futex);
+#endif
+
#ifdef NEED_DL_SYSINFO
/* Copy the sysinfo value from the parent. */
THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
#endif
set_robust_list_not_avail ();
+#ifndef __ASSUME_PRIVATE_FUTEX
+ /* Private futexes are always used (at least internally) so that
+ doing the test once this early is beneficial. */
+ {
+ int word;
+ res = INTERNAL_SYSCALL (futex, err, 3, &word,
+ FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1);
+ if (!INTERNAL_SYSCALL_ERROR_P (res, err))
+ THREAD_SETMEM (pd, header.private_futex, FUTEX_PRIVATE_FLAG);
+ }
+#endif
+
/* Set initial thread's stack block from 0 up to __libc_stack_end.
It will be bigger than it actually is, but for unwind.c/pt-longjmp.c
purposes this is good enough. */
CLEANUP_PREV offsetof (struct _pthread_cleanup_buffer, __prev)
MUTEX_FUTEX offsetof (pthread_mutex_t, __data.__lock)
POINTER_GUARD offsetof (tcbhead_t, pointer_guard)
+#ifndef __ASSUME_PRIVATE_FUTEX
+PRIVATE_FUTEX offsetof (tcbhead_t, private_futex)
+#endif
# include <stdlib.h>
# include <list.h>
# include <sysdep.h>
+# include <kernel-features.h>
/* Type for the dtv. */
uintptr_t stack_guard;
uintptr_t pointer_guard;
int gscope_flag;
+#ifndef __ASSUME_PRIVATE_FUTEX
+ int private_futex;
+#endif
} tcbhead_t;
# define TLS_MULTIPLE_THREADS_IN_TCB 1
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
+#include <kernel-features.h>
+
/* In libc.so we do not unconditionally use the lock prefix. Only if
the application is using threads. */
#ifndef UP
0:
#endif
+/* All locks in libc are private. Use the kernel feature if possible. */
+#define FUTEX_PRIVATE_FLAG 128
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define FUTEX_WAIT (0 | FUTEX_PRIVATE_FLAG)
+# define FUTEX_WAKE (1 | FUTEX_PRIVATE_FLAG)
+#else
+# define LOAD_FUTEX_WAIT(reg) \
+ movl %gs:PRIVATE_FUTEX, reg
+# define LOAD_FUTEX_WAKE(reg) \
+ movl %gs:PRIVATE_FUTEX, reg ; \
+ orl $FUTEX_WAKE, reg
+#endif
+
#include "lowlevellock.S"
-/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
#define SYS_gettimeofday __NR_gettimeofday
#define SYS_futex 240
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
+#ifndef FUTEX_WAIT
+# define FUTEX_WAIT 0
+# define FUTEX_WAKE 1
+#endif
+
+#ifndef LOAD_FUTEX_WAIT
+# if FUTEX_WAIT == 0
+# define LOAD_FUTEX_WAIT(reg) \
+ xorl reg, reg
+# else
+# define LOAD_FUTEX_WAIT(reg) \
+ movl $FUTEX_WAIT, reg
+# endif
+# define LOAD_FUTEX_WAKE(reg) \
+ movl $FUTEX_WAKE, reg
+#endif
.globl __lll_mutex_lock_wait
movl $2, %edx
movl %ecx, %ebx
xorl %esi, %esi /* No timeout. */
- xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */
+ LOAD_FUTEX_WAIT (%ecx)
cmpl %edx, %eax /* NB: %edx == 2 */
jne 2f
/* Futex call. */
movl %esp, %esi
- xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */
+ LOAD_FUTEX_WAIT (%ecx)
movl $SYS_futex, %eax
ENTER_KERNEL
movl %eax, %ecx
movl %eax, %ebx
movl $0, (%eax)
- movl $FUTEX_WAKE, %ecx
+ LOAD_FUTEX_WAKE (%ecx)
movl $1, %edx /* Wake one thread. */
movl $SYS_futex, %eax
ENTER_KERNEL
jz 4f
movl %esp, %esi
+ /* XXX The kernel so far uses global futex for the wakeup at
+ all times. */
xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */
movl %ebp, %ebx
movl $SYS_futex, %eax
-/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
#define FUTEX_LOCK_PI 6
#define FUTEX_UNLOCK_PI 7
#define FUTEX_TRYLOCK_PI 8
+#define FUTEX_PRIVATE_FLAG 128
/* Initializer for compatibility lock. */
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
#include <unwindbuf.h>
#include <sysdep.h>
+#include <kernel-features.h>
+
#ifndef UP
# define LOCK lock
# define LOCK
#endif
-#define SYS_futex 240
-#define FUTEX_WAKE 1
+#define SYS_futex 240
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+#define FUTEX_PRIVATE_FLAG 128
.comm __fork_generation, 4, 4
jnz 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
- movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */
+#ifdef __ASSUME_PRIVATE_FUTEX
+ movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
+#else
+# if FUTEX_WAIT == 0
+ movl %gs:PRIVATE_FUTEX, %ecx
+# else
+ movl $FUTEX_WAIT, %ecx
+ orl %gs:PRIVATE_FUTEX, %ecx
+# endif
+#endif
movl $SYS_futex, %eax
ENTER_KERNEL
jmp 6b
/* Wake up all other threads. */
movl $0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+ movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
+#else
movl $FUTEX_WAKE, %ecx
+ orl %gs:PRIVATE_FUTEX, %ecx
+#endif
movl $SYS_futex, %eax
ENTER_KERNEL
movl $0, (%ebx)
movl $0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+ movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
+#else
movl $FUTEX_WAKE, %ecx
+ orl %gs:PRIVATE_FUTEX, %ecx
+#endif
movl $SYS_futex, %eax
ENTER_KERNEL
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
+#include <kernel-features.h>
+
/* In libc.so we do not unconditionally use the lock prefix. Only if
the application is using threads. */
#ifndef UP
0:
#endif
+/* All locks in libc are private. Use the kernel feature if possible. */
+#define FUTEX_PRIVATE_FLAG 128
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define FUTEX_WAIT (0 | FUTEX_PRIVATE_FLAG)
+# define FUTEX_WAKE (1 | FUTEX_PRIVATE_FLAG)
+#else
+# define LOAD_FUTEX_WAIT(reg) \
+ movl %fs:PRIVATE_FUTEX, reg
+# define LOAD_FUTEX_WAKE(reg) \
+ movl %fs:PRIVATE_FUTEX, reg ; \
+ orl $FUTEX_WAKE, reg
+#endif
+
#include "lowlevellock.S"
#endif
#define SYS_futex 202
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
+#ifndef FUTEX_WAIT
+# define FUTEX_WAIT 0
+# define FUTEX_WAKE 1
+#endif
+
+#ifndef LOAD_FUTEX_WAIT
+# if FUTEX_WAIT == 0
+# define LOAD_FUTEX_WAIT(reg) \
+ xorl reg, reg
+# else
+# define LOAD_FUTEX_WAIT(reg) \
+ movl $FUTEX_WAIT, reg
+# endif
+# define LOAD_FUTEX_WAKE(reg) \
+ movl $FUTEX_WAKE, reg
+#endif
+
/* For the calculation see asm/vsyscall.h. */
#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000
cfi_offset(%rdx, -24)
xorq %r10, %r10 /* No timeout. */
movl $2, %edx
-#if FUTEX_WAIT == 0
- xorl %esi, %esi
-#else
- movl $FUTEX_WAIT, %esi
-#endif
+ LOAD_FUTEX_WAIT (%esi)
cmpl %edx, %eax /* NB: %edx == 2 */
jne 2f
je 8f
movq %rsp, %r10
-#if FUTEX_WAIT == 0
- xorl %esi, %esi
-#else
- movl $FUTEX_WAIT, %esi
-#endif
+ LOAD_FUTEX_WAIT (%esi)
movq %r12, %rdi
movl $SYS_futex, %eax
syscall
cfi_offset(%rdx, -24)
movl $0, (%rdi)
- movl $FUTEX_WAKE, %esi
+ LOAD_FUTEX_WAKE (%esi)
movl $1, %edx /* Wake one thread. */
movl $SYS_futex, %eax
syscall
jz 4f
movq %rsp, %r10
+ /* XXX The kernel so far uses global futex for the wakeup at
+ all times. */
#if FUTEX_WAIT == 0
xorl %esi, %esi
#else
#define FUTEX_LOCK_PI 6
#define FUTEX_UNLOCK_PI 7
#define FUTEX_TRYLOCK_PI 8
+#define FUTEX_PRIVATE_FLAG 128
/* Initializer for compatibility lock. */
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
+#include <kernel-features.h>
+#include <tcb-offsets.h>
+
#ifndef UP
# define LOCK lock
#else
# define LOCK
#endif
-#define SYS_futex 202
-#define FUTEX_WAKE 1
+#define SYS_futex 202
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+#define FUTEX_PRIVATE_FLAG 128
.comm __fork_generation, 4, 4
jnz 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
-#if FUTEX_WAIT == 0
- xorl %esi, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+ movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
#else
+# if FUTEX_WAIT == 0
+ movl %fs:PRIVATE_FUTEX, %esi
+# else
movl $FUTEX_WAIT, %esi
+ orl %fs:PRIVATE_FUTEX, %esi
+# endif
#endif
movl $SYS_futex, %eax
syscall
/* Wake up all other threads. */
movl $0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+ movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
+#else
movl $FUTEX_WAKE, %esi
+ orl %fs:PRIVATE_FUTEX, %esi
+#endif
movl $SYS_futex, %eax
syscall
movl $0, (%rdi)
movl $0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+ movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
+#else
movl $FUTEX_WAKE, %esi
+ orl %fs:PRIVATE_FUTEX, %esi
+#endif
movl $SYS_futex, %eax
syscall
MULTIPLE_THREADS_OFFSET offsetof (tcbhead_t, multiple_threads)
POINTER_GUARD offsetof (tcbhead_t, pointer_guard)
VGETCPU_CACHE_OFFSET offsetof (tcbhead_t, vgetcpu_cache)
+#ifndef __ASSUME_PRIVATE_FUTEX
+PRIVATE_FUTEX offsetof (tcbhead_t, private_futex)
+#endif
# include <stdint.h>
# include <stdlib.h>
# include <sysdep.h>
+# include <kernel-features.h>
/* Type for the dtv. */
uintptr_t stack_guard;
uintptr_t pointer_guard;
unsigned long int vgetcpu_cache[2];
+#ifndef __ASSUME_PRIVATE_FUTEX
+ int private_futex;
+#endif
} tcbhead_t;
#else /* __ASSEMBLER__ */
projects / platform / upstream / glibc.git / commitdiff