Extend x86-64 sem_timedwait to use futex syscall with absolute timeout.

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 867493c..98d3a4d 100644 (file)
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,5 +1,9 @@
 2009-07-17  Ulrich Drepper  <drepper@redhat.com>
 
+       * sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S (sem_timedwait):
+       If possible use FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME to directly
+       use absolute timeout.
+
        * sysdeps/unix/sysv/linux/x86_64/sem_wait.S (sem_wait): Optimize
        handling of uncontested semaphore.
 

diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
index c96b373..9576283 100644 (file)
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
@@ -18,6 +18,7 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <kernel-features.h>
 #include <lowlevellock.h>
 #include <shlib-compat.h>
 #include <pthread-errnos.h>
@@ -73,8 +74,13 @@ sem_timedwait:
        pushq   %r14
        cfi_adjust_cfa_offset(8)
        cfi_rel_offset(%r14, 0)
-       subq    $24, %rsp
-       cfi_adjust_cfa_offset(24)
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define STACKFRAME 8
+#else
+# define STACKFRAME 24
+#endif
+       subq    $STACKFRAME, %rsp
+       cfi_adjust_cfa_offset(STACKFRAME)
 
        movq    %rdi, %r12
        movq    %rsi, %r13
@@ -87,67 +93,50 @@ sem_timedwait:
        LOCK
        addq    $1, NWAITERS(%r12)
 
-7:     xorl    %esi, %esi
-       movq    %rsp, %rdi
-       movq    $VSYSCALL_ADDR_vgettimeofday, %rax
-       callq   *%rax
-
-       /* Compute relative timeout.  */
-       movq    8(%rsp), %rax
-       movl    $1000, %edi
-       mul     %rdi            /* Milli seconds to nano seconds.  */
-       movq    (%r13), %rdi
-       movq    8(%r13), %rsi
-       subq    (%rsp), %rdi
-       subq    %rax, %rsi
-       jns     5f
-       addq    $1000000000, %rsi
-       decq    %rdi
-5:     testq   %rdi, %rdi
-       movl    $ETIMEDOUT, %r14d
-       js      6f              /* Time is already up.  */
-
-       movq    %rdi, (%rsp)    /* Store relative timeout.  */
-       movq    %rsi, 8(%rsp)
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+       cmpl    $0, __have_futex_clock_realtime(%rip)
+#  else
+       cmpl    $0, __have_futex_clock_realtime
+#  endif
+       je      .Lreltmo
+#endif
 
 .LcleanupSTART:
-       call    __pthread_enable_asynccancel
-       movl    %eax, 16(%rsp)
+13:    call    __pthread_enable_asynccancel
+       movl    %eax, (%rsp)
 
-       movq    %rsp, %r10
+       movq    %r13, %r10
 #if VALUE == 0
        movq    %r12, %rdi
 #else
        leaq    VALUE(%r12), %rdi
 #endif
-#if FUTEX_WAIT == 0
-       movl    PRIVATE(%rdi), %esi
-#else
-       movl    $FUTEX_WAIT, %esi
+       movl    $0xffffffff, %r9d
+       movl    $FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi
        orl     PRIVATE(%rdi), %esi
-#endif
        movl    $SYS_futex, %eax
        xorl    %edx, %edx
        syscall
        movq    %rax, %r14
 
-       movl    16(%rsp), %edi
+       movl    (%rsp), %edi
        call    __pthread_disable_asynccancel
 .LcleanupEND:
 
        testq   %r14, %r14
-       je      9f
+       je      11f
        cmpq    $-EWOULDBLOCK, %r14
        jne     3f
 
-9:
+11:
 #if VALUE == 0
        movl    (%r12), %eax
 #else
        movl    VALUE(%r12), %eax
 #endif
-8:     testl   %eax, %eax
-       je      7b
+14:    testl   %eax, %eax
+       je      13b
 
        leaq    -1(%rax), %rcx
        LOCK
@@ -156,15 +145,15 @@ sem_timedwait:
 #else
        cmpxchgl %ecx, VALUE(%r12)
 #endif
-       jne     8b
+       jne     14b
 
-       xorl    %eax, %eax
+10:    xorl    %eax, %eax
 
-10:    LOCK
+15:    LOCK
        subq    $1, NWAITERS(%r12)
 
-       addq    $24, %rsp
-       cfi_adjust_cfa_offset(-24)
+       addq    $STACKFRAME, %rsp
+       cfi_adjust_cfa_offset(-STACKFRAME)
        popq    %r14
        cfi_adjust_cfa_offset(-8)
        cfi_restore(%r14)
@@ -176,10 +165,10 @@ sem_timedwait:
        cfi_restore(%r12)
        retq
 
-       cfi_adjust_cfa_offset(3 * 8 + 24)
-       cfi_rel_offset(%r12, 24 + 2 * 8)
-       cfi_rel_offset(%r13, 24 + 1 * 8)
-       cfi_rel_offset(%r14, 24)
+       cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
+       cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
+       cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
+       cfi_rel_offset(%r14, STACKFRAME)
 3:     negq    %r14
 6:
 #if USE___THREAD
@@ -191,7 +180,82 @@ sem_timedwait:
 #endif
 
        orl     $-1, %eax
+       jmp     15b
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+.Lreltmo:
+7:     xorl    %esi, %esi
+       movq    %rsp, %rdi
+       movq    $VSYSCALL_ADDR_vgettimeofday, %rax
+       callq   *%rax
+
+       /* Compute relative timeout.  */
+       movq    8(%rsp), %rax
+       movl    $1000, %edi
+       mul     %rdi            /* Milli seconds to nano seconds.  */
+       movq    (%r13), %rdi
+       movq    8(%r13), %rsi
+       subq    (%rsp), %rdi
+       subq    %rax, %rsi
+       jns     5f
+       addq    $1000000000, %rsi
+       decq    %rdi
+5:     testq   %rdi, %rdi
+       movl    $ETIMEDOUT, %r14d
+       js      6b              /* Time is already up.  */
+
+       movq    %rdi, (%rsp)    /* Store relative timeout.  */
+       movq    %rsi, 8(%rsp)
+
+.LcleanupSTART2:
+       call    __pthread_enable_asynccancel
+       movl    %eax, 16(%rsp)
+
+       movq    %rsp, %r10
+# if VALUE == 0
+       movq    %r12, %rdi
+# else
+       leaq    VALUE(%r12), %rdi
+# endif
+# if FUTEX_WAIT == 0
+       movl    PRIVATE(%rdi), %esi
+# else
+       movl    $FUTEX_WAIT, %esi
+       orl     PRIVATE(%rdi), %esi
+# endif
+       movl    $SYS_futex, %eax
+       xorl    %edx, %edx
+       syscall
+       movq    %rax, %r14
+
+       movl    16(%rsp), %edi
+       call    __pthread_disable_asynccancel
+.LcleanupEND2:
+
+       testq   %r14, %r14
+       je      9f
+       cmpq    $-EWOULDBLOCK, %r14
+       jne     3b
+
+9:
+# if VALUE == 0
+       movl    (%r12), %eax
+# else
+       movl    VALUE(%r12), %eax
+# endif
+8:     testl   %eax, %eax
+       je      7b
+
+       leaq    -1(%rax), %rcx
+       LOCK
+# if VALUE == 0
+       cmpxchgl %ecx, (%r12)
+# else
+       cmpxchgl %ecx, VALUE(%r12)
+# endif
+       jne     8b
        jmp     10b
+#endif
        .size   sem_timedwait,.-sem_timedwait
 
 
@@ -219,6 +283,12 @@ sem_timedwait_cleanup:
        .uleb128 .LcleanupEND-.LcleanupSTART
        .uleb128 sem_timedwait_cleanup-.LSTARTCODE
        .uleb128  0
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+       .uleb128 .LcleanupSTART2-.LSTARTCODE
+       .uleb128 .LcleanupEND2-.LcleanupSTART2
+       .uleb128 sem_timedwait_cleanup-.LSTARTCODE
+       .uleb128  0
+#endif
        .uleb128 .LcallUR-.LSTARTCODE
        .uleb128 .LENDCODE-.LcallUR
        .uleb128 0