+2007-08-14 Kaz Kojima <kkojima@rr.iij4u.or.jp>
+
+ * sysdeps/unix/sysv/linux/sh/pthread_cond_broadcast.S
+ (__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
+ FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
+ Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
+ * sysdeps/unix/sysv/linux/shpthread_cond_signal.S
+ (__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
+ FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
+ Use FUTEX_WAKE_OP.
+ * sysdeps/unix/sysv/linux/sh/pthread_cond_wait.S: Include
+ kernel-features.h and tcb-offsets.h.
+ (__pthread_cond_wait, __condvar_w_cleanup): Pass LLL_PRIVATE to
+ lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is
+ process private.
+ * sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S: Include
+ tcb-offsets.h.
+ (__pthread_cond_timedwait, __condvar_tw_cleanup): Pass LLL_PRIVATE
+ to lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is
+ process private.
+ * sysdeps/unix/sysv/linux/sh/pthread_once.S: Use #ifdef
+ __ASSUME_PRIVATE_FUTEX instead of #if __ASSUME_PRIVATE_FUTEX.
+ * sysdeps/unix/sysv/linux/sh/pthread_rwlock_rdlock.S: Likewise.
+ * sysdeps/unix/sysv/linux/sh/pthread_rwlock_timedrdlock.S: Likewise.
+ * sysdeps/unix/sysv/linux/sh/pthread_rwlock_timedwrlock.S: Likewise.
+ * sysdeps/unix/sysv/linux/sh/pthread_rwlock_unlock.S: Likewise.
+ * sysdeps/unix/sysv/linux/sh/pthread_rwlock_wrlock.S: Likewise.
+
2007-08-14 Jakub Jelinek <jakub@redhat.com>
* sysdeps/unix/sysv/linux/lowlevellock.c: Comment fix.
/* Wake up one thread. */
mov r8, r4
add #cond_futex, r4
- mov #FUTEX_WAKE, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAKE_OP, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAKE_OP, r0
+ or r0, r5
+#endif
+99:
mov #1, r6
mov #0, r7
+ mov r8, r0
+ add #cond_lock, r0
+ mov.l .Lfutexop, r1
mov #SYS_futex, r3
extu.b r3, r3
trapa #0x14
SYSCALL_INST_PAD
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ mov r0, r1
+ mov #-12, r2
+ shad r2, r1
+ not r1, r1
+ tst r1, r1
+ bt 7f
+
+6:
+ mov #0, r0
+ lds.l @r15+, pr
+ rts
+ mov.l @r15+, r8
+
+#ifndef __ASSUME_PRIVATE_FUTEX
+.Lpfoff:
+ .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
+#endif
+ .align 2
+.Lfutexop:
+ .long FUTEX_OP_CLEAR_WAKE_IF_GT_ONE
+
+7:
+ /* r5 should be either FUTEX_WAKE_OP or
+ FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
+ mov #(FUTEX_WAKE ^ FUTEX_WAKE_OP), r0
+ xor r0, r5
+ trapa #0x14
+ SYSCALL_INST_PAD
+
4:
/* Unlock. */
#if cond_lock != 0
DEC (@r8, r2)
#endif
tst r2, r2
- bf 5f
-6:
- mov #0, r0
- lds.l @r15+, pr
- rts
- mov.l @r15+, r8
+ bt 6b
+
+5:
+ /* Unlock in loop requires wakeup. */
+ mov r8, r4
+#if cond_lock != 0
+ add #cond_lock, r4
+#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
+ mov #LLL_SHARED, r5
+99:
+ mov.l .Lwake4, r1
+ bsrf r1
+ extu.b r5, r5
+.Lwake4b:
+ bra 6b
+ nop
1:
/* Initial locking failed. */
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait4, r1
bsrf r1
bra 2b
nop
-5:
- /* Unlock in loop requires wakeup. */
- mov r8, r4
-#if cond_lock != 0
- add #cond_lock, r4
-#endif
- mov #LLL_SHARED, r5
- mov.l .Lwake4, r1
- bsrf r1
- extu.b r5, r5
-.Lwake4b:
- bra 6b
- nop
-
.align 2
.Lwait4:
.long __lll_lock_wait-.Lwait4b
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
-#include "lowlevel-atomic.h"
#include <kernel-features.h>
+#include <tcb-offsets.h>
+#include "lowlevel-atomic.h"
.text
mov r15, r7
add #16, r7
- mov #FUTEX_WAIT, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAIT, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAIT, r0
+ or r0, r5
+#endif
+99:
mov.l @(8,r15), r6
mov r8, r4
add #cond_futex, r4
mov r8, r4
add #cond_nwaiters, r4
- mov #FUTEX_WAKE, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAKE, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAKE, r0
+ or r0, r5
+#endif
+99:
mov #1, r6
mov #0, r7
mov #SYS_futex, r3
rts
mov.l @r15+, r8
+#ifndef __ASSUME_PRIVATE_FUTEX
+.Lpfoff:
+ .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
+#endif
.L1k:
.word 1000
.align 2
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait2, r1
bsrf r1
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lmwait2, r1
bsrf r1
extu.b r5, r5
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait3, r1
bsrf r1
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lmwait3, r1
bsrf r1
extu.b r5, r5
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lmwait4, r1
bsrf r1
extu.b r5, r5
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait5, r1
bsrf r1
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lmwait5, r1
bsrf r1
extu.b r5, r5
#include <shlib-compat.h>
#include <lowlevellock.h>
#include <lowlevelcond.h>
+#include <tcb-offsets.h>
+#include <kernel-features.h>
#include "lowlevel-atomic.h"
.text
mov.l r0, @r15
mov #0, r7
- mov #FUTEX_WAIT, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAIT, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff0, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAIT, r0
+ or r0, r5
+#endif
+99:
mov.l @(8,r15), r6
mov r8, r4
add #cond_futex, r4
mov r8, r4
add #cond_nwaiters, r4
- mov #FUTEX_WAKE, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAKE, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff0, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAKE, r0
+ or r0, r5
+#endif
+99:
mov #1, r6
mov #0, r7
mov #SYS_futex, r3
rts
mov.l @r15+, r8
+#ifndef __ASSUME_PRIVATE_FUTEX
+.Lpfoff0:
+ .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
+#endif
.align 2
.Lmunlock0:
.long __pthread_mutex_unlock_usercnt-.Lmunlock0b
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait0, r1
bsrf r1
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lwake0, r1
bsrf r1
extu.b r5, r5
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait1, r1
bsrf r1
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lwake1, r1
bsrf r1
extu.b r5, r5
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lwake2, r1
bsrf r1
extu.b r5, r5
#if cond_lock != 0
add #cond_lock, r5
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r6
mov #LLL_SHARED, r6
+99:
extu.b r6, r6
mov.l .Lwait3, r1
bsrf r1
mov r8, r4
add #cond_nwaiters, r4
- mov #FUTEX_WAKE, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAKE, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff1, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAKE, r0
+ or r0, r5
+#endif
+99:
mov #1, r6
mov #0, r7
mov #SYS_futex, r3
#if cond_lock != 0
add #cond_lock, r4
#endif
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bf/s 99f
+ mov #LLL_PRIVATE, r5
mov #LLL_SHARED, r5
+99:
mov.l .Lwake3, r1
bsrf r1
extu.b r5, r5
bf/s 5f
mov r8, r4
add #cond_futex, r4
- mov #FUTEX_WAKE, r5
+ mov.l @(dep_mutex,r8), r0
+ cmp/eq #-1, r0
+ bt/s 99f
+ mov #FUTEX_WAKE, r5
+#ifdef __ASSUME_PRIVATE_FUTEX
+ mov #(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), r5
+ extu.b r5, r5
+#else
+ stc gbr, r1
+ mov.w .Lpfoff1, r2
+ add r2, r1
+ mov.l @r1, r5
+ mov #FUTEX_WAKE, r0
+ or r0, r5
+#endif
+99:
mov #-1, r6
shlr r6 /* r6 = 0x7fffffff */
mov #0, r7
mov r11, r4
sleep
+#ifndef __ASSUME_PRIVATE_FUTEX
+.Lpfoff1:
+ .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
+#endif
.align 2
.Lwait3:
.long __lll_lock_wait-.Lwait3b
bf 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAIT), r5
extu.b r5, r5
#else
INC (@r9, r2)
/* Wake up all other threads. */
mov r9, r4
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAKE), r5
extu.b r5, r5
#else
mov #0, r7
mov.l r7, @r9
mov r9, r4
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAKE), r5
#else
stc gbr, r1
sleep
cfi_endproc
-#if !__ASSUME_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
.Lpfoff:
.word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
#endif
tst r2, r2
bf 10f
11:
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #PSHARED, r0
mov.b @(r0,r8), r5
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAIT), r0
rts
mov r3, r0
-#if !__ASSUME_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
.Lpfoff:
.word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
#endif
/* Futex call. */
mov r15, r7
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #PSHARED, r0
mov.b @(r0,r8), r5
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAIT), r0
rts
mov r3, r0
-#if !__ASSUME_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
.Lpfoff:
.word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
#endif
/* Futex call. */
mov r15, r7
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #PSHARED, r0
mov.b @(r0,r8), r5
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAIT), r0
rts
mov r3, r0
-#if !__ASSUME_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
.Lpfoff:
.word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
#endif
bf 7f
8:
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #PSHARED, r0
mov.b @(r0,r8), r5
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAKE), r0
bra 8b
mov.l @r15+, r4
-#if !__ASSUME_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
.Lpfoff:
.word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
#endif
11:
mov r8, r4
add #WRITERS_WAKEUP, r4
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
mov #PSHARED, r0
mov.b @(r0,r8), r5
mov #(FUTEX_PRIVATE_FLAG|FUTEX_WAIT), r0
bra 7b
mov #0, r3
-#if !__ASSUME_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
.Lpfoff:
.word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE
#endif