From 12751058515860ed43c8f874ebcb2097b323736a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 11 Sep 2009 10:28:34 +0200 Subject: [PATCH] [S390] atomic ops: add effecient atomic64 support for 31 bit Use compare double and swap to implement efficient atomic64 ops for 31 bit. Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- arch/s390/Kconfig | 1 - arch/s390/include/asm/atomic.h | 164 +++++++++++++++++++++++++++++++---------- 2 files changed, 127 insertions(+), 38 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 2ae5d72..47836b9 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -95,7 +95,6 @@ config S390 select HAVE_ARCH_TRACEHOOK select INIT_ALL_POSSIBLE select HAVE_PERF_COUNTERS - select GENERIC_ATOMIC64 if !64BIT config SCHED_OMIT_FRAME_POINTER bool diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index c7d0abf..b491d5e 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -1,28 +1,20 @@ #ifndef __ARCH_S390_ATOMIC__ #define __ARCH_S390_ATOMIC__ -#include -#include - /* - * include/asm-s390/atomic.h + * Copyright 1999,2009 IBM Corp. + * Author(s): Martin Schwidefsky , + * Denis Joseph Barrow, + * Arnd Bergmann , * - * S390 version - * Copyright (C) 1999-2005 IBM Deutschland Entwicklung GmbH, IBM Corporation - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), - * Denis Joseph Barrow, - * Arnd Bergmann (arndb@de.ibm.com) - * - * Derived from "include/asm-i386/bitops.h" - * Copyright (C) 1992, Linus Torvalds + * Atomic operations that C can't guarantee us. + * Useful for resource counting etc. + * s390 uses 'Compare And Swap' for atomicity in SMP enviroment. * */ -/* - * Atomic operations that C can't guarantee us. Useful for - * resource counting etc.. - * S390 uses 'Compare And Swap' for atomicity in SMP enviroment - */ +#include +#include #define ATOMIC_INIT(i) { (i) } @@ -146,9 +138,10 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) #undef __CS_LOOP -#ifdef __s390x__ #define ATOMIC64_INIT(i) { (i) } +#ifdef CONFIG_64BIT + #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2) #define __CSG_LOOP(ptr, op_val, op_string) ({ \ @@ -202,21 +195,11 @@ static __inline__ long long atomic64_add_return(long long i, atomic64_t * v) { return __CSG_LOOP(v, i, "agr"); } -#define atomic64_add(_i, _v) atomic64_add_return(_i, _v) -#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) -#define atomic64_inc(_v) atomic64_add_return(1, _v) -#define atomic64_inc_return(_v) atomic64_add_return(1, _v) -#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) static __inline__ long long atomic64_sub_return(long long i, atomic64_t * v) { return __CSG_LOOP(v, i, "sgr"); } -#define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v) -#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) -#define atomic64_dec(_v) atomic64_sub_return(1, _v) -#define atomic64_dec_return(_v) atomic64_sub_return(1, _v) -#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) static __inline__ void atomic64_clear_mask(unsigned long mask, atomic64_t * v) { @@ -249,6 +232,111 @@ static __inline__ long long atomic64_cmpxchg(atomic64_t *v, return old; } +#undef __CSG_LOOP + +#else /* CONFIG_64BIT */ + +typedef struct { + long long counter; +} atomic64_t; + +static inline long long atomic64_read(const atomic64_t *v) +{ + register_pair rp; + + asm volatile( + " lm %0,%N0,0(%1)" + : "=&d" (rp) + : "a" (&v->counter), "m" (v->counter) + ); + return rp.pair; +} + +static inline void atomic64_set(atomic64_t *v, long long i) +{ + register_pair rp = {.pair = i}; + + asm volatile( + " stm %1,%N1,0(%2)" + : "=m" (v->counter) + : "d" (rp), "a" (&v->counter) + ); +} + +static inline long long atomic64_xchg(atomic64_t *v, long long new) +{ + register_pair rp_new = {.pair = new}; + register_pair rp_old; + + asm volatile( + " lm %0,%N0,0(%2)\n" + "0: cds %0,%3,0(%2)\n" + " jl 0b\n" + : "=&d" (rp_old), "+m" (v->counter) + : "a" (&v->counter), "d" (rp_new) + : "cc"); + return rp_old.pair; +} + +static inline long long atomic64_cmpxchg(atomic64_t *v, + long long old, long long new) +{ + register_pair rp_old = {.pair = old}; + register_pair rp_new = {.pair = new}; + + asm volatile( + " cds %0,%3,0(%2)" + : "+&d" (rp_old), "+m" (v->counter) + : "a" (&v->counter), "d" (rp_new) + : "cc"); + return rp_old.pair; +} + + +static inline long long atomic64_add_return(long long i, atomic64_t *v) +{ + long long old, new; + + do { + old = atomic64_read(v); + new = old + i; + } while (atomic64_cmpxchg(v, old, new) != old); + return new; +} + +static inline long long atomic64_sub_return(long long i, atomic64_t *v) +{ + long long old, new; + + do { + old = atomic64_read(v); + new = old - i; + } while (atomic64_cmpxchg(v, old, new) != old); + return new; +} + +static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v) +{ + long long old, new; + + do { + old = atomic64_read(v); + new = old | mask; + } while (atomic64_cmpxchg(v, old, new) != old); +} + +static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v) +{ + long long old, new; + + do { + old = atomic64_read(v); + new = old & mask; + } while (atomic64_cmpxchg(v, old, new) != old); +} + +#endif /* CONFIG_64BIT */ + static __inline__ int atomic64_add_unless(atomic64_t *v, long long a, long long u) { @@ -265,15 +353,17 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, return c != u; } -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - -#undef __CSG_LOOP - -#else /* __s390x__ */ - -#include - -#endif /* __s390x__ */ +#define atomic64_add(_i, _v) atomic64_add_return(_i, _v) +#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) +#define atomic64_inc(_v) atomic64_add_return(1, _v) +#define atomic64_inc_return(_v) atomic64_add_return(1, _v) +#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) +#define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v) +#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) +#define atomic64_dec(_v) atomic64_sub_return(1, _v) +#define atomic64_dec_return(_v) atomic64_sub_return(1, _v) +#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #define smp_mb__before_atomic_dec() smp_mb() #define smp_mb__after_atomic_dec() smp_mb() -- 2.7.4