From 9a9f03225e27ce7e0d34557fb87868b9fdab89ce Mon Sep 17 00:00:00 2001 From: Ivan Maidanski Date: Fri, 4 Jan 2013 00:09:50 +0400 Subject: [PATCH] Add AO_load primitives generalization based on CAS * src/atomic_ops/generalize-small.template (AO_XSIZE_load_acquire, AO_XSIZE_load, AO_XSIZE_load_read, AO_XSIZE_load_acquire_read, AO_XSIZE_load_full, AO_XSIZE_load_dd_acquire_read, AO_XSIZE_store_release, AO_XSIZE_store, AO_XSIZE_store_write, AO_XSIZE_store_release_write, AO_XSIZE_store_full): Move primitive down to be after CAS defintion section. * src/atomic_ops/generalize-small.template (AO_XSIZE_load_read, AO_XSIZE_load_full, AO_XSIZE_load_acquire, AO_XSIZE_load): Add generalization template primitive based on CAS. * src/atomic_ops/generalize-small.h: Regenerate. --- src/atomic_ops/generalize-small.h | 1186 +++++++++++++++++++----------- src/atomic_ops/generalize-small.template | 345 +++++---- 2 files changed, 948 insertions(+), 583 deletions(-) diff --git a/src/atomic_ops/generalize-small.h b/src/atomic_ops/generalize-small.h index 30339de..2bdac01 100644 --- a/src/atomic_ops/generalize-small.h +++ b/src/atomic_ops/generalize-small.h @@ -20,142 +20,6 @@ * SOFTWARE. */ -/* char_load */ -#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire) -# define AO_char_load_acquire(addr) AO_char_load_full(addr) -# define AO_HAVE_char_load_acquire -#endif - -#if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load) -# define AO_char_load(addr) AO_char_load_acquire(addr) -# define AO_HAVE_char_load -#endif - -#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read) -# define AO_char_load_read(addr) AO_char_load_full(addr) -# define AO_HAVE_char_load_read -#endif - -#if !defined(AO_HAVE_char_load_acquire_read) \ - && defined(AO_HAVE_char_load_acquire) -# define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr) -# define AO_HAVE_char_load_acquire_read -#endif - -#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_char_load_acquire) - AO_INLINE unsigned/**/char - AO_char_load_acquire(const volatile unsigned/**/char *addr) - { - unsigned/**/char result = AO_char_load(addr); - - /* Acquire barrier would be useless, since the load could be delayed */ - /* beyond it. */ - AO_nop_full(); - return result; - } -# define AO_HAVE_char_load_acquire -#endif - -#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \ - && !defined(AO_HAVE_char_load_read) - AO_INLINE unsigned/**/char - AO_char_load_read(const volatile unsigned/**/char *addr) - { - unsigned/**/char result = AO_char_load(addr); - - /* Acquire barrier would be useless, since the load could be delayed */ - /* beyond it. */ - AO_nop_read(); - return result; - } -# define AO_HAVE_char_load_read -#endif - -#if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_char_load_full) -# define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr)) -# define AO_HAVE_char_load_full -#endif - -#if !defined(AO_HAVE_char_load_acquire_read) \ - && defined(AO_HAVE_char_load_read) -# define AO_char_load_acquire_read(addr) AO_char_load_read(addr) -# define AO_HAVE_char_load_acquire_read -#endif - -#if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load) -# define AO_char_load(addr) AO_char_load_acquire_read(addr) -# define AO_HAVE_char_load -#endif - -#ifdef AO_NO_DD_ORDERING -# if defined(AO_HAVE_char_load_acquire_read) -# define AO_char_load_dd_acquire_read(addr) AO_char_load_acquire_read(addr) -# define AO_HAVE_char_load_dd_acquire_read -# endif -#else -# if defined(AO_HAVE_char_load) -# define AO_char_load_dd_acquire_read(addr) AO_char_load(addr) -# define AO_HAVE_char_load_dd_acquire_read -# endif -#endif /* !AO_NO_DD_ORDERING */ - -/* char_store */ -#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release) -# define AO_char_store_release(addr,val) AO_char_store_full(addr,val) -# define AO_HAVE_char_store_release -#endif - -#if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store) -# define AO_char_store(addr, val) AO_char_store_release(addr,val) -# define AO_HAVE_char_store -#endif - -#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write) -# define AO_char_store_write(addr,val) AO_char_store_full(addr,val) -# define AO_HAVE_char_store_write -#endif - -#if defined(AO_HAVE_char_store_release) \ - && !defined(AO_HAVE_char_store_release_write) -# define AO_char_store_release_write(addr, val) \ - AO_char_store_release(addr,val) -# define AO_HAVE_char_store_release_write -#endif - -#if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store) -# define AO_char_store(addr, val) AO_char_store_write(addr,val) -# define AO_HAVE_char_store -#endif - -#if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_char_store_release) -# define AO_char_store_release(addr,val) \ - (AO_nop_full(), AO_char_store(addr,val)) -# define AO_HAVE_char_store_release -#endif - -#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_char_store) \ - && !defined(AO_HAVE_char_store_write) -# define AO_char_store_write(addr, val) \ - (AO_nop_write(), AO_char_store(addr,val)) -# define AO_HAVE_char_store_write -#endif - -#if defined(AO_HAVE_char_store_write) \ - && !defined(AO_HAVE_char_store_release_write) -# define AO_char_store_release_write(addr, val) AO_char_store_write(addr,val) -# define AO_HAVE_char_store_release_write -#endif - -#if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_char_store_full) -# define AO_char_store_full(addr, val) \ - (AO_char_store_release(addr, val), AO_nop_full()) -# define AO_HAVE_char_store_full -#endif - /* char_fetch_compare_and_swap */ #if defined(AO_HAVE_char_fetch_compare_and_swap) \ && defined(AO_HAVE_nop_full) \ @@ -388,163 +252,236 @@ # define AO_HAVE_char_compare_and_swap_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ -/* - * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -/* short_load */ -#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire) -# define AO_short_load_acquire(addr) AO_short_load_full(addr) -# define AO_HAVE_short_load_acquire +/* char_load */ +#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire) +# define AO_char_load_acquire(addr) AO_char_load_full(addr) +# define AO_HAVE_char_load_acquire #endif -#if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load) -# define AO_short_load(addr) AO_short_load_acquire(addr) -# define AO_HAVE_short_load +#if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load) +# define AO_char_load(addr) AO_char_load_acquire(addr) +# define AO_HAVE_char_load #endif -#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read) -# define AO_short_load_read(addr) AO_short_load_full(addr) -# define AO_HAVE_short_load_read +#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read) +# define AO_char_load_read(addr) AO_char_load_full(addr) +# define AO_HAVE_char_load_read #endif -#if !defined(AO_HAVE_short_load_acquire_read) \ - && defined(AO_HAVE_short_load_acquire) -# define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr) -# define AO_HAVE_short_load_acquire_read +#if !defined(AO_HAVE_char_load_acquire_read) \ + && defined(AO_HAVE_char_load_acquire) +# define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr) +# define AO_HAVE_char_load_acquire_read #endif -#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_short_load_acquire) - AO_INLINE unsigned/**/short - AO_short_load_acquire(const volatile unsigned/**/short *addr) +#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_char_load_acquire) + AO_INLINE unsigned/**/char + AO_char_load_acquire(const volatile unsigned/**/char *addr) { - unsigned/**/short result = AO_short_load(addr); + unsigned/**/char result = AO_char_load(addr); /* Acquire barrier would be useless, since the load could be delayed */ /* beyond it. */ AO_nop_full(); return result; } -# define AO_HAVE_short_load_acquire +# define AO_HAVE_char_load_acquire #endif -#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \ - && !defined(AO_HAVE_short_load_read) - AO_INLINE unsigned/**/short - AO_short_load_read(const volatile unsigned/**/short *addr) +#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \ + && !defined(AO_HAVE_char_load_read) + AO_INLINE unsigned/**/char + AO_char_load_read(const volatile unsigned/**/char *addr) { - unsigned/**/short result = AO_short_load(addr); + unsigned/**/char result = AO_char_load(addr); /* Acquire barrier would be useless, since the load could be delayed */ /* beyond it. */ AO_nop_read(); return result; } -# define AO_HAVE_short_load_read +# define AO_HAVE_char_load_read #endif -#if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_short_load_full) -# define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr)) -# define AO_HAVE_short_load_full +#if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_char_load_full) +# define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr)) +# define AO_HAVE_char_load_full #endif -#if !defined(AO_HAVE_short_load_acquire_read) \ - && defined(AO_HAVE_short_load_read) -# define AO_short_load_acquire_read(addr) AO_short_load_read(addr) -# define AO_HAVE_short_load_acquire_read -#endif +#if defined(AO_HAVE_char_compare_and_swap_read) \ + && !defined(AO_HAVE_char_load_read) +# define AO_char_CAS_BASED_LOAD_READ + AO_INLINE unsigned/**/char + AO_char_load_read(const volatile unsigned/**/char *addr) + { + unsigned/**/char result; -#if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load) -# define AO_short_load(addr) AO_short_load_acquire_read(addr) -# define AO_HAVE_short_load + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_read( + (volatile unsigned/**/char *)addr, + result, result))); + return result; + } +# define AO_HAVE_char_load_read #endif -#ifdef AO_NO_DD_ORDERING -# if defined(AO_HAVE_short_load_acquire_read) -# define AO_short_load_dd_acquire_read(addr) AO_short_load_acquire_read(addr) -# define AO_HAVE_short_load_dd_acquire_read -# endif -#else -# if defined(AO_HAVE_short_load) -# define AO_short_load_dd_acquire_read(addr) AO_short_load(addr) -# define AO_HAVE_short_load_dd_acquire_read -# endif -#endif /* !AO_NO_DD_ORDERING */ - -/* short_store */ -#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release) -# define AO_short_store_release(addr,val) AO_short_store_full(addr,val) -# define AO_HAVE_short_store_release +#if !defined(AO_HAVE_char_load_acquire_read) \ + && defined(AO_HAVE_char_load_read) +# define AO_char_load_acquire_read(addr) AO_char_load_read(addr) +# define AO_HAVE_char_load_acquire_read #endif -#if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store) -# define AO_short_store(addr, val) AO_short_store_release(addr,val) -# define AO_HAVE_short_store +#if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load) \ + && (!defined(AO_char_CAS_BASED_LOAD_READ) \ + || !defined(AO_HAVE_char_compare_and_swap)) +# define AO_char_load(addr) AO_char_load_acquire_read(addr) +# define AO_HAVE_char_load #endif -#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write) -# define AO_short_store_write(addr,val) AO_short_store_full(addr,val) -# define AO_HAVE_short_store_write -#endif +#if defined(AO_HAVE_char_compare_and_swap_full) \ + && !defined(AO_HAVE_char_load_full) + AO_INLINE unsigned/**/char + AO_char_load_full(const volatile unsigned/**/char *addr) + { + unsigned/**/char result; -#if defined(AO_HAVE_short_store_release) \ - && !defined(AO_HAVE_short_store_release_write) -# define AO_short_store_release_write(addr, val) \ - AO_short_store_release(addr,val) -# define AO_HAVE_short_store_release_write + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full( + (volatile unsigned/**/char *)addr, + result, result))); + return result; + } +# define AO_HAVE_char_load_full #endif -#if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store) -# define AO_short_store(addr, val) AO_short_store_write(addr,val) -# define AO_HAVE_short_store +#if defined(AO_HAVE_char_compare_and_swap_acquire) \ + && !defined(AO_HAVE_char_load_acquire) + AO_INLINE unsigned/**/char + AO_char_load_acquire(const volatile unsigned/**/char *addr) + { + unsigned/**/char result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire( + (volatile unsigned/**/char *)addr, + result, result))); + return result; + } +# define AO_HAVE_char_load_acquire #endif -#if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_short_store_release) -# define AO_short_store_release(addr,val) \ - (AO_nop_full(), AO_short_store(addr,val)) -# define AO_HAVE_short_store_release +#if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_load) + AO_INLINE unsigned/**/char + AO_char_load(const volatile unsigned/**/char *addr) + { + unsigned/**/char result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap( + (volatile unsigned/**/char *)addr, + result, result))); + return result; + } +# define AO_HAVE_char_load #endif -#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_short_store) \ - && !defined(AO_HAVE_short_store_write) -# define AO_short_store_write(addr, val) \ - (AO_nop_write(), AO_short_store(addr,val)) -# define AO_HAVE_short_store_write +#ifdef AO_NO_DD_ORDERING +# if defined(AO_HAVE_char_load_acquire_read) +# define AO_char_load_dd_acquire_read(addr) \ + AO_char_load_acquire_read(addr) +# define AO_HAVE_char_load_dd_acquire_read +# endif +#else +# if defined(AO_HAVE_char_load) +# define AO_char_load_dd_acquire_read(addr) AO_char_load(addr) +# define AO_HAVE_char_load_dd_acquire_read +# endif +#endif /* !AO_NO_DD_ORDERING */ + +/* char_store */ +#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release) +# define AO_char_store_release(addr, val) AO_char_store_full(addr, val) +# define AO_HAVE_char_store_release #endif -#if defined(AO_HAVE_short_store_write) \ - && !defined(AO_HAVE_short_store_release_write) -# define AO_short_store_release_write(addr, val) AO_short_store_write(addr,val) -# define AO_HAVE_short_store_release_write +#if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store) +# define AO_char_store(addr, val) AO_char_store_release(addr, val) +# define AO_HAVE_char_store #endif -#if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_short_store_full) -# define AO_short_store_full(addr, val) \ - (AO_short_store_release(addr, val), AO_nop_full()) -# define AO_HAVE_short_store_full +#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write) +# define AO_char_store_write(addr, val) AO_char_store_full(addr, val) +# define AO_HAVE_char_store_write +#endif + +#if defined(AO_HAVE_char_store_release) \ + && !defined(AO_HAVE_char_store_release_write) +# define AO_char_store_release_write(addr, val) \ + AO_char_store_release(addr, val) +# define AO_HAVE_char_store_release_write +#endif + +#if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store) +# define AO_char_store(addr, val) AO_char_store_write(addr, val) +# define AO_HAVE_char_store +#endif + +#if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_char_store_release) +# define AO_char_store_release(addr, val) \ + (AO_nop_full(), AO_char_store(addr, val)) +# define AO_HAVE_char_store_release +#endif + +#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_char_store) \ + && !defined(AO_HAVE_char_store_write) +# define AO_char_store_write(addr, val) \ + (AO_nop_write(), AO_char_store(addr, val)) +# define AO_HAVE_char_store_write +#endif + +#if defined(AO_HAVE_char_store_write) \ + && !defined(AO_HAVE_char_store_release_write) +# define AO_char_store_release_write(addr, val) \ + AO_char_store_write(addr, val) +# define AO_HAVE_char_store_release_write +#endif + +#if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_char_store_full) +# define AO_char_store_full(addr, val) \ + (AO_char_store_release(addr, val), \ + AO_nop_full()) +# define AO_HAVE_char_store_full #endif +/* + * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ /* short_fetch_compare_and_swap */ #if defined(AO_HAVE_short_fetch_compare_and_swap) \ @@ -778,163 +715,236 @@ # define AO_HAVE_short_compare_and_swap_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ -/* - * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -/* int_load */ -#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire) -# define AO_int_load_acquire(addr) AO_int_load_full(addr) -# define AO_HAVE_int_load_acquire +/* short_load */ +#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire) +# define AO_short_load_acquire(addr) AO_short_load_full(addr) +# define AO_HAVE_short_load_acquire #endif -#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load) -# define AO_int_load(addr) AO_int_load_acquire(addr) -# define AO_HAVE_int_load +#if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load) +# define AO_short_load(addr) AO_short_load_acquire(addr) +# define AO_HAVE_short_load #endif -#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read) -# define AO_int_load_read(addr) AO_int_load_full(addr) -# define AO_HAVE_int_load_read +#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read) +# define AO_short_load_read(addr) AO_short_load_full(addr) +# define AO_HAVE_short_load_read #endif -#if !defined(AO_HAVE_int_load_acquire_read) \ - && defined(AO_HAVE_int_load_acquire) -# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr) -# define AO_HAVE_int_load_acquire_read +#if !defined(AO_HAVE_short_load_acquire_read) \ + && defined(AO_HAVE_short_load_acquire) +# define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr) +# define AO_HAVE_short_load_acquire_read #endif -#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_int_load_acquire) - AO_INLINE unsigned - AO_int_load_acquire(const volatile unsigned *addr) +#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_short_load_acquire) + AO_INLINE unsigned/**/short + AO_short_load_acquire(const volatile unsigned/**/short *addr) { - unsigned result = AO_int_load(addr); + unsigned/**/short result = AO_short_load(addr); /* Acquire barrier would be useless, since the load could be delayed */ /* beyond it. */ AO_nop_full(); return result; } -# define AO_HAVE_int_load_acquire +# define AO_HAVE_short_load_acquire #endif -#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \ - && !defined(AO_HAVE_int_load_read) - AO_INLINE unsigned - AO_int_load_read(const volatile unsigned *addr) +#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \ + && !defined(AO_HAVE_short_load_read) + AO_INLINE unsigned/**/short + AO_short_load_read(const volatile unsigned/**/short *addr) { - unsigned result = AO_int_load(addr); + unsigned/**/short result = AO_short_load(addr); /* Acquire barrier would be useless, since the load could be delayed */ /* beyond it. */ AO_nop_read(); return result; } -# define AO_HAVE_int_load_read +# define AO_HAVE_short_load_read #endif -#if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_int_load_full) -# define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr)) -# define AO_HAVE_int_load_full +#if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_short_load_full) +# define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr)) +# define AO_HAVE_short_load_full #endif -#if !defined(AO_HAVE_int_load_acquire_read) \ - && defined(AO_HAVE_int_load_read) -# define AO_int_load_acquire_read(addr) AO_int_load_read(addr) -# define AO_HAVE_int_load_acquire_read +#if defined(AO_HAVE_short_compare_and_swap_read) \ + && !defined(AO_HAVE_short_load_read) +# define AO_short_CAS_BASED_LOAD_READ + AO_INLINE unsigned/**/short + AO_short_load_read(const volatile unsigned/**/short *addr) + { + unsigned/**/short result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_read( + (volatile unsigned/**/short *)addr, + result, result))); + return result; + } +# define AO_HAVE_short_load_read #endif -#if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load) -# define AO_int_load(addr) AO_int_load_acquire_read(addr) -# define AO_HAVE_int_load +#if !defined(AO_HAVE_short_load_acquire_read) \ + && defined(AO_HAVE_short_load_read) +# define AO_short_load_acquire_read(addr) AO_short_load_read(addr) +# define AO_HAVE_short_load_acquire_read +#endif + +#if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load) \ + && (!defined(AO_short_CAS_BASED_LOAD_READ) \ + || !defined(AO_HAVE_short_compare_and_swap)) +# define AO_short_load(addr) AO_short_load_acquire_read(addr) +# define AO_HAVE_short_load +#endif + +#if defined(AO_HAVE_short_compare_and_swap_full) \ + && !defined(AO_HAVE_short_load_full) + AO_INLINE unsigned/**/short + AO_short_load_full(const volatile unsigned/**/short *addr) + { + unsigned/**/short result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full( + (volatile unsigned/**/short *)addr, + result, result))); + return result; + } +# define AO_HAVE_short_load_full +#endif + +#if defined(AO_HAVE_short_compare_and_swap_acquire) \ + && !defined(AO_HAVE_short_load_acquire) + AO_INLINE unsigned/**/short + AO_short_load_acquire(const volatile unsigned/**/short *addr) + { + unsigned/**/short result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire( + (volatile unsigned/**/short *)addr, + result, result))); + return result; + } +# define AO_HAVE_short_load_acquire +#endif + +#if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_load) + AO_INLINE unsigned/**/short + AO_short_load(const volatile unsigned/**/short *addr) + { + unsigned/**/short result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap( + (volatile unsigned/**/short *)addr, + result, result))); + return result; + } +# define AO_HAVE_short_load #endif #ifdef AO_NO_DD_ORDERING -# if defined(AO_HAVE_int_load_acquire_read) -# define AO_int_load_dd_acquire_read(addr) AO_int_load_acquire_read(addr) -# define AO_HAVE_int_load_dd_acquire_read +# if defined(AO_HAVE_short_load_acquire_read) +# define AO_short_load_dd_acquire_read(addr) \ + AO_short_load_acquire_read(addr) +# define AO_HAVE_short_load_dd_acquire_read # endif #else -# if defined(AO_HAVE_int_load) -# define AO_int_load_dd_acquire_read(addr) AO_int_load(addr) -# define AO_HAVE_int_load_dd_acquire_read +# if defined(AO_HAVE_short_load) +# define AO_short_load_dd_acquire_read(addr) AO_short_load(addr) +# define AO_HAVE_short_load_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ -/* int_store */ -#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release) -# define AO_int_store_release(addr,val) AO_int_store_full(addr,val) -# define AO_HAVE_int_store_release +/* short_store */ +#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release) +# define AO_short_store_release(addr, val) AO_short_store_full(addr, val) +# define AO_HAVE_short_store_release #endif -#if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store) -# define AO_int_store(addr, val) AO_int_store_release(addr,val) -# define AO_HAVE_int_store +#if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store) +# define AO_short_store(addr, val) AO_short_store_release(addr, val) +# define AO_HAVE_short_store #endif -#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write) -# define AO_int_store_write(addr,val) AO_int_store_full(addr,val) -# define AO_HAVE_int_store_write +#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write) +# define AO_short_store_write(addr, val) AO_short_store_full(addr, val) +# define AO_HAVE_short_store_write #endif -#if defined(AO_HAVE_int_store_release) \ - && !defined(AO_HAVE_int_store_release_write) -# define AO_int_store_release_write(addr, val) \ - AO_int_store_release(addr,val) -# define AO_HAVE_int_store_release_write +#if defined(AO_HAVE_short_store_release) \ + && !defined(AO_HAVE_short_store_release_write) +# define AO_short_store_release_write(addr, val) \ + AO_short_store_release(addr, val) +# define AO_HAVE_short_store_release_write #endif -#if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store) -# define AO_int_store(addr, val) AO_int_store_write(addr,val) -# define AO_HAVE_int_store +#if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store) +# define AO_short_store(addr, val) AO_short_store_write(addr, val) +# define AO_HAVE_short_store #endif -#if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_int_store_release) -# define AO_int_store_release(addr,val) \ - (AO_nop_full(), AO_int_store(addr,val)) -# define AO_HAVE_int_store_release +#if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_short_store_release) +# define AO_short_store_release(addr, val) \ + (AO_nop_full(), AO_short_store(addr, val)) +# define AO_HAVE_short_store_release #endif -#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_int_store) \ - && !defined(AO_HAVE_int_store_write) -# define AO_int_store_write(addr, val) \ - (AO_nop_write(), AO_int_store(addr,val)) -# define AO_HAVE_int_store_write +#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_short_store) \ + && !defined(AO_HAVE_short_store_write) +# define AO_short_store_write(addr, val) \ + (AO_nop_write(), AO_short_store(addr, val)) +# define AO_HAVE_short_store_write #endif -#if defined(AO_HAVE_int_store_write) \ - && !defined(AO_HAVE_int_store_release_write) -# define AO_int_store_release_write(addr, val) AO_int_store_write(addr,val) -# define AO_HAVE_int_store_release_write +#if defined(AO_HAVE_short_store_write) \ + && !defined(AO_HAVE_short_store_release_write) +# define AO_short_store_release_write(addr, val) \ + AO_short_store_write(addr, val) +# define AO_HAVE_short_store_release_write #endif -#if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_int_store_full) -# define AO_int_store_full(addr, val) \ - (AO_int_store_release(addr, val), AO_nop_full()) -# define AO_HAVE_int_store_full +#if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_short_store_full) +# define AO_short_store_full(addr, val) \ + (AO_short_store_release(addr, val), \ + AO_nop_full()) +# define AO_HAVE_short_store_full #endif +/* + * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ /* int_fetch_compare_and_swap */ #if defined(AO_HAVE_int_fetch_compare_and_swap) \ @@ -1168,163 +1178,236 @@ # define AO_HAVE_int_compare_and_swap_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ -/* - * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -/* load */ -#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire) -# define AO_load_acquire(addr) AO_load_full(addr) -# define AO_HAVE_load_acquire +/* int_load */ +#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire) +# define AO_int_load_acquire(addr) AO_int_load_full(addr) +# define AO_HAVE_int_load_acquire #endif -#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load) -# define AO_load(addr) AO_load_acquire(addr) -# define AO_HAVE_load +#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load) +# define AO_int_load(addr) AO_int_load_acquire(addr) +# define AO_HAVE_int_load #endif -#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read) -# define AO_load_read(addr) AO_load_full(addr) -# define AO_HAVE_load_read +#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read) +# define AO_int_load_read(addr) AO_int_load_full(addr) +# define AO_HAVE_int_load_read #endif -#if !defined(AO_HAVE_load_acquire_read) \ - && defined(AO_HAVE_load_acquire) -# define AO_load_acquire_read(addr) AO_load_acquire(addr) -# define AO_HAVE_load_acquire_read +#if !defined(AO_HAVE_int_load_acquire_read) \ + && defined(AO_HAVE_int_load_acquire) +# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr) +# define AO_HAVE_int_load_acquire_read #endif -#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_load_acquire) - AO_INLINE AO_t - AO_load_acquire(const volatile AO_t *addr) +#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_int_load_acquire) + AO_INLINE unsigned + AO_int_load_acquire(const volatile unsigned *addr) { - AO_t result = AO_load(addr); + unsigned result = AO_int_load(addr); /* Acquire barrier would be useless, since the load could be delayed */ /* beyond it. */ AO_nop_full(); return result; } -# define AO_HAVE_load_acquire +# define AO_HAVE_int_load_acquire #endif -#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \ - && !defined(AO_HAVE_load_read) - AO_INLINE AO_t - AO_load_read(const volatile AO_t *addr) +#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \ + && !defined(AO_HAVE_int_load_read) + AO_INLINE unsigned + AO_int_load_read(const volatile unsigned *addr) { - AO_t result = AO_load(addr); + unsigned result = AO_int_load(addr); /* Acquire barrier would be useless, since the load could be delayed */ /* beyond it. */ AO_nop_read(); return result; } -# define AO_HAVE_load_read +# define AO_HAVE_int_load_read #endif -#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_load_full) -# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr)) -# define AO_HAVE_load_full +#if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_int_load_full) +# define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr)) +# define AO_HAVE_int_load_full #endif -#if !defined(AO_HAVE_load_acquire_read) \ - && defined(AO_HAVE_load_read) -# define AO_load_acquire_read(addr) AO_load_read(addr) -# define AO_HAVE_load_acquire_read +#if defined(AO_HAVE_int_compare_and_swap_read) \ + && !defined(AO_HAVE_int_load_read) +# define AO_int_CAS_BASED_LOAD_READ + AO_INLINE unsigned + AO_int_load_read(const volatile unsigned *addr) + { + unsigned result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_read( + (volatile unsigned *)addr, + result, result))); + return result; + } +# define AO_HAVE_int_load_read #endif -#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load) -# define AO_load(addr) AO_load_acquire_read(addr) -# define AO_HAVE_load +#if !defined(AO_HAVE_int_load_acquire_read) \ + && defined(AO_HAVE_int_load_read) +# define AO_int_load_acquire_read(addr) AO_int_load_read(addr) +# define AO_HAVE_int_load_acquire_read +#endif + +#if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load) \ + && (!defined(AO_int_CAS_BASED_LOAD_READ) \ + || !defined(AO_HAVE_int_compare_and_swap)) +# define AO_int_load(addr) AO_int_load_acquire_read(addr) +# define AO_HAVE_int_load +#endif + +#if defined(AO_HAVE_int_compare_and_swap_full) \ + && !defined(AO_HAVE_int_load_full) + AO_INLINE unsigned + AO_int_load_full(const volatile unsigned *addr) + { + unsigned result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full( + (volatile unsigned *)addr, + result, result))); + return result; + } +# define AO_HAVE_int_load_full +#endif + +#if defined(AO_HAVE_int_compare_and_swap_acquire) \ + && !defined(AO_HAVE_int_load_acquire) + AO_INLINE unsigned + AO_int_load_acquire(const volatile unsigned *addr) + { + unsigned result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire( + (volatile unsigned *)addr, + result, result))); + return result; + } +# define AO_HAVE_int_load_acquire +#endif + +#if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_load) + AO_INLINE unsigned + AO_int_load(const volatile unsigned *addr) + { + unsigned result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap( + (volatile unsigned *)addr, + result, result))); + return result; + } +# define AO_HAVE_int_load #endif #ifdef AO_NO_DD_ORDERING -# if defined(AO_HAVE_load_acquire_read) -# define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr) -# define AO_HAVE_load_dd_acquire_read +# if defined(AO_HAVE_int_load_acquire_read) +# define AO_int_load_dd_acquire_read(addr) \ + AO_int_load_acquire_read(addr) +# define AO_HAVE_int_load_dd_acquire_read # endif #else -# if defined(AO_HAVE_load) -# define AO_load_dd_acquire_read(addr) AO_load(addr) -# define AO_HAVE_load_dd_acquire_read +# if defined(AO_HAVE_int_load) +# define AO_int_load_dd_acquire_read(addr) AO_int_load(addr) +# define AO_HAVE_int_load_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ -/* store */ -#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release) -# define AO_store_release(addr,val) AO_store_full(addr,val) -# define AO_HAVE_store_release +/* int_store */ +#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release) +# define AO_int_store_release(addr, val) AO_int_store_full(addr, val) +# define AO_HAVE_int_store_release #endif -#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store) -# define AO_store(addr, val) AO_store_release(addr,val) -# define AO_HAVE_store +#if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store) +# define AO_int_store(addr, val) AO_int_store_release(addr, val) +# define AO_HAVE_int_store #endif -#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write) -# define AO_store_write(addr,val) AO_store_full(addr,val) -# define AO_HAVE_store_write +#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write) +# define AO_int_store_write(addr, val) AO_int_store_full(addr, val) +# define AO_HAVE_int_store_write #endif -#if defined(AO_HAVE_store_release) \ - && !defined(AO_HAVE_store_release_write) -# define AO_store_release_write(addr, val) \ - AO_store_release(addr,val) -# define AO_HAVE_store_release_write +#if defined(AO_HAVE_int_store_release) \ + && !defined(AO_HAVE_int_store_release_write) +# define AO_int_store_release_write(addr, val) \ + AO_int_store_release(addr, val) +# define AO_HAVE_int_store_release_write #endif -#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store) -# define AO_store(addr, val) AO_store_write(addr,val) -# define AO_HAVE_store +#if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store) +# define AO_int_store(addr, val) AO_int_store_write(addr, val) +# define AO_HAVE_int_store #endif -#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_store_release) -# define AO_store_release(addr,val) \ - (AO_nop_full(), AO_store(addr,val)) -# define AO_HAVE_store_release +#if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_int_store_release) +# define AO_int_store_release(addr, val) \ + (AO_nop_full(), AO_int_store(addr, val)) +# define AO_HAVE_int_store_release #endif -#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) \ - && !defined(AO_HAVE_store_write) -# define AO_store_write(addr, val) \ - (AO_nop_write(), AO_store(addr,val)) -# define AO_HAVE_store_write +#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_int_store) \ + && !defined(AO_HAVE_int_store_write) +# define AO_int_store_write(addr, val) \ + (AO_nop_write(), AO_int_store(addr, val)) +# define AO_HAVE_int_store_write #endif -#if defined(AO_HAVE_store_write) \ - && !defined(AO_HAVE_store_release_write) -# define AO_store_release_write(addr, val) AO_store_write(addr,val) -# define AO_HAVE_store_release_write +#if defined(AO_HAVE_int_store_write) \ + && !defined(AO_HAVE_int_store_release_write) +# define AO_int_store_release_write(addr, val) \ + AO_int_store_write(addr, val) +# define AO_HAVE_int_store_release_write #endif -#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_store_full) -# define AO_store_full(addr, val) \ - (AO_store_release(addr, val), AO_nop_full()) -# define AO_HAVE_store_full +#if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_int_store_full) +# define AO_int_store_full(addr, val) \ + (AO_int_store_release(addr, val), \ + AO_nop_full()) +# define AO_HAVE_int_store_full #endif +/* + * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ /* fetch_compare_and_swap */ #if defined(AO_HAVE_fetch_compare_and_swap) \ @@ -1558,3 +1641,212 @@ # define AO_HAVE_compare_and_swap_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ + +/* load */ +#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire) +# define AO_load_acquire(addr) AO_load_full(addr) +# define AO_HAVE_load_acquire +#endif + +#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load) +# define AO_load(addr) AO_load_acquire(addr) +# define AO_HAVE_load +#endif + +#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read) +# define AO_load_read(addr) AO_load_full(addr) +# define AO_HAVE_load_read +#endif + +#if !defined(AO_HAVE_load_acquire_read) \ + && defined(AO_HAVE_load_acquire) +# define AO_load_acquire_read(addr) AO_load_acquire(addr) +# define AO_HAVE_load_acquire_read +#endif + +#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_load_acquire) + AO_INLINE AO_t + AO_load_acquire(const volatile AO_t *addr) + { + AO_t result = AO_load(addr); + + /* Acquire barrier would be useless, since the load could be delayed */ + /* beyond it. */ + AO_nop_full(); + return result; + } +# define AO_HAVE_load_acquire +#endif + +#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \ + && !defined(AO_HAVE_load_read) + AO_INLINE AO_t + AO_load_read(const volatile AO_t *addr) + { + AO_t result = AO_load(addr); + + /* Acquire barrier would be useless, since the load could be delayed */ + /* beyond it. */ + AO_nop_read(); + return result; + } +# define AO_HAVE_load_read +#endif + +#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_load_full) +# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr)) +# define AO_HAVE_load_full +#endif + +#if defined(AO_HAVE_compare_and_swap_read) \ + && !defined(AO_HAVE_load_read) +# define AO_CAS_BASED_LOAD_READ + AO_INLINE AO_t + AO_load_read(const volatile AO_t *addr) + { + AO_t result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_compare_and_swap_read( + (volatile AO_t *)addr, + result, result))); + return result; + } +# define AO_HAVE_load_read +#endif + +#if !defined(AO_HAVE_load_acquire_read) \ + && defined(AO_HAVE_load_read) +# define AO_load_acquire_read(addr) AO_load_read(addr) +# define AO_HAVE_load_acquire_read +#endif + +#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load) \ + && (!defined(AO_CAS_BASED_LOAD_READ) \ + || !defined(AO_HAVE_compare_and_swap)) +# define AO_load(addr) AO_load_acquire_read(addr) +# define AO_HAVE_load +#endif + +#if defined(AO_HAVE_compare_and_swap_full) \ + && !defined(AO_HAVE_load_full) + AO_INLINE AO_t + AO_load_full(const volatile AO_t *addr) + { + AO_t result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full( + (volatile AO_t *)addr, + result, result))); + return result; + } +# define AO_HAVE_load_full +#endif + +#if defined(AO_HAVE_compare_and_swap_acquire) \ + && !defined(AO_HAVE_load_acquire) + AO_INLINE AO_t + AO_load_acquire(const volatile AO_t *addr) + { + AO_t result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire( + (volatile AO_t *)addr, + result, result))); + return result; + } +# define AO_HAVE_load_acquire +#endif + +#if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_load) + AO_INLINE AO_t + AO_load(const volatile AO_t *addr) + { + AO_t result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_compare_and_swap( + (volatile AO_t *)addr, + result, result))); + return result; + } +# define AO_HAVE_load +#endif + +#ifdef AO_NO_DD_ORDERING +# if defined(AO_HAVE_load_acquire_read) +# define AO_load_dd_acquire_read(addr) \ + AO_load_acquire_read(addr) +# define AO_HAVE_load_dd_acquire_read +# endif +#else +# if defined(AO_HAVE_load) +# define AO_load_dd_acquire_read(addr) AO_load(addr) +# define AO_HAVE_load_dd_acquire_read +# endif +#endif /* !AO_NO_DD_ORDERING */ + +/* store */ +#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release) +# define AO_store_release(addr, val) AO_store_full(addr, val) +# define AO_HAVE_store_release +#endif + +#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store) +# define AO_store(addr, val) AO_store_release(addr, val) +# define AO_HAVE_store +#endif + +#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write) +# define AO_store_write(addr, val) AO_store_full(addr, val) +# define AO_HAVE_store_write +#endif + +#if defined(AO_HAVE_store_release) \ + && !defined(AO_HAVE_store_release_write) +# define AO_store_release_write(addr, val) \ + AO_store_release(addr, val) +# define AO_HAVE_store_release_write +#endif + +#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store) +# define AO_store(addr, val) AO_store_write(addr, val) +# define AO_HAVE_store +#endif + +#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_store_release) +# define AO_store_release(addr, val) \ + (AO_nop_full(), AO_store(addr, val)) +# define AO_HAVE_store_release +#endif + +#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) \ + && !defined(AO_HAVE_store_write) +# define AO_store_write(addr, val) \ + (AO_nop_write(), AO_store(addr, val)) +# define AO_HAVE_store_write +#endif + +#if defined(AO_HAVE_store_write) \ + && !defined(AO_HAVE_store_release_write) +# define AO_store_release_write(addr, val) \ + AO_store_write(addr, val) +# define AO_HAVE_store_release_write +#endif + +#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_store_full) +# define AO_store_full(addr, val) \ + (AO_store_release(addr, val), \ + AO_nop_full()) +# define AO_HAVE_store_full +#endif diff --git a/src/atomic_ops/generalize-small.template b/src/atomic_ops/generalize-small.template index 8ac9cfd..ae06901 100644 --- a/src/atomic_ops/generalize-small.template +++ b/src/atomic_ops/generalize-small.template @@ -20,142 +20,6 @@ * SOFTWARE. */ -/* XSIZE_load */ -#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_acquire) -# define AO_XSIZE_load_acquire(addr) AO_XSIZE_load_full(addr) -# define AO_HAVE_XSIZE_load_acquire -#endif - -#if defined(AO_HAVE_XSIZE_load_acquire) && !defined(AO_HAVE_XSIZE_load) -# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire(addr) -# define AO_HAVE_XSIZE_load -#endif - -#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_read) -# define AO_XSIZE_load_read(addr) AO_XSIZE_load_full(addr) -# define AO_HAVE_XSIZE_load_read -#endif - -#if !defined(AO_HAVE_XSIZE_load_acquire_read) \ - && defined(AO_HAVE_XSIZE_load_acquire) -# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_acquire(addr) -# define AO_HAVE_XSIZE_load_acquire_read -#endif - -#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_XSIZE_load_acquire) - AO_INLINE XCTYPE - AO_XSIZE_load_acquire(const volatile XCTYPE *addr) - { - XCTYPE result = AO_XSIZE_load(addr); - - /* Acquire barrier would be useless, since the load could be delayed */ - /* beyond it. */ - AO_nop_full(); - return result; - } -# define AO_HAVE_XSIZE_load_acquire -#endif - -#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_read) \ - && !defined(AO_HAVE_XSIZE_load_read) - AO_INLINE XCTYPE - AO_XSIZE_load_read(const volatile XCTYPE *addr) - { - XCTYPE result = AO_XSIZE_load(addr); - - /* Acquire barrier would be useless, since the load could be delayed */ - /* beyond it. */ - AO_nop_read(); - return result; - } -# define AO_HAVE_XSIZE_load_read -#endif - -#if defined(AO_HAVE_XSIZE_load_acquire) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_XSIZE_load_full) -# define AO_XSIZE_load_full(addr) (AO_nop_full(), AO_XSIZE_load_acquire(addr)) -# define AO_HAVE_XSIZE_load_full -#endif - -#if !defined(AO_HAVE_XSIZE_load_acquire_read) \ - && defined(AO_HAVE_XSIZE_load_read) -# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_read(addr) -# define AO_HAVE_XSIZE_load_acquire_read -#endif - -#if defined(AO_HAVE_XSIZE_load_acquire_read) && !defined(AO_HAVE_XSIZE_load) -# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire_read(addr) -# define AO_HAVE_XSIZE_load -#endif - -#ifdef AO_NO_DD_ORDERING -# if defined(AO_HAVE_XSIZE_load_acquire_read) -# define AO_XSIZE_load_dd_acquire_read(addr) AO_XSIZE_load_acquire_read(addr) -# define AO_HAVE_XSIZE_load_dd_acquire_read -# endif -#else -# if defined(AO_HAVE_XSIZE_load) -# define AO_XSIZE_load_dd_acquire_read(addr) AO_XSIZE_load(addr) -# define AO_HAVE_XSIZE_load_dd_acquire_read -# endif -#endif /* !AO_NO_DD_ORDERING */ - -/* XSIZE_store */ -#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_release) -# define AO_XSIZE_store_release(addr,val) AO_XSIZE_store_full(addr,val) -# define AO_HAVE_XSIZE_store_release -#endif - -#if defined(AO_HAVE_XSIZE_store_release) && !defined(AO_HAVE_XSIZE_store) -# define AO_XSIZE_store(addr, val) AO_XSIZE_store_release(addr,val) -# define AO_HAVE_XSIZE_store -#endif - -#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_write) -# define AO_XSIZE_store_write(addr,val) AO_XSIZE_store_full(addr,val) -# define AO_HAVE_XSIZE_store_write -#endif - -#if defined(AO_HAVE_XSIZE_store_release) \ - && !defined(AO_HAVE_XSIZE_store_release_write) -# define AO_XSIZE_store_release_write(addr, val) \ - AO_XSIZE_store_release(addr,val) -# define AO_HAVE_XSIZE_store_release_write -#endif - -#if defined(AO_HAVE_XSIZE_store_write) && !defined(AO_HAVE_XSIZE_store) -# define AO_XSIZE_store(addr, val) AO_XSIZE_store_write(addr,val) -# define AO_HAVE_XSIZE_store -#endif - -#if defined(AO_HAVE_XSIZE_store) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_XSIZE_store_release) -# define AO_XSIZE_store_release(addr,val) \ - (AO_nop_full(), AO_XSIZE_store(addr,val)) -# define AO_HAVE_XSIZE_store_release -#endif - -#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_XSIZE_store) \ - && !defined(AO_HAVE_XSIZE_store_write) -# define AO_XSIZE_store_write(addr, val) \ - (AO_nop_write(), AO_XSIZE_store(addr,val)) -# define AO_HAVE_XSIZE_store_write -#endif - -#if defined(AO_HAVE_XSIZE_store_write) \ - && !defined(AO_HAVE_XSIZE_store_release_write) -# define AO_XSIZE_store_release_write(addr, val) AO_XSIZE_store_write(addr,val) -# define AO_HAVE_XSIZE_store_release_write -#endif - -#if defined(AO_HAVE_XSIZE_store_release) && defined(AO_HAVE_nop_full) \ - && !defined(AO_HAVE_XSIZE_store_full) -# define AO_XSIZE_store_full(addr, val) \ - (AO_XSIZE_store_release(addr, val), AO_nop_full()) -# define AO_HAVE_XSIZE_store_full -#endif - /* XSIZE_fetch_compare_and_swap */ #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \ && defined(AO_HAVE_nop_full) \ @@ -388,3 +252,212 @@ # define AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ + +/* XSIZE_load */ +#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_acquire) +# define AO_XSIZE_load_acquire(addr) AO_XSIZE_load_full(addr) +# define AO_HAVE_XSIZE_load_acquire +#endif + +#if defined(AO_HAVE_XSIZE_load_acquire) && !defined(AO_HAVE_XSIZE_load) +# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire(addr) +# define AO_HAVE_XSIZE_load +#endif + +#if defined(AO_HAVE_XSIZE_load_full) && !defined(AO_HAVE_XSIZE_load_read) +# define AO_XSIZE_load_read(addr) AO_XSIZE_load_full(addr) +# define AO_HAVE_XSIZE_load_read +#endif + +#if !defined(AO_HAVE_XSIZE_load_acquire_read) \ + && defined(AO_HAVE_XSIZE_load_acquire) +# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_acquire(addr) +# define AO_HAVE_XSIZE_load_acquire_read +#endif + +#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_XSIZE_load_acquire) + AO_INLINE XCTYPE + AO_XSIZE_load_acquire(const volatile XCTYPE *addr) + { + XCTYPE result = AO_XSIZE_load(addr); + + /* Acquire barrier would be useless, since the load could be delayed */ + /* beyond it. */ + AO_nop_full(); + return result; + } +# define AO_HAVE_XSIZE_load_acquire +#endif + +#if defined(AO_HAVE_XSIZE_load) && defined(AO_HAVE_nop_read) \ + && !defined(AO_HAVE_XSIZE_load_read) + AO_INLINE XCTYPE + AO_XSIZE_load_read(const volatile XCTYPE *addr) + { + XCTYPE result = AO_XSIZE_load(addr); + + /* Acquire barrier would be useless, since the load could be delayed */ + /* beyond it. */ + AO_nop_read(); + return result; + } +# define AO_HAVE_XSIZE_load_read +#endif + +#if defined(AO_HAVE_XSIZE_load_acquire) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_XSIZE_load_full) +# define AO_XSIZE_load_full(addr) (AO_nop_full(), AO_XSIZE_load_acquire(addr)) +# define AO_HAVE_XSIZE_load_full +#endif + +#if defined(AO_HAVE_XSIZE_compare_and_swap_read) \ + && !defined(AO_HAVE_XSIZE_load_read) +# define AO_XSIZE_CAS_BASED_LOAD_READ + AO_INLINE XCTYPE + AO_XSIZE_load_read(const volatile XCTYPE *addr) + { + XCTYPE result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_read( + (volatile XCTYPE *)addr, + result, result))); + return result; + } +# define AO_HAVE_XSIZE_load_read +#endif + +#if !defined(AO_HAVE_XSIZE_load_acquire_read) \ + && defined(AO_HAVE_XSIZE_load_read) +# define AO_XSIZE_load_acquire_read(addr) AO_XSIZE_load_read(addr) +# define AO_HAVE_XSIZE_load_acquire_read +#endif + +#if defined(AO_HAVE_XSIZE_load_acquire_read) && !defined(AO_HAVE_XSIZE_load) \ + && (!defined(AO_XSIZE_CAS_BASED_LOAD_READ) \ + || !defined(AO_HAVE_XSIZE_compare_and_swap)) +# define AO_XSIZE_load(addr) AO_XSIZE_load_acquire_read(addr) +# define AO_HAVE_XSIZE_load +#endif + +#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ + && !defined(AO_HAVE_XSIZE_load_full) + AO_INLINE XCTYPE + AO_XSIZE_load_full(const volatile XCTYPE *addr) + { + XCTYPE result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full( + (volatile XCTYPE *)addr, + result, result))); + return result; + } +# define AO_HAVE_XSIZE_load_full +#endif + +#if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \ + && !defined(AO_HAVE_XSIZE_load_acquire) + AO_INLINE XCTYPE + AO_XSIZE_load_acquire(const volatile XCTYPE *addr) + { + XCTYPE result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire( + (volatile XCTYPE *)addr, + result, result))); + return result; + } +# define AO_HAVE_XSIZE_load_acquire +#endif + +#if defined(AO_HAVE_XSIZE_compare_and_swap) && !defined(AO_HAVE_XSIZE_load) + AO_INLINE XCTYPE + AO_XSIZE_load(const volatile XCTYPE *addr) + { + XCTYPE result; + + do { + result = *addr; + } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap( + (volatile XCTYPE *)addr, + result, result))); + return result; + } +# define AO_HAVE_XSIZE_load +#endif + +#ifdef AO_NO_DD_ORDERING +# if defined(AO_HAVE_XSIZE_load_acquire_read) +# define AO_XSIZE_load_dd_acquire_read(addr) \ + AO_XSIZE_load_acquire_read(addr) +# define AO_HAVE_XSIZE_load_dd_acquire_read +# endif +#else +# if defined(AO_HAVE_XSIZE_load) +# define AO_XSIZE_load_dd_acquire_read(addr) AO_XSIZE_load(addr) +# define AO_HAVE_XSIZE_load_dd_acquire_read +# endif +#endif /* !AO_NO_DD_ORDERING */ + +/* XSIZE_store */ +#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_release) +# define AO_XSIZE_store_release(addr, val) AO_XSIZE_store_full(addr, val) +# define AO_HAVE_XSIZE_store_release +#endif + +#if defined(AO_HAVE_XSIZE_store_release) && !defined(AO_HAVE_XSIZE_store) +# define AO_XSIZE_store(addr, val) AO_XSIZE_store_release(addr, val) +# define AO_HAVE_XSIZE_store +#endif + +#if defined(AO_HAVE_XSIZE_store_full) && !defined(AO_HAVE_XSIZE_store_write) +# define AO_XSIZE_store_write(addr, val) AO_XSIZE_store_full(addr, val) +# define AO_HAVE_XSIZE_store_write +#endif + +#if defined(AO_HAVE_XSIZE_store_release) \ + && !defined(AO_HAVE_XSIZE_store_release_write) +# define AO_XSIZE_store_release_write(addr, val) \ + AO_XSIZE_store_release(addr, val) +# define AO_HAVE_XSIZE_store_release_write +#endif + +#if defined(AO_HAVE_XSIZE_store_write) && !defined(AO_HAVE_XSIZE_store) +# define AO_XSIZE_store(addr, val) AO_XSIZE_store_write(addr, val) +# define AO_HAVE_XSIZE_store +#endif + +#if defined(AO_HAVE_XSIZE_store) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_XSIZE_store_release) +# define AO_XSIZE_store_release(addr, val) \ + (AO_nop_full(), AO_XSIZE_store(addr, val)) +# define AO_HAVE_XSIZE_store_release +#endif + +#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_XSIZE_store) \ + && !defined(AO_HAVE_XSIZE_store_write) +# define AO_XSIZE_store_write(addr, val) \ + (AO_nop_write(), AO_XSIZE_store(addr, val)) +# define AO_HAVE_XSIZE_store_write +#endif + +#if defined(AO_HAVE_XSIZE_store_write) \ + && !defined(AO_HAVE_XSIZE_store_release_write) +# define AO_XSIZE_store_release_write(addr, val) \ + AO_XSIZE_store_write(addr, val) +# define AO_HAVE_XSIZE_store_release_write +#endif + +#if defined(AO_HAVE_XSIZE_store_release) && defined(AO_HAVE_nop_full) \ + && !defined(AO_HAVE_XSIZE_store_full) +# define AO_XSIZE_store_full(addr, val) \ + (AO_XSIZE_store_release(addr, val), \ + AO_nop_full()) +# define AO_HAVE_XSIZE_store_full +#endif -- 2.7.4