Generate generalized AO_t load/store/fetch_and_add primitives from template

(code refactoring)

* src/Makefile.am (atomic_ops/generalize-small.h): Append
generalize-small.template to generalize-small.h replacing XCTYPE with
AO_t (and "XSIZE_" with empty sequence).
* src/atomic_ops/generalize-small.h: Regenerate.
* src/atomic_ops/generalize.h (AO_load_acquire, AO_load, AO_load_read,
AO_load_acquire_read, AO_load_full, AO_load_dd_acquire_read,
AO_store_release, AO_store, AO_store_write, AO_store_release_write,
AO_store_full, AO_fetch_and_add_full, AO_fetch_and_add_acquire,
AO_fetch_and_add_release, AO_fetch_and_add, AO_fetch_and_add_write,
AO_fetch_and_add_read, AO_fetch_and_add_release_write,
AO_fetch_and_add_acquire_read, AO_fetch_and_add_dd_acquire_read,
AO_fetch_and_add1_full, AO_fetch_and_add1_release,
AO_fetch_and_add1_acquire, AO_fetch_and_add1_write,
AO_fetch_and_add1_read, AO_fetch_and_add1_release_write,
AO_fetch_and_add1_acquire_read, AO_fetch_and_add1,
AO_fetch_and_add1_dd_acquire_read, AO_fetch_and_sub1_full,
AO_fetch_and_sub1_release, AO_fetch_and_sub1_acquire,
AO_fetch_and_sub1_write, AO_fetch_and_sub1_read,
AO_fetch_and_sub1_release_write, AO_fetch_and_sub1_acquire_read,
AO_fetch_and_sub1, AO_fetch_and_sub1_dd_acquire_read): Remove
generalized primitives (and the corresponding AO_HAVE_x macros) that
exist in auto-generated generalize-small.h file.

diff --git a/src/Makefile.am b/src/Makefile.am
index f5bad92..8f187cf 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -82,3 +82,4 @@ atomic_ops/generalize-small.h: atomic_ops/generalize-small.template
        sed -e s:XSIZE_:char_:g -e s:XCTYPE:unsigned/**/char:g $? > $@
        sed -e s:XSIZE_:short_:g -e s:XCTYPE:unsigned/**/short:g $? >> $@
        sed -e s:XSIZE_:int_:g -e s:XCTYPE:unsigned:g $? >> $@
+       sed -e s:XSIZE_::g -e s:XCTYPE:AO_t:g $? >> $@

diff --git a/src/atomic_ops/generalize-small.h b/src/atomic_ops/generalize-small.h
index 3dfc18b..06aebf9 100644 (file)
--- a/src/atomic_ops/generalize-small.h
+++ b/src/atomic_ops/generalize-small.h
@@ -1966,3 +1966,659 @@
 #   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* load */
+#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
+# define AO_load_acquire(addr) AO_load_full(addr)
+# define AO_HAVE_load_acquire
+#endif
+
+#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
+# define AO_load(addr) AO_load_acquire(addr)
+# define AO_HAVE_load
+#endif
+
+#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
+# define AO_load_read(addr) AO_load_full(addr)
+# define AO_HAVE_load_read
+#endif
+
+#if !defined(AO_HAVE_load_acquire_read) \
+    && defined(AO_HAVE_load_acquire)
+# define AO_load_acquire_read(addr) AO_load_acquire(addr)
+# define AO_HAVE_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_load_acquire)
+  AO_INLINE AO_t
+  AO_load_acquire(const volatile AO_t *addr)
+  {
+    AO_t result = AO_load(addr);
+
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_full();
+    return result;
+  }
+# define AO_HAVE_load_acquire
+#endif
+
+#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_load_read)
+  AO_INLINE AO_t
+  AO_load_read(const volatile AO_t *addr)
+  {
+    AO_t result = AO_load(addr);
+
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_read();
+    return result;
+  }
+# define AO_HAVE_load_read
+#endif
+
+#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_load_full)
+# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
+# define AO_HAVE_load_full
+#endif
+
+#if !defined(AO_HAVE_load_acquire_read) \
+    && defined(AO_HAVE_load_read)
+# define AO_load_acquire_read(addr) AO_load_read(addr)
+# define AO_HAVE_load_acquire_read
+#endif
+
+#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
+# define AO_load(addr) AO_load_acquire_read(addr)
+# define AO_HAVE_load
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_load_acquire_read)
+#   define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
+#   define AO_HAVE_load_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_load)
+#   define AO_load_dd_acquire_read(addr) AO_load(addr)
+#   define AO_HAVE_load_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* store */
+#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
+# define AO_store_release(addr,val) AO_store_full(addr,val)
+# define AO_HAVE_store_release
+#endif
+
+#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
+# define AO_store(addr, val) AO_store_release(addr,val)
+# define AO_HAVE_store
+#endif
+
+#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
+# define AO_store_write(addr,val) AO_store_full(addr,val)
+# define AO_HAVE_store_write
+#endif
+
+#if defined(AO_HAVE_store_release) \
+    && !defined(AO_HAVE_store_release_write)
+# define AO_store_release_write(addr, val) \
+                            AO_store_release(addr,val)
+# define AO_HAVE_store_release_write
+#endif
+
+#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
+# define AO_store(addr, val) AO_store_write(addr,val)
+# define AO_HAVE_store
+#endif
+
+#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_store_release)
+# define AO_store_release(addr,val) \
+                                (AO_nop_full(), AO_store(addr,val))
+# define AO_HAVE_store_release
+#endif
+
+#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) \
+    && !defined(AO_HAVE_store_write)
+# define AO_store_write(addr, val) \
+                                (AO_nop_write(), AO_store(addr,val))
+# define AO_HAVE_store_write
+#endif
+
+#if defined(AO_HAVE_store_write) \
+    && !defined(AO_HAVE_store_release_write)
+# define AO_store_release_write(addr, val) AO_store_write(addr,val)
+# define AO_HAVE_store_release_write
+#endif
+
+#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_store_full)
+# define AO_store_full(addr, val) \
+                        (AO_store_release(addr, val), AO_nop_full())
+# define AO_HAVE_store_full
+#endif
+
+/* compare_and_swap */
+#if defined(AO_HAVE_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_compare_and_swap_full)
+  AO_INLINE int
+  AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
+                                 AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap_full(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_compare_and_swap_full
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_compare_and_swap_acquire)
+  AO_INLINE int
+  AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
+                                    AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_compare_and_swap_acquire
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_compare_and_swap_release)
+  AO_INLINE int
+  AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val,
+                                    AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap_release(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_compare_and_swap_release
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_compare_and_swap)
+  AO_INLINE int
+  AO_compare_and_swap(volatile AO_t *addr, AO_t old_val,
+                            AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
+  }
+# define AO_HAVE_compare_and_swap
+#endif
+
+/* fetch_and_add */
+/* We first try to implement fetch_and_add variants in terms of the     */
+/* corresponding compare_and_swap variants to minimize adding barriers. */
+#if defined(AO_HAVE_compare_and_swap_full) \
+    && !defined(AO_HAVE_fetch_and_add_full)
+  AO_INLINE AO_t
+  AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
+                                                           old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add_full
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_fetch_and_add_acquire)
+  AO_INLINE AO_t
+  AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add_acquire
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_release) \
+    && !defined(AO_HAVE_fetch_and_add_release)
+  AO_INLINE AO_t
+  AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add_release
+#endif
+
+#if defined(AO_HAVE_compare_and_swap) \
+    && !defined(AO_HAVE_fetch_and_add)
+  AO_INLINE AO_t
+  AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old,
+                                                      old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_fetch_and_add_full)
+# if !defined(AO_HAVE_fetch_and_add_release)
+#   define AO_fetch_and_add_release(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_release
+# endif
+# if !defined(AO_HAVE_fetch_and_add_acquire)
+#   define AO_fetch_and_add_acquire(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_acquire
+# endif
+# if !defined(AO_HAVE_fetch_and_add_write)
+#   define AO_fetch_and_add_write(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_write
+# endif
+# if !defined(AO_HAVE_fetch_and_add_read)
+#   define AO_fetch_and_add_read(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_read
+# endif
+#endif /* AO_HAVE_fetch_and_add_full */
+
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_release)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_release(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_acquire)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_write)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_write(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_read)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_read(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_fetch_and_add_acquire) \
+    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full)
+# define AO_fetch_and_add_full(addr, val) \
+                (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
+# define AO_HAVE_fetch_and_add_full
+#endif
+
+#if !defined(AO_HAVE_fetch_and_add_release_write) \
+    && defined(AO_HAVE_fetch_and_add_write)
+# define AO_fetch_and_add_release_write(addr, val) \
+                                AO_fetch_and_add_write(addr, val)
+# define AO_HAVE_fetch_and_add_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_add_release_write) \
+    && defined(AO_HAVE_fetch_and_add_release)
+# define AO_fetch_and_add_release_write(addr, val) \
+                                AO_fetch_and_add_release(addr, val)
+# define AO_HAVE_fetch_and_add_release_write
+#endif
+
+#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add_read)
+# define AO_fetch_and_add_acquire_read(addr, val) \
+                                AO_fetch_and_add_read(addr, val)
+# define AO_HAVE_fetch_and_add_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add_acquire)
+# define AO_fetch_and_add_acquire_read(addr, val) \
+                                AO_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_fetch_and_add_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_and_add_acquire_read)
+#   define AO_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_fetch_and_add_acquire_read(addr, val)
+#   define AO_HAVE_fetch_and_add_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_and_add)
+#   define AO_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_fetch_and_add(addr, val)
+#   define AO_HAVE_fetch_and_add_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* fetch_and_add1 */
+#if defined(AO_HAVE_fetch_and_add_full) \
+    && !defined(AO_HAVE_fetch_and_add1_full)
+# define AO_fetch_and_add1_full(addr) \
+                                AO_fetch_and_add_full(addr,1)
+# define AO_HAVE_fetch_and_add1_full
+#endif
+#if defined(AO_HAVE_fetch_and_add_release) \
+    && !defined(AO_HAVE_fetch_and_add1_release)
+# define AO_fetch_and_add1_release(addr) \
+                                AO_fetch_and_add_release(addr,1)
+# define AO_HAVE_fetch_and_add1_release
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_fetch_and_add1_acquire)
+# define AO_fetch_and_add1_acquire(addr) \
+                                AO_fetch_and_add_acquire(addr,1)
+# define AO_HAVE_fetch_and_add1_acquire
+#endif
+#if defined(AO_HAVE_fetch_and_add_write) \
+    && !defined(AO_HAVE_fetch_and_add1_write)
+# define AO_fetch_and_add1_write(addr) \
+                                AO_fetch_and_add_write(addr,1)
+# define AO_HAVE_fetch_and_add1_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_read) \
+    && !defined(AO_HAVE_fetch_and_add1_read)
+# define AO_fetch_and_add1_read(addr) \
+                                AO_fetch_and_add_read(addr,1)
+# define AO_HAVE_fetch_and_add1_read
+#endif
+#if defined(AO_HAVE_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_fetch_and_add1_release_write)
+# define AO_fetch_and_add1_release_write(addr) \
+                                AO_fetch_and_add_release_write(addr,1)
+# define AO_HAVE_fetch_and_add1_release_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_fetch_and_add1_acquire_read)
+# define AO_fetch_and_add1_acquire_read(addr) \
+                                AO_fetch_and_add_acquire_read(addr,1)
+# define AO_HAVE_fetch_and_add1_acquire_read
+#endif
+#if defined(AO_HAVE_fetch_and_add) \
+    && !defined(AO_HAVE_fetch_and_add1)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add(addr,1)
+# define AO_HAVE_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_fetch_and_add1_full)
+# if !defined(AO_HAVE_fetch_and_add1_release)
+#   define AO_fetch_and_add1_release(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_release
+# endif
+# if !defined(AO_HAVE_fetch_and_add1_acquire)
+#   define AO_fetch_and_add1_acquire(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_acquire
+# endif
+# if !defined(AO_HAVE_fetch_and_add1_write)
+#   define AO_fetch_and_add1_write(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_write
+# endif
+# if !defined(AO_HAVE_fetch_and_add1_read)
+#   define AO_fetch_and_add1_read(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_read
+# endif
+#endif /* AO_HAVE_fetch_and_add1_full */
+
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_release)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_acquire)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_write)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_read)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_fetch_and_add1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_and_add1_full)
+# define AO_fetch_and_add1_full(addr) \
+                        (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
+# define AO_HAVE_fetch_and_add1_full
+#endif
+
+#if !defined(AO_HAVE_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_fetch_and_add1_write)
+# define AO_fetch_and_add1_release_write(addr) \
+                                AO_fetch_and_add1_write(addr)
+# define AO_HAVE_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_fetch_and_add1_release)
+# define AO_fetch_and_add1_release_write(addr) \
+                                AO_fetch_and_add1_release(addr)
+# define AO_HAVE_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add1_read)
+# define AO_fetch_and_add1_acquire_read(addr) \
+                                AO_fetch_and_add1_read(addr)
+# define AO_HAVE_fetch_and_add1_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add1_acquire)
+# define AO_fetch_and_add1_acquire_read(addr) \
+                                AO_fetch_and_add1_acquire(addr)
+# define AO_HAVE_fetch_and_add1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_and_add1_acquire_read)
+#   define AO_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_fetch_and_add1_acquire_read(addr)
+#   define AO_HAVE_fetch_and_add1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_and_add1)
+#   define AO_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_fetch_and_add1(addr)
+#   define AO_HAVE_fetch_and_add1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* fetch_and_sub1 */
+#if defined(AO_HAVE_fetch_and_add_full) \
+    && !defined(AO_HAVE_fetch_and_sub1_full)
+# define AO_fetch_and_sub1_full(addr) \
+                AO_fetch_and_add_full(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_full
+#endif
+#if defined(AO_HAVE_fetch_and_add_release) \
+    && !defined(AO_HAVE_fetch_and_sub1_release)
+# define AO_fetch_and_sub1_release(addr) \
+                AO_fetch_and_add_release(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_release
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_fetch_and_sub1_acquire)
+# define AO_fetch_and_sub1_acquire(addr) \
+                AO_fetch_and_add_acquire(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_acquire
+#endif
+#if defined(AO_HAVE_fetch_and_add_write) \
+    && !defined(AO_HAVE_fetch_and_sub1_write)
+# define AO_fetch_and_sub1_write(addr) \
+                AO_fetch_and_add_write(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_read) \
+    && !defined(AO_HAVE_fetch_and_sub1_read)
+# define AO_fetch_and_sub1_read(addr) \
+                AO_fetch_and_add_read(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_read
+#endif
+#if defined(AO_HAVE_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_fetch_and_sub1_release_write)
+# define AO_fetch_and_sub1_release_write(addr) \
+                AO_fetch_and_add_release_write(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_release_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_fetch_and_sub1_acquire_read)
+# define AO_fetch_and_sub1_acquire_read(addr) \
+                AO_fetch_and_add_acquire_read(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_acquire_read
+#endif
+#if defined(AO_HAVE_fetch_and_add) \
+    && !defined(AO_HAVE_fetch_and_sub1)
+# define AO_fetch_and_sub1(addr) \
+                AO_fetch_and_add(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_fetch_and_sub1_full)
+# if !defined(AO_HAVE_fetch_and_sub1_release)
+#   define AO_fetch_and_sub1_release(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_release
+# endif
+# if !defined(AO_HAVE_fetch_and_sub1_acquire)
+#   define AO_fetch_and_sub1_acquire(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_acquire
+# endif
+# if !defined(AO_HAVE_fetch_and_sub1_write)
+#   define AO_fetch_and_sub1_write(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_write
+# endif
+# if !defined(AO_HAVE_fetch_and_sub1_read)
+#   define AO_fetch_and_sub1_read(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_read
+# endif
+#endif /* AO_HAVE_fetch_and_sub1_full */
+
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_release)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_acquire)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_write)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_read)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_fetch_and_sub1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_and_sub1_full)
+# define AO_fetch_and_sub1_full(addr) \
+                        (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
+# define AO_HAVE_fetch_and_sub1_full
+#endif
+
+#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_fetch_and_sub1_write)
+# define AO_fetch_and_sub1_release_write(addr) \
+                                AO_fetch_and_sub1_write(addr)
+# define AO_HAVE_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_fetch_and_sub1_release)
+# define AO_fetch_and_sub1_release_write(addr) \
+                                AO_fetch_and_sub1_release(addr)
+# define AO_HAVE_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_sub1_read)
+# define AO_fetch_and_sub1_acquire_read(addr) \
+                                AO_fetch_and_sub1_read(addr)
+# define AO_HAVE_fetch_and_sub1_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_sub1_acquire)
+# define AO_fetch_and_sub1_acquire_read(addr) \
+                                AO_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_fetch_and_sub1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_and_sub1_acquire_read)
+#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_fetch_and_sub1_acquire_read(addr)
+#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_and_sub1)
+#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_fetch_and_sub1(addr)
+#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */

diff --git a/src/atomic_ops/generalize.h b/src/atomic_ops/generalize.h
index bb72c7b..dd6b1b8 100644 (file)
--- a/src/atomic_ops/generalize.h
+++ b/src/atomic_ops/generalize.h
@@ -212,132 +212,6 @@
 # define AO_HAVE_nop_write
 #endif
 
-/* Load */
-#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
-# define AO_load_acquire(addr) AO_load_full(addr)
-# define AO_HAVE_load_acquire
-#endif
-
-#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
-# define AO_load(addr) AO_load_acquire(addr)
-# define AO_HAVE_load
-#endif
-
-#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
-# define AO_load_read(addr) AO_load_full(addr)
-# define AO_HAVE_load_read
-#endif
-
-#if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_acquire)
-# define AO_load_acquire_read(addr) AO_load_acquire(addr)
-# define AO_HAVE_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_load_acquire)
-  AO_INLINE AO_t
-  AO_load_acquire(const volatile AO_t *addr)
-  {
-    AO_t result = AO_load(addr);
-    /* Acquire barrier would be useless, since the load could be delayed */
-    /* beyond it.                                                        */
-    AO_nop_full();
-    return result;
-  }
-# define AO_HAVE_load_acquire
-#endif
-
-#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_load_read)
-  AO_INLINE AO_t
-  AO_load_read(const volatile AO_t *addr)
-  {
-    AO_t result = AO_load(addr);
-    /* Acquire barrier would be useless, since the load could be delayed */
-    /* beyond it.                                                        */
-    AO_nop_read();
-    return result;
-  }
-# define AO_HAVE_load_read
-#endif
-
-#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_load_full)
-# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
-# define AO_HAVE_load_full
-#endif
-
-#if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_read)
-# define AO_load_acquire_read(addr) AO_load_read(addr)
-# define AO_HAVE_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
-# define AO_load(addr) AO_load_acquire_read(addr)
-# define AO_HAVE_load
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_load_acquire_read)
-#   define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
-#   define AO_HAVE_load_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_load)
-#   define AO_load_dd_acquire_read(addr) AO_load(addr)
-#   define AO_HAVE_load_dd_acquire_read
-# endif
-#endif
-
-/* Store */
-#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
-# define AO_store_release(addr,val) AO_store_full(addr,val)
-# define AO_HAVE_store_release
-#endif
-
-#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
-# define AO_store(addr,val) AO_store_release(addr,val)
-# define AO_HAVE_store
-#endif
-
-#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
-# define AO_store_write(addr,val) AO_store_full(addr,val)
-# define AO_HAVE_store_write
-#endif
-
-#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store_release_write)
-# define AO_store_release_write(addr,val) AO_store_release(addr,val)
-# define AO_HAVE_store_release_write
-#endif
-
-#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
-# define AO_store(addr,val) AO_store_write(addr,val)
-# define AO_HAVE_store
-#endif
-
-#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_store_release)
-# define AO_store_release(addr,val) (AO_nop_full(), AO_store(addr,val))
-# define AO_HAVE_store_release
-#endif
-
-#if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) \
-    && !defined(AO_HAVE_store_write)
-# define AO_store_write(addr,val) (AO_nop_write(), AO_store(addr,val))
-# define AO_HAVE_store_write
-#endif
-
-#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store_release_write)
-# define AO_store_release_write(addr,val) AO_store_write(addr,val)
-# define AO_HAVE_store_release_write
-#endif
-
-#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_store_full)
-# define AO_store_full(addr,val) (AO_store_release(addr,val), AO_nop_full())
-# define AO_HAVE_store_full
-#endif
-
 /* NEC LE-IT: Test and set */
 #if defined(AO_HAVE_test_and_set) && defined(AO_HAVE_nop_full) \
     && !defined(AO_HAVE_test_and_set_release)
@@ -357,402 +231,6 @@
 # define AO_HAVE_test_and_set_acquire
 #endif
 
-/* Fetch_and_add */
-/* We first try to implement fetch_and_add variants in terms    */
-/* of the corresponding compare_and_swap variants to minimize   */
-/* adding barriers.                                             */
-#if defined(AO_HAVE_compare_and_swap_full) \
-    && !defined(AO_HAVE_fetch_and_add_full)
-  AO_INLINE AO_t
-  AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old+incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add_full
-#endif
-
-#if defined(AO_HAVE_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_fetch_and_add_acquire)
-  AO_INLINE AO_t
-  AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old, old+incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add_acquire
-#endif
-
-#if defined(AO_HAVE_compare_and_swap_release) \
-    && !defined(AO_HAVE_fetch_and_add_release)
-  AO_INLINE AO_t
-  AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old, old+incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add_release
-#endif
-
-#if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_fetch_and_add)
-  AO_INLINE AO_t
-  AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old, old+incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_fetch_and_add_full)
-# if !defined(AO_HAVE_fetch_and_add_release)
-#   define AO_fetch_and_add_release(addr,val) AO_fetch_and_add_full(addr,val)
-#   define AO_HAVE_fetch_and_add_release
-# endif
-# if !defined(AO_HAVE_fetch_and_add_acquire)
-#   define AO_fetch_and_add_acquire(addr,val) AO_fetch_and_add_full(addr,val)
-#   define AO_HAVE_fetch_and_add_acquire
-# endif
-# if !defined(AO_HAVE_fetch_and_add_write)
-#   define AO_fetch_and_add_write(addr,val) AO_fetch_and_add_full(addr,val)
-#   define AO_HAVE_fetch_and_add_write
-# endif
-# if !defined(AO_HAVE_fetch_and_add_read)
-#   define AO_fetch_and_add_read(addr,val) AO_fetch_and_add_full(addr,val)
-#   define AO_HAVE_fetch_and_add_read
-# endif
-#endif /* AO_HAVE_fetch_and_add_full */
-
-#if !defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_fetch_and_add_release)
-# define AO_fetch_and_add(addr,val) AO_fetch_and_add_release(addr,val)
-# define AO_HAVE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_fetch_and_add_acquire)
-# define AO_fetch_and_add(addr,val) AO_fetch_and_add_acquire(addr,val)
-# define AO_HAVE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_fetch_and_add_write)
-# define AO_fetch_and_add(addr,val) AO_fetch_and_add_write(addr,val)
-# define AO_HAVE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_fetch_and_add_read)
-# define AO_fetch_and_add(addr,val) AO_fetch_and_add_read(addr,val)
-# define AO_HAVE_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_fetch_and_add_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_fetch_and_add_full)
-# define AO_fetch_and_add_full(addr,val) \
-                        (AO_nop_full(), AO_fetch_and_add_acquire(addr,val))
-# define AO_HAVE_fetch_and_add_full
-#endif
-
-#if !defined(AO_HAVE_fetch_and_add_release_write) \
-    && defined(AO_HAVE_fetch_and_add_write)
-# define AO_fetch_and_add_release_write(addr,val) \
-                                        AO_fetch_and_add_write(addr,val)
-# define AO_HAVE_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add_release_write) \
-    && defined(AO_HAVE_fetch_and_add_release)
-# define AO_fetch_and_add_release_write(addr,val) \
-                                        AO_fetch_and_add_release(addr,val)
-# define AO_HAVE_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add_read)
-# define AO_fetch_and_add_acquire_read(addr,val) \
-                                        AO_fetch_and_add_read(addr,val)
-# define AO_HAVE_fetch_and_add_acquire_read
-#endif
-#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add_acquire)
-# define AO_fetch_and_add_acquire_read(addr,val) \
-                                        AO_fetch_and_add_acquire(addr,val)
-# define AO_HAVE_fetch_and_add_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_fetch_and_add_acquire_read)
-#   define AO_fetch_and_add_dd_acquire_read(addr,val) \
-                                AO_fetch_and_add_acquire_read(addr,val)
-#   define AO_HAVE_fetch_and_add_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_fetch_and_add)
-#   define AO_fetch_and_add_dd_acquire_read(addr,val) \
-                                        AO_fetch_and_add(addr,val)
-#   define AO_HAVE_fetch_and_add_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* Fetch_and_add1 */
-
-#if defined(AO_HAVE_fetch_and_add_full) \
-    && !defined(AO_HAVE_fetch_and_add1_full)
-# define AO_fetch_and_add1_full(addr) AO_fetch_and_add_full(addr,1)
-# define AO_HAVE_fetch_and_add1_full
-#endif
-#if defined(AO_HAVE_fetch_and_add_release) \
-    && !defined(AO_HAVE_fetch_and_add1_release)
-# define AO_fetch_and_add1_release(addr) AO_fetch_and_add_release(addr,1)
-# define AO_HAVE_fetch_and_add1_release
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_fetch_and_add1_acquire)
-# define AO_fetch_and_add1_acquire(addr) AO_fetch_and_add_acquire(addr,1)
-# define AO_HAVE_fetch_and_add1_acquire
-#endif
-#if defined(AO_HAVE_fetch_and_add_write) \
-    && !defined(AO_HAVE_fetch_and_add1_write)
-# define AO_fetch_and_add1_write(addr) AO_fetch_and_add_write(addr,1)
-# define AO_HAVE_fetch_and_add1_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_read) \
-    && !defined(AO_HAVE_fetch_and_add1_read)
-# define AO_fetch_and_add1_read(addr) AO_fetch_and_add_read(addr,1)
-# define AO_HAVE_fetch_and_add1_read
-#endif
-#if defined(AO_HAVE_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_fetch_and_add1_release_write)
-# define AO_fetch_and_add1_release_write(addr) \
-                                        AO_fetch_and_add_release_write(addr,1)
-# define AO_HAVE_fetch_and_add1_release_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_fetch_and_add1_acquire_read)
-# define AO_fetch_and_add1_acquire_read(addr) \
-                                        AO_fetch_and_add_acquire_read(addr,1)
-# define AO_HAVE_fetch_and_add1_acquire_read
-#endif
-#if defined(AO_HAVE_fetch_and_add) && !defined(AO_HAVE_fetch_and_add1)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add(addr,1)
-# define AO_HAVE_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_fetch_and_add1_full)
-# if !defined(AO_HAVE_fetch_and_add1_release)
-#   define AO_fetch_and_add1_release(addr) AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_release
-# endif
-# if !defined(AO_HAVE_fetch_and_add1_acquire)
-#   define AO_fetch_and_add1_acquire(addr) AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_acquire
-# endif
-# if !defined(AO_HAVE_fetch_and_add1_write)
-#   define AO_fetch_and_add1_write(addr) AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_write
-# endif
-# if !defined(AO_HAVE_fetch_and_add1_read)
-#   define AO_fetch_and_add1_read(addr) AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_read
-# endif
-#endif /* AO_HAVE_fetch_and_add1_full */
-
-#if !defined(AO_HAVE_fetch_and_add1) \
-    && defined(AO_HAVE_fetch_and_add1_release)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_fetch_and_add1) \
-    && defined(AO_HAVE_fetch_and_add1_acquire)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_add1_write)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_add1_read)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_fetch_and_add1_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_fetch_and_add1_full)
-# define AO_fetch_and_add1_full(addr) \
-                        (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
-# define AO_HAVE_fetch_and_add1_full
-#endif
-
-#if !defined(AO_HAVE_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_fetch_and_add1_write)
-# define AO_fetch_and_add1_release_write(addr) AO_fetch_and_add1_write(addr)
-# define AO_HAVE_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_fetch_and_add1_release)
-# define AO_fetch_and_add1_release_write(addr) AO_fetch_and_add1_release(addr)
-# define AO_HAVE_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add1_read)
-# define AO_fetch_and_add1_acquire_read(addr) AO_fetch_and_add1_read(addr)
-# define AO_HAVE_fetch_and_add1_acquire_read
-#endif
-#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add1_acquire)
-# define AO_fetch_and_add1_acquire_read(addr) AO_fetch_and_add1_acquire(addr)
-# define AO_HAVE_fetch_and_add1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_fetch_and_add1_acquire_read)
-#   define AO_fetch_and_add1_dd_acquire_read(addr) \
-                                        AO_fetch_and_add1_acquire_read(addr)
-#   define AO_HAVE_fetch_and_add1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_fetch_and_add1)
-#   define AO_fetch_and_add1_dd_acquire_read(addr) AO_fetch_and_add1(addr)
-#   define AO_HAVE_fetch_and_add1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* Fetch_and_sub1 */
-#if defined(AO_HAVE_fetch_and_add_full) \
-    && !defined(AO_HAVE_fetch_and_sub1_full)
-# define AO_fetch_and_sub1_full(addr) AO_fetch_and_add_full(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_full
-#endif
-#if defined(AO_HAVE_fetch_and_add_release) \
-    && !defined(AO_HAVE_fetch_and_sub1_release)
-# define AO_fetch_and_sub1_release(addr) \
-                                AO_fetch_and_add_release(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_release
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_fetch_and_sub1_acquire)
-# define AO_fetch_and_sub1_acquire(addr) \
-                                AO_fetch_and_add_acquire(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_acquire
-#endif
-#if defined(AO_HAVE_fetch_and_add_write) \
-    && !defined(AO_HAVE_fetch_and_sub1_write)
-# define AO_fetch_and_sub1_write(addr) AO_fetch_and_add_write(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_read) \
-    && !defined(AO_HAVE_fetch_and_sub1_read)
-# define AO_fetch_and_sub1_read(addr) AO_fetch_and_add_read(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_read
-#endif
-#if defined(AO_HAVE_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_fetch_and_sub1_release_write)
-# define AO_fetch_and_sub1_release_write(addr) \
-                        AO_fetch_and_add_release_write(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_release_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_fetch_and_sub1_acquire_read)
-# define AO_fetch_and_sub1_acquire_read(addr) \
-                        AO_fetch_and_add_acquire_read(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_acquire_read
-#endif
-#if defined(AO_HAVE_fetch_and_add) && !defined(AO_HAVE_fetch_and_sub1)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_add(addr,(AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_fetch_and_sub1_full)
-# if !defined(AO_HAVE_fetch_and_sub1_release)
-#   define AO_fetch_and_sub1_release(addr) AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_release
-# endif
-# if !defined(AO_HAVE_fetch_and_sub1_acquire)
-#   define AO_fetch_and_sub1_acquire(addr) AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_acquire
-# endif
-# if !defined(AO_HAVE_fetch_and_sub1_write)
-#   define AO_fetch_and_sub1_write(addr) AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_write
-# endif
-# if !defined(AO_HAVE_fetch_and_sub1_read)
-#   define AO_fetch_and_sub1_read(addr) AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_read
-# endif
-#endif /* AO_HAVE_fetch_and_sub1_full */
-
-#if !defined(AO_HAVE_fetch_and_sub1) \
-    && defined(AO_HAVE_fetch_and_sub1_release)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1) \
-    && defined(AO_HAVE_fetch_and_sub1_acquire)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1) && defined(AO_HAVE_fetch_and_sub1_write)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1) && defined(AO_HAVE_fetch_and_sub1_read)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_fetch_and_sub1_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_fetch_and_sub1_full)
-# define AO_fetch_and_sub1_full(addr) \
-                        (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
-# define AO_HAVE_fetch_and_sub1_full
-#endif
-
-#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_fetch_and_sub1_write)
-# define AO_fetch_and_sub1_release_write(addr) AO_fetch_and_sub1_write(addr)
-# define AO_HAVE_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_fetch_and_sub1_release)
-# define AO_fetch_and_sub1_release_write(addr) AO_fetch_and_sub1_release(addr)
-# define AO_HAVE_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_sub1_read)
-# define AO_fetch_and_sub1_acquire_read(addr) AO_fetch_and_sub1_read(addr)
-# define AO_HAVE_fetch_and_sub1_acquire_read
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_sub1_acquire)
-# define AO_fetch_and_sub1_acquire_read(addr) AO_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_fetch_and_sub1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_fetch_and_sub1_acquire_read)
-#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
-                                        AO_fetch_and_sub1_acquire_read(addr)
-#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_fetch_and_sub1)
-#   define AO_fetch_and_sub1_dd_acquire_read(addr) AO_fetch_and_sub1(addr)
-#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
 /* Atomic "and" */
 #if defined(AO_HAVE_compare_and_swap_full) && !defined(AO_HAVE_and_full)
   AO_INLINE void