Move generalized arithmetical primitives to 'generalize-arithm' template

(code refactoring)

* src/Makefile.am (EXTRA_DIST): Add generalize-arithm.template.
* src/Makefile.am (nobase_private_HEADERS): Add generalize-arithm.h.
* src/Makefile.am (atomic_ops/generalize-arithm.h): New rule (similar
as generalize-small.h but using generalize-arithm.template).
* src/atomic_ops/generalize-arithm.template: New file.
* src/atomic_ops/generalize-small.template: Move compare_and_swap
(only which is based on fetch_compare_and_swap), fetch_and_add,
fetch_and_add, fetch_and_add1, fetch_and_sub1, and/or/xor template
primitives to generalize-arithm.template).
* src/atomic_ops/generalize-arithm.h: Regenerate (new file).
* src/atomic_ops/generalize-small.h: Regenerate.
* src/atomic_ops/generalize.h: Include generalize-arithm.h (just after
inclusion of generalize-small.h).

diff --git a/src/Makefile.am b/src/Makefile.am
index 8f187cf..c6f951a 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -14,12 +14,14 @@ libatomic_ops_gpl_la_SOURCES = atomic_ops_stack.c atomic_ops_malloc.c
 libatomic_ops_gpl_la_LDFLAGS = -version-info 1:3:0 -no-undefined
 
 EXTRA_DIST=Makefile.msft atomic_ops/sysdeps/README \
+        atomic_ops/generalize-arithm.template \
         atomic_ops/generalize-small.template atomic_ops/sysdeps/sunc/sparc.S
 
 #Private Headers
 privatedir=${includedir}/
 nobase_private_HEADERS = atomic_ops/ao_version.h \
           atomic_ops/generalize.h \
+          atomic_ops/generalize-arithm.h \
           atomic_ops/generalize-small.h \
         \
           atomic_ops/sysdeps/acquire_release_volatile.h \
@@ -83,3 +85,9 @@ atomic_ops/generalize-small.h: atomic_ops/generalize-small.template
        sed -e s:XSIZE_:short_:g -e s:XCTYPE:unsigned/**/short:g $? >> $@
        sed -e s:XSIZE_:int_:g -e s:XCTYPE:unsigned:g $? >> $@
        sed -e s:XSIZE_::g -e s:XCTYPE:AO_t:g $? >> $@
+
+atomic_ops/generalize-arithm.h: atomic_ops/generalize-arithm.template
+       sed -e s:XSIZE_:char_:g -e s:XCTYPE:unsigned/**/char:g $? > $@
+       sed -e s:XSIZE_:short_:g -e s:XCTYPE:unsigned/**/short:g $? >> $@
+       sed -e s:XSIZE_:int_:g -e s:XCTYPE:unsigned:g $? >> $@
+       sed -e s:XSIZE_::g -e s:XCTYPE:AO_t:g $? >> $@

diff --git a/src/atomic_ops/generalize-arithm.h b/src/atomic_ops/generalize-arithm.h
new file mode 100644 (file)
index 0000000..00d35a8
--- /dev/null
+++ b/src/atomic_ops/generalize-arithm.h
@@ -0,0 +1,3068 @@
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* char_compare_and_swap (based on fetch_compare_and_swap) */
+#if defined(AO_HAVE_char_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_char_compare_and_swap_full)
+  AO_INLINE int
+  AO_char_compare_and_swap_full(volatile unsigned/**/char *addr, unsigned/**/char old_val,
+                                 unsigned/**/char new_val)
+  {
+    return AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_char_compare_and_swap_full
+#endif
+
+#if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_char_compare_and_swap_acquire)
+  AO_INLINE int
+  AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old_val,
+                                    unsigned/**/char new_val)
+  {
+    return AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_char_compare_and_swap_acquire
+#endif
+
+#if defined(AO_HAVE_char_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_char_compare_and_swap_release)
+  AO_INLINE int
+  AO_char_compare_and_swap_release(volatile unsigned/**/char *addr, unsigned/**/char old_val,
+                                    unsigned/**/char new_val)
+  {
+    return AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_char_compare_and_swap_release
+#endif
+
+#if defined(AO_HAVE_char_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_char_compare_and_swap)
+  AO_INLINE int
+  AO_char_compare_and_swap(volatile unsigned/**/char *addr, unsigned/**/char old_val,
+                            unsigned/**/char new_val)
+  {
+    return AO_char_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
+  }
+# define AO_HAVE_char_compare_and_swap
+#endif
+
+/* char_fetch_and_add */
+/* We first try to implement fetch_and_add variants in terms of the     */
+/* corresponding compare_and_swap variants to minimize adding barriers. */
+#if defined(AO_HAVE_char_compare_and_swap_full) \
+    && !defined(AO_HAVE_char_fetch_and_add_full)
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_and_add_full(volatile unsigned/**/char *addr, unsigned/**/char incr)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
+                                                           old + incr)));
+    return old;
+  }
+# define AO_HAVE_char_fetch_and_add_full
+#endif
+
+#if defined(AO_HAVE_char_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_char_fetch_and_add_acquire)
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_char_fetch_and_add_acquire
+#endif
+
+#if defined(AO_HAVE_char_compare_and_swap_release) \
+    && !defined(AO_HAVE_char_fetch_and_add_release)
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_and_add_release(volatile unsigned/**/char *addr, unsigned/**/char incr)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_char_fetch_and_add_release
+#endif
+
+#if defined(AO_HAVE_char_compare_and_swap) \
+    && !defined(AO_HAVE_char_fetch_and_add)
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_and_add(volatile unsigned/**/char *addr, unsigned/**/char incr)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr, old,
+                                                      old + incr)));
+    return old;
+  }
+# define AO_HAVE_char_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_char_fetch_and_add_full)
+# if !defined(AO_HAVE_char_fetch_and_add_release)
+#   define AO_char_fetch_and_add_release(addr, val) \
+                                AO_char_fetch_and_add_full(addr, val)
+#   define AO_HAVE_char_fetch_and_add_release
+# endif
+# if !defined(AO_HAVE_char_fetch_and_add_acquire)
+#   define AO_char_fetch_and_add_acquire(addr, val) \
+                                AO_char_fetch_and_add_full(addr, val)
+#   define AO_HAVE_char_fetch_and_add_acquire
+# endif
+# if !defined(AO_HAVE_char_fetch_and_add_write)
+#   define AO_char_fetch_and_add_write(addr, val) \
+                                AO_char_fetch_and_add_full(addr, val)
+#   define AO_HAVE_char_fetch_and_add_write
+# endif
+# if !defined(AO_HAVE_char_fetch_and_add_read)
+#   define AO_char_fetch_and_add_read(addr, val) \
+                                AO_char_fetch_and_add_full(addr, val)
+#   define AO_HAVE_char_fetch_and_add_read
+# endif
+#endif /* AO_HAVE_char_fetch_and_add_full */
+
+#if !defined(AO_HAVE_char_fetch_and_add) \
+    && defined(AO_HAVE_char_fetch_and_add_release)
+# define AO_char_fetch_and_add(addr, val) \
+                                AO_char_fetch_and_add_release(addr, val)
+# define AO_HAVE_char_fetch_and_add
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add) \
+    && defined(AO_HAVE_char_fetch_and_add_acquire)
+# define AO_char_fetch_and_add(addr, val) \
+                                AO_char_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_char_fetch_and_add
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add) \
+    && defined(AO_HAVE_char_fetch_and_add_write)
+# define AO_char_fetch_and_add(addr, val) \
+                                AO_char_fetch_and_add_write(addr, val)
+# define AO_HAVE_char_fetch_and_add
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add) \
+    && defined(AO_HAVE_char_fetch_and_add_read)
+# define AO_char_fetch_and_add(addr, val) \
+                                AO_char_fetch_and_add_read(addr, val)
+# define AO_HAVE_char_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_char_fetch_and_add_acquire) \
+    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_char_fetch_and_add_full)
+# define AO_char_fetch_and_add_full(addr, val) \
+                (AO_nop_full(), AO_char_fetch_and_add_acquire(addr, val))
+# define AO_HAVE_char_fetch_and_add_full
+#endif
+
+#if !defined(AO_HAVE_char_fetch_and_add_release_write) \
+    && defined(AO_HAVE_char_fetch_and_add_write)
+# define AO_char_fetch_and_add_release_write(addr, val) \
+                                AO_char_fetch_and_add_write(addr, val)
+# define AO_HAVE_char_fetch_and_add_release_write
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add_release_write) \
+    && defined(AO_HAVE_char_fetch_and_add_release)
+# define AO_char_fetch_and_add_release_write(addr, val) \
+                                AO_char_fetch_and_add_release(addr, val)
+# define AO_HAVE_char_fetch_and_add_release_write
+#endif
+
+#if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_char_fetch_and_add_read)
+# define AO_char_fetch_and_add_acquire_read(addr, val) \
+                                AO_char_fetch_and_add_read(addr, val)
+# define AO_HAVE_char_fetch_and_add_acquire_read
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_char_fetch_and_add_acquire)
+# define AO_char_fetch_and_add_acquire_read(addr, val) \
+                                AO_char_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_char_fetch_and_add_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_char_fetch_and_add_acquire_read)
+#   define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_char_fetch_and_add_acquire_read(addr, val)
+#   define AO_HAVE_char_fetch_and_add_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_char_fetch_and_add)
+#   define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_char_fetch_and_add(addr, val)
+#   define AO_HAVE_char_fetch_and_add_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* char_fetch_and_add1 */
+#if defined(AO_HAVE_char_fetch_and_add_full) \
+    && !defined(AO_HAVE_char_fetch_and_add1_full)
+# define AO_char_fetch_and_add1_full(addr) \
+                                AO_char_fetch_and_add_full(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_full
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_release) \
+    && !defined(AO_HAVE_char_fetch_and_add1_release)
+# define AO_char_fetch_and_add1_release(addr) \
+                                AO_char_fetch_and_add_release(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_release
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_char_fetch_and_add1_acquire)
+# define AO_char_fetch_and_add1_acquire(addr) \
+                                AO_char_fetch_and_add_acquire(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_acquire
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_write) \
+    && !defined(AO_HAVE_char_fetch_and_add1_write)
+# define AO_char_fetch_and_add1_write(addr) \
+                                AO_char_fetch_and_add_write(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_write
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_read) \
+    && !defined(AO_HAVE_char_fetch_and_add1_read)
+# define AO_char_fetch_and_add1_read(addr) \
+                                AO_char_fetch_and_add_read(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_read
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_char_fetch_and_add1_release_write)
+# define AO_char_fetch_and_add1_release_write(addr) \
+                                AO_char_fetch_and_add_release_write(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_release_write
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_char_fetch_and_add1_acquire_read)
+# define AO_char_fetch_and_add1_acquire_read(addr) \
+                                AO_char_fetch_and_add_acquire_read(addr, 1)
+# define AO_HAVE_char_fetch_and_add1_acquire_read
+#endif
+#if defined(AO_HAVE_char_fetch_and_add) \
+    && !defined(AO_HAVE_char_fetch_and_add1)
+# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add(addr, 1)
+# define AO_HAVE_char_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_char_fetch_and_add1_full)
+# if !defined(AO_HAVE_char_fetch_and_add1_release)
+#   define AO_char_fetch_and_add1_release(addr) \
+                                AO_char_fetch_and_add1_full(addr)
+#   define AO_HAVE_char_fetch_and_add1_release
+# endif
+# if !defined(AO_HAVE_char_fetch_and_add1_acquire)
+#   define AO_char_fetch_and_add1_acquire(addr) \
+                                AO_char_fetch_and_add1_full(addr)
+#   define AO_HAVE_char_fetch_and_add1_acquire
+# endif
+# if !defined(AO_HAVE_char_fetch_and_add1_write)
+#   define AO_char_fetch_and_add1_write(addr) \
+                                AO_char_fetch_and_add1_full(addr)
+#   define AO_HAVE_char_fetch_and_add1_write
+# endif
+# if !defined(AO_HAVE_char_fetch_and_add1_read)
+#   define AO_char_fetch_and_add1_read(addr) \
+                                AO_char_fetch_and_add1_full(addr)
+#   define AO_HAVE_char_fetch_and_add1_read
+# endif
+#endif /* AO_HAVE_char_fetch_and_add1_full */
+
+#if !defined(AO_HAVE_char_fetch_and_add1) \
+    && defined(AO_HAVE_char_fetch_and_add1_release)
+# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_release(addr)
+# define AO_HAVE_char_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add1) \
+    && defined(AO_HAVE_char_fetch_and_add1_acquire)
+# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_acquire(addr)
+# define AO_HAVE_char_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add1) \
+    && defined(AO_HAVE_char_fetch_and_add1_write)
+# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_write(addr)
+# define AO_HAVE_char_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add1) \
+    && defined(AO_HAVE_char_fetch_and_add1_read)
+# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_read(addr)
+# define AO_HAVE_char_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_char_fetch_and_add1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_fetch_and_add1_full)
+# define AO_char_fetch_and_add1_full(addr) \
+                        (AO_nop_full(), AO_char_fetch_and_add1_acquire(addr))
+# define AO_HAVE_char_fetch_and_add1_full
+#endif
+
+#if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_char_fetch_and_add1_write)
+# define AO_char_fetch_and_add1_release_write(addr) \
+                                AO_char_fetch_and_add1_write(addr)
+# define AO_HAVE_char_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_char_fetch_and_add1_release)
+# define AO_char_fetch_and_add1_release_write(addr) \
+                                AO_char_fetch_and_add1_release(addr)
+# define AO_HAVE_char_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_char_fetch_and_add1_read)
+# define AO_char_fetch_and_add1_acquire_read(addr) \
+                                AO_char_fetch_and_add1_read(addr)
+# define AO_HAVE_char_fetch_and_add1_acquire_read
+#endif
+#if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_char_fetch_and_add1_acquire)
+# define AO_char_fetch_and_add1_acquire_read(addr) \
+                                AO_char_fetch_and_add1_acquire(addr)
+# define AO_HAVE_char_fetch_and_add1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_char_fetch_and_add1_acquire_read)
+#   define AO_char_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_char_fetch_and_add1_acquire_read(addr)
+#   define AO_HAVE_char_fetch_and_add1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_char_fetch_and_add1)
+#   define AO_char_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_char_fetch_and_add1(addr)
+#   define AO_HAVE_char_fetch_and_add1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* char_fetch_and_sub1 */
+#if defined(AO_HAVE_char_fetch_and_add_full) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_full)
+# define AO_char_fetch_and_sub1_full(addr) \
+                AO_char_fetch_and_add_full(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_full
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_release) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_release)
+# define AO_char_fetch_and_sub1_release(addr) \
+                AO_char_fetch_and_add_release(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_release
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_acquire)
+# define AO_char_fetch_and_sub1_acquire(addr) \
+                AO_char_fetch_and_add_acquire(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_acquire
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_write) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_write)
+# define AO_char_fetch_and_sub1_write(addr) \
+                AO_char_fetch_and_add_write(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_write
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_read) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_read)
+# define AO_char_fetch_and_sub1_read(addr) \
+                AO_char_fetch_and_add_read(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_read
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_release_write)
+# define AO_char_fetch_and_sub1_release_write(addr) \
+                AO_char_fetch_and_add_release_write(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_release_write
+#endif
+#if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
+# define AO_char_fetch_and_sub1_acquire_read(addr) \
+                AO_char_fetch_and_add_acquire_read(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1_acquire_read
+#endif
+#if defined(AO_HAVE_char_fetch_and_add) \
+    && !defined(AO_HAVE_char_fetch_and_sub1)
+# define AO_char_fetch_and_sub1(addr) \
+                AO_char_fetch_and_add(addr, (unsigned/**/char)(-1))
+# define AO_HAVE_char_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_char_fetch_and_sub1_full)
+# if !defined(AO_HAVE_char_fetch_and_sub1_release)
+#   define AO_char_fetch_and_sub1_release(addr) \
+                                AO_char_fetch_and_sub1_full(addr)
+#   define AO_HAVE_char_fetch_and_sub1_release
+# endif
+# if !defined(AO_HAVE_char_fetch_and_sub1_acquire)
+#   define AO_char_fetch_and_sub1_acquire(addr) \
+                                AO_char_fetch_and_sub1_full(addr)
+#   define AO_HAVE_char_fetch_and_sub1_acquire
+# endif
+# if !defined(AO_HAVE_char_fetch_and_sub1_write)
+#   define AO_char_fetch_and_sub1_write(addr) \
+                                AO_char_fetch_and_sub1_full(addr)
+#   define AO_HAVE_char_fetch_and_sub1_write
+# endif
+# if !defined(AO_HAVE_char_fetch_and_sub1_read)
+#   define AO_char_fetch_and_sub1_read(addr) \
+                                AO_char_fetch_and_sub1_full(addr)
+#   define AO_HAVE_char_fetch_and_sub1_read
+# endif
+#endif /* AO_HAVE_char_fetch_and_sub1_full */
+
+#if !defined(AO_HAVE_char_fetch_and_sub1) \
+    && defined(AO_HAVE_char_fetch_and_sub1_release)
+# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_release(addr)
+# define AO_HAVE_char_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_char_fetch_and_sub1) \
+    && defined(AO_HAVE_char_fetch_and_sub1_acquire)
+# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_char_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_char_fetch_and_sub1) \
+    && defined(AO_HAVE_char_fetch_and_sub1_write)
+# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_write(addr)
+# define AO_HAVE_char_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_char_fetch_and_sub1) \
+    && defined(AO_HAVE_char_fetch_and_sub1_read)
+# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_read(addr)
+# define AO_HAVE_char_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_char_fetch_and_sub1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_fetch_and_sub1_full)
+# define AO_char_fetch_and_sub1_full(addr) \
+                        (AO_nop_full(), AO_char_fetch_and_sub1_acquire(addr))
+# define AO_HAVE_char_fetch_and_sub1_full
+#endif
+
+#if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_char_fetch_and_sub1_write)
+# define AO_char_fetch_and_sub1_release_write(addr) \
+                                AO_char_fetch_and_sub1_write(addr)
+# define AO_HAVE_char_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_char_fetch_and_sub1_release)
+# define AO_char_fetch_and_sub1_release_write(addr) \
+                                AO_char_fetch_and_sub1_release(addr)
+# define AO_HAVE_char_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_char_fetch_and_sub1_read)
+# define AO_char_fetch_and_sub1_acquire_read(addr) \
+                                AO_char_fetch_and_sub1_read(addr)
+# define AO_HAVE_char_fetch_and_sub1_acquire_read
+#endif
+#if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_char_fetch_and_sub1_acquire)
+# define AO_char_fetch_and_sub1_acquire_read(addr) \
+                                AO_char_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_char_fetch_and_sub1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
+#   define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_char_fetch_and_sub1_acquire_read(addr)
+#   define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_char_fetch_and_sub1)
+#   define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_char_fetch_and_sub1(addr)
+#   define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* char_and */
+#if defined(AO_HAVE_char_compare_and_swap_full) \
+    && !defined(AO_HAVE_char_and_full)
+  AO_INLINE void
+  AO_char_and_full(volatile unsigned/**/char *addr, unsigned/**/char value)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
+                                                           old & value)));
+  }
+# define AO_HAVE_char_and_full
+#endif
+
+#if defined(AO_HAVE_char_and_full)
+# if !defined(AO_HAVE_char_and_release)
+#   define AO_char_and_release(addr, val) AO_char_and_full(addr, val)
+#   define AO_HAVE_char_and_release
+# endif
+# if !defined(AO_HAVE_char_and_acquire)
+#   define AO_char_and_acquire(addr, val) AO_char_and_full(addr, val)
+#   define AO_HAVE_char_and_acquire
+# endif
+# if !defined(AO_HAVE_char_and_write)
+#   define AO_char_and_write(addr, val) AO_char_and_full(addr, val)
+#   define AO_HAVE_char_and_write
+# endif
+# if !defined(AO_HAVE_char_and_read)
+#   define AO_char_and_read(addr, val) AO_char_and_full(addr, val)
+#   define AO_HAVE_char_and_read
+# endif
+#endif /* AO_HAVE_char_and_full */
+
+#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_release)
+# define AO_char_and(addr, val) AO_char_and_release(addr, val)
+# define AO_HAVE_char_and
+#endif
+#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_acquire)
+# define AO_char_and(addr, val) AO_char_and_acquire(addr, val)
+# define AO_HAVE_char_and
+#endif
+#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_write)
+# define AO_char_and(addr, val) AO_char_and_write(addr, val)
+# define AO_HAVE_char_and
+#endif
+#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_read)
+# define AO_char_and(addr, val) AO_char_and_read(addr, val)
+# define AO_HAVE_char_and
+#endif
+
+#if defined(AO_HAVE_char_and_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_and_full)
+# define AO_char_and_full(addr, val) \
+                        (AO_nop_full(), AO_char_and_acquire(addr, val))
+# define AO_HAVE_char_and_full
+#endif
+
+#if !defined(AO_HAVE_char_and_release_write) \
+    && defined(AO_HAVE_char_and_write)
+# define AO_char_and_release_write(addr, val) AO_char_and_write(addr, val)
+# define AO_HAVE_char_and_release_write
+#endif
+#if !defined(AO_HAVE_char_and_release_write) \
+    && defined(AO_HAVE_char_and_release)
+# define AO_char_and_release_write(addr, val) AO_char_and_release(addr, val)
+# define AO_HAVE_char_and_release_write
+#endif
+#if !defined(AO_HAVE_char_and_acquire_read) \
+    && defined(AO_HAVE_char_and_read)
+# define AO_char_and_acquire_read(addr, val) AO_char_and_read(addr, val)
+# define AO_HAVE_char_and_acquire_read
+#endif
+#if !defined(AO_HAVE_char_and_acquire_read) \
+    && defined(AO_HAVE_char_and_acquire)
+# define AO_char_and_acquire_read(addr, val) AO_char_and_acquire(addr, val)
+# define AO_HAVE_char_and_acquire_read
+#endif
+
+/* char_or */
+#if defined(AO_HAVE_char_compare_and_swap_full) \
+    && !defined(AO_HAVE_char_or_full)
+  AO_INLINE void
+  AO_char_or_full(volatile unsigned/**/char *addr, unsigned/**/char value)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
+                                                           old | value)));
+  }
+# define AO_HAVE_char_or_full
+#endif
+
+#if defined(AO_HAVE_char_or_full)
+# if !defined(AO_HAVE_char_or_release)
+#   define AO_char_or_release(addr, val) AO_char_or_full(addr, val)
+#   define AO_HAVE_char_or_release
+# endif
+# if !defined(AO_HAVE_char_or_acquire)
+#   define AO_char_or_acquire(addr, val) AO_char_or_full(addr, val)
+#   define AO_HAVE_char_or_acquire
+# endif
+# if !defined(AO_HAVE_char_or_write)
+#   define AO_char_or_write(addr, val) AO_char_or_full(addr, val)
+#   define AO_HAVE_char_or_write
+# endif
+# if !defined(AO_HAVE_char_or_read)
+#   define AO_char_or_read(addr, val) AO_char_or_full(addr, val)
+#   define AO_HAVE_char_or_read
+# endif
+#endif /* AO_HAVE_char_or_full */
+
+#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_release)
+# define AO_char_or(addr, val) AO_char_or_release(addr, val)
+# define AO_HAVE_char_or
+#endif
+#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_acquire)
+# define AO_char_or(addr, val) AO_char_or_acquire(addr, val)
+# define AO_HAVE_char_or
+#endif
+#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_write)
+# define AO_char_or(addr, val) AO_char_or_write(addr, val)
+# define AO_HAVE_char_or
+#endif
+#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_read)
+# define AO_char_or(addr, val) AO_char_or_read(addr, val)
+# define AO_HAVE_char_or
+#endif
+
+#if defined(AO_HAVE_char_or_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_or_full)
+# define AO_char_or_full(addr, val) \
+                        (AO_nop_full(), AO_char_or_acquire(addr, val))
+# define AO_HAVE_char_or_full
+#endif
+
+#if !defined(AO_HAVE_char_or_release_write) \
+    && defined(AO_HAVE_char_or_write)
+# define AO_char_or_release_write(addr, val) AO_char_or_write(addr, val)
+# define AO_HAVE_char_or_release_write
+#endif
+#if !defined(AO_HAVE_char_or_release_write) \
+    && defined(AO_HAVE_char_or_release)
+# define AO_char_or_release_write(addr, val) AO_char_or_release(addr, val)
+# define AO_HAVE_char_or_release_write
+#endif
+#if !defined(AO_HAVE_char_or_acquire_read) && defined(AO_HAVE_char_or_read)
+# define AO_char_or_acquire_read(addr, val) AO_char_or_read(addr, val)
+# define AO_HAVE_char_or_acquire_read
+#endif
+#if !defined(AO_HAVE_char_or_acquire_read) \
+    && defined(AO_HAVE_char_or_acquire)
+# define AO_char_or_acquire_read(addr, val) AO_char_or_acquire(addr, val)
+# define AO_HAVE_char_or_acquire_read
+#endif
+
+/* char_xor */
+#if defined(AO_HAVE_char_compare_and_swap_full) \
+    && !defined(AO_HAVE_char_xor_full)
+  AO_INLINE void
+  AO_char_xor_full(volatile unsigned/**/char *addr, unsigned/**/char value)
+  {
+    unsigned/**/char old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
+                                                           old ^ value)));
+  }
+# define AO_HAVE_char_xor_full
+#endif
+
+#if defined(AO_HAVE_char_xor_full)
+# if !defined(AO_HAVE_char_xor_release)
+#   define AO_char_xor_release(addr, val) AO_char_xor_full(addr, val)
+#   define AO_HAVE_char_xor_release
+# endif
+# if !defined(AO_HAVE_char_xor_acquire)
+#   define AO_char_xor_acquire(addr, val) AO_char_xor_full(addr, val)
+#   define AO_HAVE_char_xor_acquire
+# endif
+# if !defined(AO_HAVE_char_xor_write)
+#   define AO_char_xor_write(addr, val) AO_char_xor_full(addr, val)
+#   define AO_HAVE_char_xor_write
+# endif
+# if !defined(AO_HAVE_char_xor_read)
+#   define AO_char_xor_read(addr, val) AO_char_xor_full(addr, val)
+#   define AO_HAVE_char_xor_read
+# endif
+#endif /* AO_HAVE_char_xor_full */
+
+#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_release)
+# define AO_char_xor(addr, val) AO_char_xor_release(addr, val)
+# define AO_HAVE_char_xor
+#endif
+#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_acquire)
+# define AO_char_xor(addr, val) AO_char_xor_acquire(addr, val)
+# define AO_HAVE_char_xor
+#endif
+#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_write)
+# define AO_char_xor(addr, val) AO_char_xor_write(addr, val)
+# define AO_HAVE_char_xor
+#endif
+#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_read)
+# define AO_char_xor(addr, val) AO_char_xor_read(addr, val)
+# define AO_HAVE_char_xor
+#endif
+
+#if defined(AO_HAVE_char_xor_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_char_xor_full)
+# define AO_char_xor_full(addr, val) \
+                        (AO_nop_full(), AO_char_xor_acquire(addr, val))
+# define AO_HAVE_char_xor_full
+#endif
+
+#if !defined(AO_HAVE_char_xor_release_write) \
+    && defined(AO_HAVE_char_xor_write)
+# define AO_char_xor_release_write(addr, val) AO_char_xor_write(addr, val)
+# define AO_HAVE_char_xor_release_write
+#endif
+#if !defined(AO_HAVE_char_xor_release_write) \
+    && defined(AO_HAVE_char_xor_release)
+# define AO_char_xor_release_write(addr, val) AO_char_xor_release(addr, val)
+# define AO_HAVE_char_xor_release_write
+#endif
+#if !defined(AO_HAVE_char_xor_acquire_read) \
+    && defined(AO_HAVE_char_xor_read)
+# define AO_char_xor_acquire_read(addr, val) AO_char_xor_read(addr, val)
+# define AO_HAVE_char_xor_acquire_read
+#endif
+#if !defined(AO_HAVE_char_xor_acquire_read) \
+    && defined(AO_HAVE_char_xor_acquire)
+# define AO_char_xor_acquire_read(addr, val) AO_char_xor_acquire(addr, val)
+# define AO_HAVE_char_xor_acquire_read
+#endif
+
+/* char_and/or/xor_dd_aquire_read are meaningless.     */
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* short_compare_and_swap (based on fetch_compare_and_swap) */
+#if defined(AO_HAVE_short_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_short_compare_and_swap_full)
+  AO_INLINE int
+  AO_short_compare_and_swap_full(volatile unsigned/**/short *addr, unsigned/**/short old_val,
+                                 unsigned/**/short new_val)
+  {
+    return AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_short_compare_and_swap_full
+#endif
+
+#if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_short_compare_and_swap_acquire)
+  AO_INLINE int
+  AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old_val,
+                                    unsigned/**/short new_val)
+  {
+    return AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_short_compare_and_swap_acquire
+#endif
+
+#if defined(AO_HAVE_short_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_short_compare_and_swap_release)
+  AO_INLINE int
+  AO_short_compare_and_swap_release(volatile unsigned/**/short *addr, unsigned/**/short old_val,
+                                    unsigned/**/short new_val)
+  {
+    return AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_short_compare_and_swap_release
+#endif
+
+#if defined(AO_HAVE_short_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_short_compare_and_swap)
+  AO_INLINE int
+  AO_short_compare_and_swap(volatile unsigned/**/short *addr, unsigned/**/short old_val,
+                            unsigned/**/short new_val)
+  {
+    return AO_short_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
+  }
+# define AO_HAVE_short_compare_and_swap
+#endif
+
+/* short_fetch_and_add */
+/* We first try to implement fetch_and_add variants in terms of the     */
+/* corresponding compare_and_swap variants to minimize adding barriers. */
+#if defined(AO_HAVE_short_compare_and_swap_full) \
+    && !defined(AO_HAVE_short_fetch_and_add_full)
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_and_add_full(volatile unsigned/**/short *addr, unsigned/**/short incr)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
+                                                           old + incr)));
+    return old;
+  }
+# define AO_HAVE_short_fetch_and_add_full
+#endif
+
+#if defined(AO_HAVE_short_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_short_fetch_and_add_acquire)
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_short_fetch_and_add_acquire
+#endif
+
+#if defined(AO_HAVE_short_compare_and_swap_release) \
+    && !defined(AO_HAVE_short_fetch_and_add_release)
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_and_add_release(volatile unsigned/**/short *addr, unsigned/**/short incr)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_short_fetch_and_add_release
+#endif
+
+#if defined(AO_HAVE_short_compare_and_swap) \
+    && !defined(AO_HAVE_short_fetch_and_add)
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_and_add(volatile unsigned/**/short *addr, unsigned/**/short incr)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr, old,
+                                                      old + incr)));
+    return old;
+  }
+# define AO_HAVE_short_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_short_fetch_and_add_full)
+# if !defined(AO_HAVE_short_fetch_and_add_release)
+#   define AO_short_fetch_and_add_release(addr, val) \
+                                AO_short_fetch_and_add_full(addr, val)
+#   define AO_HAVE_short_fetch_and_add_release
+# endif
+# if !defined(AO_HAVE_short_fetch_and_add_acquire)
+#   define AO_short_fetch_and_add_acquire(addr, val) \
+                                AO_short_fetch_and_add_full(addr, val)
+#   define AO_HAVE_short_fetch_and_add_acquire
+# endif
+# if !defined(AO_HAVE_short_fetch_and_add_write)
+#   define AO_short_fetch_and_add_write(addr, val) \
+                                AO_short_fetch_and_add_full(addr, val)
+#   define AO_HAVE_short_fetch_and_add_write
+# endif
+# if !defined(AO_HAVE_short_fetch_and_add_read)
+#   define AO_short_fetch_and_add_read(addr, val) \
+                                AO_short_fetch_and_add_full(addr, val)
+#   define AO_HAVE_short_fetch_and_add_read
+# endif
+#endif /* AO_HAVE_short_fetch_and_add_full */
+
+#if !defined(AO_HAVE_short_fetch_and_add) \
+    && defined(AO_HAVE_short_fetch_and_add_release)
+# define AO_short_fetch_and_add(addr, val) \
+                                AO_short_fetch_and_add_release(addr, val)
+# define AO_HAVE_short_fetch_and_add
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add) \
+    && defined(AO_HAVE_short_fetch_and_add_acquire)
+# define AO_short_fetch_and_add(addr, val) \
+                                AO_short_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_short_fetch_and_add
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add) \
+    && defined(AO_HAVE_short_fetch_and_add_write)
+# define AO_short_fetch_and_add(addr, val) \
+                                AO_short_fetch_and_add_write(addr, val)
+# define AO_HAVE_short_fetch_and_add
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add) \
+    && defined(AO_HAVE_short_fetch_and_add_read)
+# define AO_short_fetch_and_add(addr, val) \
+                                AO_short_fetch_and_add_read(addr, val)
+# define AO_HAVE_short_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_short_fetch_and_add_acquire) \
+    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_short_fetch_and_add_full)
+# define AO_short_fetch_and_add_full(addr, val) \
+                (AO_nop_full(), AO_short_fetch_and_add_acquire(addr, val))
+# define AO_HAVE_short_fetch_and_add_full
+#endif
+
+#if !defined(AO_HAVE_short_fetch_and_add_release_write) \
+    && defined(AO_HAVE_short_fetch_and_add_write)
+# define AO_short_fetch_and_add_release_write(addr, val) \
+                                AO_short_fetch_and_add_write(addr, val)
+# define AO_HAVE_short_fetch_and_add_release_write
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add_release_write) \
+    && defined(AO_HAVE_short_fetch_and_add_release)
+# define AO_short_fetch_and_add_release_write(addr, val) \
+                                AO_short_fetch_and_add_release(addr, val)
+# define AO_HAVE_short_fetch_and_add_release_write
+#endif
+
+#if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_short_fetch_and_add_read)
+# define AO_short_fetch_and_add_acquire_read(addr, val) \
+                                AO_short_fetch_and_add_read(addr, val)
+# define AO_HAVE_short_fetch_and_add_acquire_read
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_short_fetch_and_add_acquire)
+# define AO_short_fetch_and_add_acquire_read(addr, val) \
+                                AO_short_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_short_fetch_and_add_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_short_fetch_and_add_acquire_read)
+#   define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_short_fetch_and_add_acquire_read(addr, val)
+#   define AO_HAVE_short_fetch_and_add_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_short_fetch_and_add)
+#   define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_short_fetch_and_add(addr, val)
+#   define AO_HAVE_short_fetch_and_add_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* short_fetch_and_add1 */
+#if defined(AO_HAVE_short_fetch_and_add_full) \
+    && !defined(AO_HAVE_short_fetch_and_add1_full)
+# define AO_short_fetch_and_add1_full(addr) \
+                                AO_short_fetch_and_add_full(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_full
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_release) \
+    && !defined(AO_HAVE_short_fetch_and_add1_release)
+# define AO_short_fetch_and_add1_release(addr) \
+                                AO_short_fetch_and_add_release(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_release
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_short_fetch_and_add1_acquire)
+# define AO_short_fetch_and_add1_acquire(addr) \
+                                AO_short_fetch_and_add_acquire(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_acquire
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_write) \
+    && !defined(AO_HAVE_short_fetch_and_add1_write)
+# define AO_short_fetch_and_add1_write(addr) \
+                                AO_short_fetch_and_add_write(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_write
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_read) \
+    && !defined(AO_HAVE_short_fetch_and_add1_read)
+# define AO_short_fetch_and_add1_read(addr) \
+                                AO_short_fetch_and_add_read(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_read
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_short_fetch_and_add1_release_write)
+# define AO_short_fetch_and_add1_release_write(addr) \
+                                AO_short_fetch_and_add_release_write(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_release_write
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_short_fetch_and_add1_acquire_read)
+# define AO_short_fetch_and_add1_acquire_read(addr) \
+                                AO_short_fetch_and_add_acquire_read(addr, 1)
+# define AO_HAVE_short_fetch_and_add1_acquire_read
+#endif
+#if defined(AO_HAVE_short_fetch_and_add) \
+    && !defined(AO_HAVE_short_fetch_and_add1)
+# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add(addr, 1)
+# define AO_HAVE_short_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_short_fetch_and_add1_full)
+# if !defined(AO_HAVE_short_fetch_and_add1_release)
+#   define AO_short_fetch_and_add1_release(addr) \
+                                AO_short_fetch_and_add1_full(addr)
+#   define AO_HAVE_short_fetch_and_add1_release
+# endif
+# if !defined(AO_HAVE_short_fetch_and_add1_acquire)
+#   define AO_short_fetch_and_add1_acquire(addr) \
+                                AO_short_fetch_and_add1_full(addr)
+#   define AO_HAVE_short_fetch_and_add1_acquire
+# endif
+# if !defined(AO_HAVE_short_fetch_and_add1_write)
+#   define AO_short_fetch_and_add1_write(addr) \
+                                AO_short_fetch_and_add1_full(addr)
+#   define AO_HAVE_short_fetch_and_add1_write
+# endif
+# if !defined(AO_HAVE_short_fetch_and_add1_read)
+#   define AO_short_fetch_and_add1_read(addr) \
+                                AO_short_fetch_and_add1_full(addr)
+#   define AO_HAVE_short_fetch_and_add1_read
+# endif
+#endif /* AO_HAVE_short_fetch_and_add1_full */
+
+#if !defined(AO_HAVE_short_fetch_and_add1) \
+    && defined(AO_HAVE_short_fetch_and_add1_release)
+# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_release(addr)
+# define AO_HAVE_short_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add1) \
+    && defined(AO_HAVE_short_fetch_and_add1_acquire)
+# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_acquire(addr)
+# define AO_HAVE_short_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add1) \
+    && defined(AO_HAVE_short_fetch_and_add1_write)
+# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_write(addr)
+# define AO_HAVE_short_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add1) \
+    && defined(AO_HAVE_short_fetch_and_add1_read)
+# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_read(addr)
+# define AO_HAVE_short_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_short_fetch_and_add1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_fetch_and_add1_full)
+# define AO_short_fetch_and_add1_full(addr) \
+                        (AO_nop_full(), AO_short_fetch_and_add1_acquire(addr))
+# define AO_HAVE_short_fetch_and_add1_full
+#endif
+
+#if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_short_fetch_and_add1_write)
+# define AO_short_fetch_and_add1_release_write(addr) \
+                                AO_short_fetch_and_add1_write(addr)
+# define AO_HAVE_short_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_short_fetch_and_add1_release)
+# define AO_short_fetch_and_add1_release_write(addr) \
+                                AO_short_fetch_and_add1_release(addr)
+# define AO_HAVE_short_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_short_fetch_and_add1_read)
+# define AO_short_fetch_and_add1_acquire_read(addr) \
+                                AO_short_fetch_and_add1_read(addr)
+# define AO_HAVE_short_fetch_and_add1_acquire_read
+#endif
+#if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_short_fetch_and_add1_acquire)
+# define AO_short_fetch_and_add1_acquire_read(addr) \
+                                AO_short_fetch_and_add1_acquire(addr)
+# define AO_HAVE_short_fetch_and_add1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_short_fetch_and_add1_acquire_read)
+#   define AO_short_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_short_fetch_and_add1_acquire_read(addr)
+#   define AO_HAVE_short_fetch_and_add1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_short_fetch_and_add1)
+#   define AO_short_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_short_fetch_and_add1(addr)
+#   define AO_HAVE_short_fetch_and_add1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* short_fetch_and_sub1 */
+#if defined(AO_HAVE_short_fetch_and_add_full) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_full)
+# define AO_short_fetch_and_sub1_full(addr) \
+                AO_short_fetch_and_add_full(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_full
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_release) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_release)
+# define AO_short_fetch_and_sub1_release(addr) \
+                AO_short_fetch_and_add_release(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_release
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_acquire)
+# define AO_short_fetch_and_sub1_acquire(addr) \
+                AO_short_fetch_and_add_acquire(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_acquire
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_write) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_write)
+# define AO_short_fetch_and_sub1_write(addr) \
+                AO_short_fetch_and_add_write(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_write
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_read) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_read)
+# define AO_short_fetch_and_sub1_read(addr) \
+                AO_short_fetch_and_add_read(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_read
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_release_write)
+# define AO_short_fetch_and_sub1_release_write(addr) \
+                AO_short_fetch_and_add_release_write(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_release_write
+#endif
+#if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
+# define AO_short_fetch_and_sub1_acquire_read(addr) \
+                AO_short_fetch_and_add_acquire_read(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1_acquire_read
+#endif
+#if defined(AO_HAVE_short_fetch_and_add) \
+    && !defined(AO_HAVE_short_fetch_and_sub1)
+# define AO_short_fetch_and_sub1(addr) \
+                AO_short_fetch_and_add(addr, (unsigned/**/short)(-1))
+# define AO_HAVE_short_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_short_fetch_and_sub1_full)
+# if !defined(AO_HAVE_short_fetch_and_sub1_release)
+#   define AO_short_fetch_and_sub1_release(addr) \
+                                AO_short_fetch_and_sub1_full(addr)
+#   define AO_HAVE_short_fetch_and_sub1_release
+# endif
+# if !defined(AO_HAVE_short_fetch_and_sub1_acquire)
+#   define AO_short_fetch_and_sub1_acquire(addr) \
+                                AO_short_fetch_and_sub1_full(addr)
+#   define AO_HAVE_short_fetch_and_sub1_acquire
+# endif
+# if !defined(AO_HAVE_short_fetch_and_sub1_write)
+#   define AO_short_fetch_and_sub1_write(addr) \
+                                AO_short_fetch_and_sub1_full(addr)
+#   define AO_HAVE_short_fetch_and_sub1_write
+# endif
+# if !defined(AO_HAVE_short_fetch_and_sub1_read)
+#   define AO_short_fetch_and_sub1_read(addr) \
+                                AO_short_fetch_and_sub1_full(addr)
+#   define AO_HAVE_short_fetch_and_sub1_read
+# endif
+#endif /* AO_HAVE_short_fetch_and_sub1_full */
+
+#if !defined(AO_HAVE_short_fetch_and_sub1) \
+    && defined(AO_HAVE_short_fetch_and_sub1_release)
+# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_release(addr)
+# define AO_HAVE_short_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_short_fetch_and_sub1) \
+    && defined(AO_HAVE_short_fetch_and_sub1_acquire)
+# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_short_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_short_fetch_and_sub1) \
+    && defined(AO_HAVE_short_fetch_and_sub1_write)
+# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_write(addr)
+# define AO_HAVE_short_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_short_fetch_and_sub1) \
+    && defined(AO_HAVE_short_fetch_and_sub1_read)
+# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_read(addr)
+# define AO_HAVE_short_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_short_fetch_and_sub1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_fetch_and_sub1_full)
+# define AO_short_fetch_and_sub1_full(addr) \
+                        (AO_nop_full(), AO_short_fetch_and_sub1_acquire(addr))
+# define AO_HAVE_short_fetch_and_sub1_full
+#endif
+
+#if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_short_fetch_and_sub1_write)
+# define AO_short_fetch_and_sub1_release_write(addr) \
+                                AO_short_fetch_and_sub1_write(addr)
+# define AO_HAVE_short_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_short_fetch_and_sub1_release)
+# define AO_short_fetch_and_sub1_release_write(addr) \
+                                AO_short_fetch_and_sub1_release(addr)
+# define AO_HAVE_short_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_short_fetch_and_sub1_read)
+# define AO_short_fetch_and_sub1_acquire_read(addr) \
+                                AO_short_fetch_and_sub1_read(addr)
+# define AO_HAVE_short_fetch_and_sub1_acquire_read
+#endif
+#if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_short_fetch_and_sub1_acquire)
+# define AO_short_fetch_and_sub1_acquire_read(addr) \
+                                AO_short_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_short_fetch_and_sub1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
+#   define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_short_fetch_and_sub1_acquire_read(addr)
+#   define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_short_fetch_and_sub1)
+#   define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_short_fetch_and_sub1(addr)
+#   define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* short_and */
+#if defined(AO_HAVE_short_compare_and_swap_full) \
+    && !defined(AO_HAVE_short_and_full)
+  AO_INLINE void
+  AO_short_and_full(volatile unsigned/**/short *addr, unsigned/**/short value)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
+                                                           old & value)));
+  }
+# define AO_HAVE_short_and_full
+#endif
+
+#if defined(AO_HAVE_short_and_full)
+# if !defined(AO_HAVE_short_and_release)
+#   define AO_short_and_release(addr, val) AO_short_and_full(addr, val)
+#   define AO_HAVE_short_and_release
+# endif
+# if !defined(AO_HAVE_short_and_acquire)
+#   define AO_short_and_acquire(addr, val) AO_short_and_full(addr, val)
+#   define AO_HAVE_short_and_acquire
+# endif
+# if !defined(AO_HAVE_short_and_write)
+#   define AO_short_and_write(addr, val) AO_short_and_full(addr, val)
+#   define AO_HAVE_short_and_write
+# endif
+# if !defined(AO_HAVE_short_and_read)
+#   define AO_short_and_read(addr, val) AO_short_and_full(addr, val)
+#   define AO_HAVE_short_and_read
+# endif
+#endif /* AO_HAVE_short_and_full */
+
+#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_release)
+# define AO_short_and(addr, val) AO_short_and_release(addr, val)
+# define AO_HAVE_short_and
+#endif
+#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_acquire)
+# define AO_short_and(addr, val) AO_short_and_acquire(addr, val)
+# define AO_HAVE_short_and
+#endif
+#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_write)
+# define AO_short_and(addr, val) AO_short_and_write(addr, val)
+# define AO_HAVE_short_and
+#endif
+#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_read)
+# define AO_short_and(addr, val) AO_short_and_read(addr, val)
+# define AO_HAVE_short_and
+#endif
+
+#if defined(AO_HAVE_short_and_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_and_full)
+# define AO_short_and_full(addr, val) \
+                        (AO_nop_full(), AO_short_and_acquire(addr, val))
+# define AO_HAVE_short_and_full
+#endif
+
+#if !defined(AO_HAVE_short_and_release_write) \
+    && defined(AO_HAVE_short_and_write)
+# define AO_short_and_release_write(addr, val) AO_short_and_write(addr, val)
+# define AO_HAVE_short_and_release_write
+#endif
+#if !defined(AO_HAVE_short_and_release_write) \
+    && defined(AO_HAVE_short_and_release)
+# define AO_short_and_release_write(addr, val) AO_short_and_release(addr, val)
+# define AO_HAVE_short_and_release_write
+#endif
+#if !defined(AO_HAVE_short_and_acquire_read) \
+    && defined(AO_HAVE_short_and_read)
+# define AO_short_and_acquire_read(addr, val) AO_short_and_read(addr, val)
+# define AO_HAVE_short_and_acquire_read
+#endif
+#if !defined(AO_HAVE_short_and_acquire_read) \
+    && defined(AO_HAVE_short_and_acquire)
+# define AO_short_and_acquire_read(addr, val) AO_short_and_acquire(addr, val)
+# define AO_HAVE_short_and_acquire_read
+#endif
+
+/* short_or */
+#if defined(AO_HAVE_short_compare_and_swap_full) \
+    && !defined(AO_HAVE_short_or_full)
+  AO_INLINE void
+  AO_short_or_full(volatile unsigned/**/short *addr, unsigned/**/short value)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
+                                                           old | value)));
+  }
+# define AO_HAVE_short_or_full
+#endif
+
+#if defined(AO_HAVE_short_or_full)
+# if !defined(AO_HAVE_short_or_release)
+#   define AO_short_or_release(addr, val) AO_short_or_full(addr, val)
+#   define AO_HAVE_short_or_release
+# endif
+# if !defined(AO_HAVE_short_or_acquire)
+#   define AO_short_or_acquire(addr, val) AO_short_or_full(addr, val)
+#   define AO_HAVE_short_or_acquire
+# endif
+# if !defined(AO_HAVE_short_or_write)
+#   define AO_short_or_write(addr, val) AO_short_or_full(addr, val)
+#   define AO_HAVE_short_or_write
+# endif
+# if !defined(AO_HAVE_short_or_read)
+#   define AO_short_or_read(addr, val) AO_short_or_full(addr, val)
+#   define AO_HAVE_short_or_read
+# endif
+#endif /* AO_HAVE_short_or_full */
+
+#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_release)
+# define AO_short_or(addr, val) AO_short_or_release(addr, val)
+# define AO_HAVE_short_or
+#endif
+#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_acquire)
+# define AO_short_or(addr, val) AO_short_or_acquire(addr, val)
+# define AO_HAVE_short_or
+#endif
+#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_write)
+# define AO_short_or(addr, val) AO_short_or_write(addr, val)
+# define AO_HAVE_short_or
+#endif
+#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_read)
+# define AO_short_or(addr, val) AO_short_or_read(addr, val)
+# define AO_HAVE_short_or
+#endif
+
+#if defined(AO_HAVE_short_or_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_or_full)
+# define AO_short_or_full(addr, val) \
+                        (AO_nop_full(), AO_short_or_acquire(addr, val))
+# define AO_HAVE_short_or_full
+#endif
+
+#if !defined(AO_HAVE_short_or_release_write) \
+    && defined(AO_HAVE_short_or_write)
+# define AO_short_or_release_write(addr, val) AO_short_or_write(addr, val)
+# define AO_HAVE_short_or_release_write
+#endif
+#if !defined(AO_HAVE_short_or_release_write) \
+    && defined(AO_HAVE_short_or_release)
+# define AO_short_or_release_write(addr, val) AO_short_or_release(addr, val)
+# define AO_HAVE_short_or_release_write
+#endif
+#if !defined(AO_HAVE_short_or_acquire_read) && defined(AO_HAVE_short_or_read)
+# define AO_short_or_acquire_read(addr, val) AO_short_or_read(addr, val)
+# define AO_HAVE_short_or_acquire_read
+#endif
+#if !defined(AO_HAVE_short_or_acquire_read) \
+    && defined(AO_HAVE_short_or_acquire)
+# define AO_short_or_acquire_read(addr, val) AO_short_or_acquire(addr, val)
+# define AO_HAVE_short_or_acquire_read
+#endif
+
+/* short_xor */
+#if defined(AO_HAVE_short_compare_and_swap_full) \
+    && !defined(AO_HAVE_short_xor_full)
+  AO_INLINE void
+  AO_short_xor_full(volatile unsigned/**/short *addr, unsigned/**/short value)
+  {
+    unsigned/**/short old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
+                                                           old ^ value)));
+  }
+# define AO_HAVE_short_xor_full
+#endif
+
+#if defined(AO_HAVE_short_xor_full)
+# if !defined(AO_HAVE_short_xor_release)
+#   define AO_short_xor_release(addr, val) AO_short_xor_full(addr, val)
+#   define AO_HAVE_short_xor_release
+# endif
+# if !defined(AO_HAVE_short_xor_acquire)
+#   define AO_short_xor_acquire(addr, val) AO_short_xor_full(addr, val)
+#   define AO_HAVE_short_xor_acquire
+# endif
+# if !defined(AO_HAVE_short_xor_write)
+#   define AO_short_xor_write(addr, val) AO_short_xor_full(addr, val)
+#   define AO_HAVE_short_xor_write
+# endif
+# if !defined(AO_HAVE_short_xor_read)
+#   define AO_short_xor_read(addr, val) AO_short_xor_full(addr, val)
+#   define AO_HAVE_short_xor_read
+# endif
+#endif /* AO_HAVE_short_xor_full */
+
+#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_release)
+# define AO_short_xor(addr, val) AO_short_xor_release(addr, val)
+# define AO_HAVE_short_xor
+#endif
+#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_acquire)
+# define AO_short_xor(addr, val) AO_short_xor_acquire(addr, val)
+# define AO_HAVE_short_xor
+#endif
+#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_write)
+# define AO_short_xor(addr, val) AO_short_xor_write(addr, val)
+# define AO_HAVE_short_xor
+#endif
+#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_read)
+# define AO_short_xor(addr, val) AO_short_xor_read(addr, val)
+# define AO_HAVE_short_xor
+#endif
+
+#if defined(AO_HAVE_short_xor_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_short_xor_full)
+# define AO_short_xor_full(addr, val) \
+                        (AO_nop_full(), AO_short_xor_acquire(addr, val))
+# define AO_HAVE_short_xor_full
+#endif
+
+#if !defined(AO_HAVE_short_xor_release_write) \
+    && defined(AO_HAVE_short_xor_write)
+# define AO_short_xor_release_write(addr, val) AO_short_xor_write(addr, val)
+# define AO_HAVE_short_xor_release_write
+#endif
+#if !defined(AO_HAVE_short_xor_release_write) \
+    && defined(AO_HAVE_short_xor_release)
+# define AO_short_xor_release_write(addr, val) AO_short_xor_release(addr, val)
+# define AO_HAVE_short_xor_release_write
+#endif
+#if !defined(AO_HAVE_short_xor_acquire_read) \
+    && defined(AO_HAVE_short_xor_read)
+# define AO_short_xor_acquire_read(addr, val) AO_short_xor_read(addr, val)
+# define AO_HAVE_short_xor_acquire_read
+#endif
+#if !defined(AO_HAVE_short_xor_acquire_read) \
+    && defined(AO_HAVE_short_xor_acquire)
+# define AO_short_xor_acquire_read(addr, val) AO_short_xor_acquire(addr, val)
+# define AO_HAVE_short_xor_acquire_read
+#endif
+
+/* short_and/or/xor_dd_aquire_read are meaningless.     */
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* int_compare_and_swap (based on fetch_compare_and_swap) */
+#if defined(AO_HAVE_int_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_compare_and_swap_full)
+  AO_INLINE int
+  AO_int_compare_and_swap_full(volatile unsigned *addr, unsigned old_val,
+                                 unsigned new_val)
+  {
+    return AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_int_compare_and_swap_full
+#endif
+
+#if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_int_compare_and_swap_acquire)
+  AO_INLINE int
+  AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old_val,
+                                    unsigned new_val)
+  {
+    return AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_int_compare_and_swap_acquire
+#endif
+
+#if defined(AO_HAVE_int_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_int_compare_and_swap_release)
+  AO_INLINE int
+  AO_int_compare_and_swap_release(volatile unsigned *addr, unsigned old_val,
+                                    unsigned new_val)
+  {
+    return AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_int_compare_and_swap_release
+#endif
+
+#if defined(AO_HAVE_int_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_int_compare_and_swap)
+  AO_INLINE int
+  AO_int_compare_and_swap(volatile unsigned *addr, unsigned old_val,
+                            unsigned new_val)
+  {
+    return AO_int_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
+  }
+# define AO_HAVE_int_compare_and_swap
+#endif
+
+/* int_fetch_and_add */
+/* We first try to implement fetch_and_add variants in terms of the     */
+/* corresponding compare_and_swap variants to minimize adding barriers. */
+#if defined(AO_HAVE_int_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_fetch_and_add_full)
+  AO_INLINE unsigned
+  AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
+                                                           old + incr)));
+    return old;
+  }
+# define AO_HAVE_int_fetch_and_add_full
+#endif
+
+#if defined(AO_HAVE_int_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_int_fetch_and_add_acquire)
+  AO_INLINE unsigned
+  AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_int_fetch_and_add_acquire
+#endif
+
+#if defined(AO_HAVE_int_compare_and_swap_release) \
+    && !defined(AO_HAVE_int_fetch_and_add_release)
+  AO_INLINE unsigned
+  AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_int_fetch_and_add_release
+#endif
+
+#if defined(AO_HAVE_int_compare_and_swap) \
+    && !defined(AO_HAVE_int_fetch_and_add)
+  AO_INLINE unsigned
+  AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr, old,
+                                                      old + incr)));
+    return old;
+  }
+# define AO_HAVE_int_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_int_fetch_and_add_full)
+# if !defined(AO_HAVE_int_fetch_and_add_release)
+#   define AO_int_fetch_and_add_release(addr, val) \
+                                AO_int_fetch_and_add_full(addr, val)
+#   define AO_HAVE_int_fetch_and_add_release
+# endif
+# if !defined(AO_HAVE_int_fetch_and_add_acquire)
+#   define AO_int_fetch_and_add_acquire(addr, val) \
+                                AO_int_fetch_and_add_full(addr, val)
+#   define AO_HAVE_int_fetch_and_add_acquire
+# endif
+# if !defined(AO_HAVE_int_fetch_and_add_write)
+#   define AO_int_fetch_and_add_write(addr, val) \
+                                AO_int_fetch_and_add_full(addr, val)
+#   define AO_HAVE_int_fetch_and_add_write
+# endif
+# if !defined(AO_HAVE_int_fetch_and_add_read)
+#   define AO_int_fetch_and_add_read(addr, val) \
+                                AO_int_fetch_and_add_full(addr, val)
+#   define AO_HAVE_int_fetch_and_add_read
+# endif
+#endif /* AO_HAVE_int_fetch_and_add_full */
+
+#if !defined(AO_HAVE_int_fetch_and_add) \
+    && defined(AO_HAVE_int_fetch_and_add_release)
+# define AO_int_fetch_and_add(addr, val) \
+                                AO_int_fetch_and_add_release(addr, val)
+# define AO_HAVE_int_fetch_and_add
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add) \
+    && defined(AO_HAVE_int_fetch_and_add_acquire)
+# define AO_int_fetch_and_add(addr, val) \
+                                AO_int_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_int_fetch_and_add
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add) \
+    && defined(AO_HAVE_int_fetch_and_add_write)
+# define AO_int_fetch_and_add(addr, val) \
+                                AO_int_fetch_and_add_write(addr, val)
+# define AO_HAVE_int_fetch_and_add
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add) \
+    && defined(AO_HAVE_int_fetch_and_add_read)
+# define AO_int_fetch_and_add(addr, val) \
+                                AO_int_fetch_and_add_read(addr, val)
+# define AO_HAVE_int_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_int_fetch_and_add_acquire) \
+    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_int_fetch_and_add_full)
+# define AO_int_fetch_and_add_full(addr, val) \
+                (AO_nop_full(), AO_int_fetch_and_add_acquire(addr, val))
+# define AO_HAVE_int_fetch_and_add_full
+#endif
+
+#if !defined(AO_HAVE_int_fetch_and_add_release_write) \
+    && defined(AO_HAVE_int_fetch_and_add_write)
+# define AO_int_fetch_and_add_release_write(addr, val) \
+                                AO_int_fetch_and_add_write(addr, val)
+# define AO_HAVE_int_fetch_and_add_release_write
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add_release_write) \
+    && defined(AO_HAVE_int_fetch_and_add_release)
+# define AO_int_fetch_and_add_release_write(addr, val) \
+                                AO_int_fetch_and_add_release(addr, val)
+# define AO_HAVE_int_fetch_and_add_release_write
+#endif
+
+#if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_int_fetch_and_add_read)
+# define AO_int_fetch_and_add_acquire_read(addr, val) \
+                                AO_int_fetch_and_add_read(addr, val)
+# define AO_HAVE_int_fetch_and_add_acquire_read
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_int_fetch_and_add_acquire)
+# define AO_int_fetch_and_add_acquire_read(addr, val) \
+                                AO_int_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_int_fetch_and_add_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_int_fetch_and_add_acquire_read)
+#   define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_int_fetch_and_add_acquire_read(addr, val)
+#   define AO_HAVE_int_fetch_and_add_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_int_fetch_and_add)
+#   define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_int_fetch_and_add(addr, val)
+#   define AO_HAVE_int_fetch_and_add_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* int_fetch_and_add1 */
+#if defined(AO_HAVE_int_fetch_and_add_full) \
+    && !defined(AO_HAVE_int_fetch_and_add1_full)
+# define AO_int_fetch_and_add1_full(addr) \
+                                AO_int_fetch_and_add_full(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_full
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_release) \
+    && !defined(AO_HAVE_int_fetch_and_add1_release)
+# define AO_int_fetch_and_add1_release(addr) \
+                                AO_int_fetch_and_add_release(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_release
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_int_fetch_and_add1_acquire)
+# define AO_int_fetch_and_add1_acquire(addr) \
+                                AO_int_fetch_and_add_acquire(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_acquire
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_write) \
+    && !defined(AO_HAVE_int_fetch_and_add1_write)
+# define AO_int_fetch_and_add1_write(addr) \
+                                AO_int_fetch_and_add_write(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_write
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_read) \
+    && !defined(AO_HAVE_int_fetch_and_add1_read)
+# define AO_int_fetch_and_add1_read(addr) \
+                                AO_int_fetch_and_add_read(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_read
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_int_fetch_and_add1_release_write)
+# define AO_int_fetch_and_add1_release_write(addr) \
+                                AO_int_fetch_and_add_release_write(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_release_write
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_int_fetch_and_add1_acquire_read)
+# define AO_int_fetch_and_add1_acquire_read(addr) \
+                                AO_int_fetch_and_add_acquire_read(addr, 1)
+# define AO_HAVE_int_fetch_and_add1_acquire_read
+#endif
+#if defined(AO_HAVE_int_fetch_and_add) \
+    && !defined(AO_HAVE_int_fetch_and_add1)
+# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add(addr, 1)
+# define AO_HAVE_int_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_int_fetch_and_add1_full)
+# if !defined(AO_HAVE_int_fetch_and_add1_release)
+#   define AO_int_fetch_and_add1_release(addr) \
+                                AO_int_fetch_and_add1_full(addr)
+#   define AO_HAVE_int_fetch_and_add1_release
+# endif
+# if !defined(AO_HAVE_int_fetch_and_add1_acquire)
+#   define AO_int_fetch_and_add1_acquire(addr) \
+                                AO_int_fetch_and_add1_full(addr)
+#   define AO_HAVE_int_fetch_and_add1_acquire
+# endif
+# if !defined(AO_HAVE_int_fetch_and_add1_write)
+#   define AO_int_fetch_and_add1_write(addr) \
+                                AO_int_fetch_and_add1_full(addr)
+#   define AO_HAVE_int_fetch_and_add1_write
+# endif
+# if !defined(AO_HAVE_int_fetch_and_add1_read)
+#   define AO_int_fetch_and_add1_read(addr) \
+                                AO_int_fetch_and_add1_full(addr)
+#   define AO_HAVE_int_fetch_and_add1_read
+# endif
+#endif /* AO_HAVE_int_fetch_and_add1_full */
+
+#if !defined(AO_HAVE_int_fetch_and_add1) \
+    && defined(AO_HAVE_int_fetch_and_add1_release)
+# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_release(addr)
+# define AO_HAVE_int_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add1) \
+    && defined(AO_HAVE_int_fetch_and_add1_acquire)
+# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_acquire(addr)
+# define AO_HAVE_int_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add1) \
+    && defined(AO_HAVE_int_fetch_and_add1_write)
+# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_write(addr)
+# define AO_HAVE_int_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add1) \
+    && defined(AO_HAVE_int_fetch_and_add1_read)
+# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_read(addr)
+# define AO_HAVE_int_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_int_fetch_and_add1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_fetch_and_add1_full)
+# define AO_int_fetch_and_add1_full(addr) \
+                        (AO_nop_full(), AO_int_fetch_and_add1_acquire(addr))
+# define AO_HAVE_int_fetch_and_add1_full
+#endif
+
+#if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_int_fetch_and_add1_write)
+# define AO_int_fetch_and_add1_release_write(addr) \
+                                AO_int_fetch_and_add1_write(addr)
+# define AO_HAVE_int_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_int_fetch_and_add1_release)
+# define AO_int_fetch_and_add1_release_write(addr) \
+                                AO_int_fetch_and_add1_release(addr)
+# define AO_HAVE_int_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_int_fetch_and_add1_read)
+# define AO_int_fetch_and_add1_acquire_read(addr) \
+                                AO_int_fetch_and_add1_read(addr)
+# define AO_HAVE_int_fetch_and_add1_acquire_read
+#endif
+#if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_int_fetch_and_add1_acquire)
+# define AO_int_fetch_and_add1_acquire_read(addr) \
+                                AO_int_fetch_and_add1_acquire(addr)
+# define AO_HAVE_int_fetch_and_add1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_int_fetch_and_add1_acquire_read)
+#   define AO_int_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_int_fetch_and_add1_acquire_read(addr)
+#   define AO_HAVE_int_fetch_and_add1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_int_fetch_and_add1)
+#   define AO_int_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_int_fetch_and_add1(addr)
+#   define AO_HAVE_int_fetch_and_add1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* int_fetch_and_sub1 */
+#if defined(AO_HAVE_int_fetch_and_add_full) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_full)
+# define AO_int_fetch_and_sub1_full(addr) \
+                AO_int_fetch_and_add_full(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_full
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_release) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_release)
+# define AO_int_fetch_and_sub1_release(addr) \
+                AO_int_fetch_and_add_release(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_release
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_acquire)
+# define AO_int_fetch_and_sub1_acquire(addr) \
+                AO_int_fetch_and_add_acquire(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_acquire
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_write) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_write)
+# define AO_int_fetch_and_sub1_write(addr) \
+                AO_int_fetch_and_add_write(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_write
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_read) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_read)
+# define AO_int_fetch_and_sub1_read(addr) \
+                AO_int_fetch_and_add_read(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_read
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_release_write)
+# define AO_int_fetch_and_sub1_release_write(addr) \
+                AO_int_fetch_and_add_release_write(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_release_write
+#endif
+#if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
+# define AO_int_fetch_and_sub1_acquire_read(addr) \
+                AO_int_fetch_and_add_acquire_read(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1_acquire_read
+#endif
+#if defined(AO_HAVE_int_fetch_and_add) \
+    && !defined(AO_HAVE_int_fetch_and_sub1)
+# define AO_int_fetch_and_sub1(addr) \
+                AO_int_fetch_and_add(addr, (unsigned)(-1))
+# define AO_HAVE_int_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_int_fetch_and_sub1_full)
+# if !defined(AO_HAVE_int_fetch_and_sub1_release)
+#   define AO_int_fetch_and_sub1_release(addr) \
+                                AO_int_fetch_and_sub1_full(addr)
+#   define AO_HAVE_int_fetch_and_sub1_release
+# endif
+# if !defined(AO_HAVE_int_fetch_and_sub1_acquire)
+#   define AO_int_fetch_and_sub1_acquire(addr) \
+                                AO_int_fetch_and_sub1_full(addr)
+#   define AO_HAVE_int_fetch_and_sub1_acquire
+# endif
+# if !defined(AO_HAVE_int_fetch_and_sub1_write)
+#   define AO_int_fetch_and_sub1_write(addr) \
+                                AO_int_fetch_and_sub1_full(addr)
+#   define AO_HAVE_int_fetch_and_sub1_write
+# endif
+# if !defined(AO_HAVE_int_fetch_and_sub1_read)
+#   define AO_int_fetch_and_sub1_read(addr) \
+                                AO_int_fetch_and_sub1_full(addr)
+#   define AO_HAVE_int_fetch_and_sub1_read
+# endif
+#endif /* AO_HAVE_int_fetch_and_sub1_full */
+
+#if !defined(AO_HAVE_int_fetch_and_sub1) \
+    && defined(AO_HAVE_int_fetch_and_sub1_release)
+# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_release(addr)
+# define AO_HAVE_int_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_int_fetch_and_sub1) \
+    && defined(AO_HAVE_int_fetch_and_sub1_acquire)
+# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_int_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_int_fetch_and_sub1) \
+    && defined(AO_HAVE_int_fetch_and_sub1_write)
+# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_write(addr)
+# define AO_HAVE_int_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_int_fetch_and_sub1) \
+    && defined(AO_HAVE_int_fetch_and_sub1_read)
+# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_read(addr)
+# define AO_HAVE_int_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_int_fetch_and_sub1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_fetch_and_sub1_full)
+# define AO_int_fetch_and_sub1_full(addr) \
+                        (AO_nop_full(), AO_int_fetch_and_sub1_acquire(addr))
+# define AO_HAVE_int_fetch_and_sub1_full
+#endif
+
+#if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_int_fetch_and_sub1_write)
+# define AO_int_fetch_and_sub1_release_write(addr) \
+                                AO_int_fetch_and_sub1_write(addr)
+# define AO_HAVE_int_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_int_fetch_and_sub1_release)
+# define AO_int_fetch_and_sub1_release_write(addr) \
+                                AO_int_fetch_and_sub1_release(addr)
+# define AO_HAVE_int_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_int_fetch_and_sub1_read)
+# define AO_int_fetch_and_sub1_acquire_read(addr) \
+                                AO_int_fetch_and_sub1_read(addr)
+# define AO_HAVE_int_fetch_and_sub1_acquire_read
+#endif
+#if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_int_fetch_and_sub1_acquire)
+# define AO_int_fetch_and_sub1_acquire_read(addr) \
+                                AO_int_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_int_fetch_and_sub1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
+#   define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_int_fetch_and_sub1_acquire_read(addr)
+#   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_int_fetch_and_sub1)
+#   define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_int_fetch_and_sub1(addr)
+#   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* int_and */
+#if defined(AO_HAVE_int_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_and_full)
+  AO_INLINE void
+  AO_int_and_full(volatile unsigned *addr, unsigned value)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
+                                                           old & value)));
+  }
+# define AO_HAVE_int_and_full
+#endif
+
+#if defined(AO_HAVE_int_and_full)
+# if !defined(AO_HAVE_int_and_release)
+#   define AO_int_and_release(addr, val) AO_int_and_full(addr, val)
+#   define AO_HAVE_int_and_release
+# endif
+# if !defined(AO_HAVE_int_and_acquire)
+#   define AO_int_and_acquire(addr, val) AO_int_and_full(addr, val)
+#   define AO_HAVE_int_and_acquire
+# endif
+# if !defined(AO_HAVE_int_and_write)
+#   define AO_int_and_write(addr, val) AO_int_and_full(addr, val)
+#   define AO_HAVE_int_and_write
+# endif
+# if !defined(AO_HAVE_int_and_read)
+#   define AO_int_and_read(addr, val) AO_int_and_full(addr, val)
+#   define AO_HAVE_int_and_read
+# endif
+#endif /* AO_HAVE_int_and_full */
+
+#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_release)
+# define AO_int_and(addr, val) AO_int_and_release(addr, val)
+# define AO_HAVE_int_and
+#endif
+#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_acquire)
+# define AO_int_and(addr, val) AO_int_and_acquire(addr, val)
+# define AO_HAVE_int_and
+#endif
+#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_write)
+# define AO_int_and(addr, val) AO_int_and_write(addr, val)
+# define AO_HAVE_int_and
+#endif
+#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_read)
+# define AO_int_and(addr, val) AO_int_and_read(addr, val)
+# define AO_HAVE_int_and
+#endif
+
+#if defined(AO_HAVE_int_and_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_and_full)
+# define AO_int_and_full(addr, val) \
+                        (AO_nop_full(), AO_int_and_acquire(addr, val))
+# define AO_HAVE_int_and_full
+#endif
+
+#if !defined(AO_HAVE_int_and_release_write) \
+    && defined(AO_HAVE_int_and_write)
+# define AO_int_and_release_write(addr, val) AO_int_and_write(addr, val)
+# define AO_HAVE_int_and_release_write
+#endif
+#if !defined(AO_HAVE_int_and_release_write) \
+    && defined(AO_HAVE_int_and_release)
+# define AO_int_and_release_write(addr, val) AO_int_and_release(addr, val)
+# define AO_HAVE_int_and_release_write
+#endif
+#if !defined(AO_HAVE_int_and_acquire_read) \
+    && defined(AO_HAVE_int_and_read)
+# define AO_int_and_acquire_read(addr, val) AO_int_and_read(addr, val)
+# define AO_HAVE_int_and_acquire_read
+#endif
+#if !defined(AO_HAVE_int_and_acquire_read) \
+    && defined(AO_HAVE_int_and_acquire)
+# define AO_int_and_acquire_read(addr, val) AO_int_and_acquire(addr, val)
+# define AO_HAVE_int_and_acquire_read
+#endif
+
+/* int_or */
+#if defined(AO_HAVE_int_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_or_full)
+  AO_INLINE void
+  AO_int_or_full(volatile unsigned *addr, unsigned value)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
+                                                           old | value)));
+  }
+# define AO_HAVE_int_or_full
+#endif
+
+#if defined(AO_HAVE_int_or_full)
+# if !defined(AO_HAVE_int_or_release)
+#   define AO_int_or_release(addr, val) AO_int_or_full(addr, val)
+#   define AO_HAVE_int_or_release
+# endif
+# if !defined(AO_HAVE_int_or_acquire)
+#   define AO_int_or_acquire(addr, val) AO_int_or_full(addr, val)
+#   define AO_HAVE_int_or_acquire
+# endif
+# if !defined(AO_HAVE_int_or_write)
+#   define AO_int_or_write(addr, val) AO_int_or_full(addr, val)
+#   define AO_HAVE_int_or_write
+# endif
+# if !defined(AO_HAVE_int_or_read)
+#   define AO_int_or_read(addr, val) AO_int_or_full(addr, val)
+#   define AO_HAVE_int_or_read
+# endif
+#endif /* AO_HAVE_int_or_full */
+
+#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_release)
+# define AO_int_or(addr, val) AO_int_or_release(addr, val)
+# define AO_HAVE_int_or
+#endif
+#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_acquire)
+# define AO_int_or(addr, val) AO_int_or_acquire(addr, val)
+# define AO_HAVE_int_or
+#endif
+#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_write)
+# define AO_int_or(addr, val) AO_int_or_write(addr, val)
+# define AO_HAVE_int_or
+#endif
+#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_read)
+# define AO_int_or(addr, val) AO_int_or_read(addr, val)
+# define AO_HAVE_int_or
+#endif
+
+#if defined(AO_HAVE_int_or_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_or_full)
+# define AO_int_or_full(addr, val) \
+                        (AO_nop_full(), AO_int_or_acquire(addr, val))
+# define AO_HAVE_int_or_full
+#endif
+
+#if !defined(AO_HAVE_int_or_release_write) \
+    && defined(AO_HAVE_int_or_write)
+# define AO_int_or_release_write(addr, val) AO_int_or_write(addr, val)
+# define AO_HAVE_int_or_release_write
+#endif
+#if !defined(AO_HAVE_int_or_release_write) \
+    && defined(AO_HAVE_int_or_release)
+# define AO_int_or_release_write(addr, val) AO_int_or_release(addr, val)
+# define AO_HAVE_int_or_release_write
+#endif
+#if !defined(AO_HAVE_int_or_acquire_read) && defined(AO_HAVE_int_or_read)
+# define AO_int_or_acquire_read(addr, val) AO_int_or_read(addr, val)
+# define AO_HAVE_int_or_acquire_read
+#endif
+#if !defined(AO_HAVE_int_or_acquire_read) \
+    && defined(AO_HAVE_int_or_acquire)
+# define AO_int_or_acquire_read(addr, val) AO_int_or_acquire(addr, val)
+# define AO_HAVE_int_or_acquire_read
+#endif
+
+/* int_xor */
+#if defined(AO_HAVE_int_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_xor_full)
+  AO_INLINE void
+  AO_int_xor_full(volatile unsigned *addr, unsigned value)
+  {
+    unsigned old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
+                                                           old ^ value)));
+  }
+# define AO_HAVE_int_xor_full
+#endif
+
+#if defined(AO_HAVE_int_xor_full)
+# if !defined(AO_HAVE_int_xor_release)
+#   define AO_int_xor_release(addr, val) AO_int_xor_full(addr, val)
+#   define AO_HAVE_int_xor_release
+# endif
+# if !defined(AO_HAVE_int_xor_acquire)
+#   define AO_int_xor_acquire(addr, val) AO_int_xor_full(addr, val)
+#   define AO_HAVE_int_xor_acquire
+# endif
+# if !defined(AO_HAVE_int_xor_write)
+#   define AO_int_xor_write(addr, val) AO_int_xor_full(addr, val)
+#   define AO_HAVE_int_xor_write
+# endif
+# if !defined(AO_HAVE_int_xor_read)
+#   define AO_int_xor_read(addr, val) AO_int_xor_full(addr, val)
+#   define AO_HAVE_int_xor_read
+# endif
+#endif /* AO_HAVE_int_xor_full */
+
+#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_release)
+# define AO_int_xor(addr, val) AO_int_xor_release(addr, val)
+# define AO_HAVE_int_xor
+#endif
+#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_acquire)
+# define AO_int_xor(addr, val) AO_int_xor_acquire(addr, val)
+# define AO_HAVE_int_xor
+#endif
+#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_write)
+# define AO_int_xor(addr, val) AO_int_xor_write(addr, val)
+# define AO_HAVE_int_xor
+#endif
+#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_read)
+# define AO_int_xor(addr, val) AO_int_xor_read(addr, val)
+# define AO_HAVE_int_xor
+#endif
+
+#if defined(AO_HAVE_int_xor_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_xor_full)
+# define AO_int_xor_full(addr, val) \
+                        (AO_nop_full(), AO_int_xor_acquire(addr, val))
+# define AO_HAVE_int_xor_full
+#endif
+
+#if !defined(AO_HAVE_int_xor_release_write) \
+    && defined(AO_HAVE_int_xor_write)
+# define AO_int_xor_release_write(addr, val) AO_int_xor_write(addr, val)
+# define AO_HAVE_int_xor_release_write
+#endif
+#if !defined(AO_HAVE_int_xor_release_write) \
+    && defined(AO_HAVE_int_xor_release)
+# define AO_int_xor_release_write(addr, val) AO_int_xor_release(addr, val)
+# define AO_HAVE_int_xor_release_write
+#endif
+#if !defined(AO_HAVE_int_xor_acquire_read) \
+    && defined(AO_HAVE_int_xor_read)
+# define AO_int_xor_acquire_read(addr, val) AO_int_xor_read(addr, val)
+# define AO_HAVE_int_xor_acquire_read
+#endif
+#if !defined(AO_HAVE_int_xor_acquire_read) \
+    && defined(AO_HAVE_int_xor_acquire)
+# define AO_int_xor_acquire_read(addr, val) AO_int_xor_acquire(addr, val)
+# define AO_HAVE_int_xor_acquire_read
+#endif
+
+/* int_and/or/xor_dd_aquire_read are meaningless.     */
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* compare_and_swap (based on fetch_compare_and_swap) */
+#if defined(AO_HAVE_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_compare_and_swap_full)
+  AO_INLINE int
+  AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
+                                 AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap_full(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_compare_and_swap_full
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_compare_and_swap_acquire)
+  AO_INLINE int
+  AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
+                                    AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_compare_and_swap_acquire
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_compare_and_swap_release)
+  AO_INLINE int
+  AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val,
+                                    AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap_release(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_compare_and_swap_release
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_compare_and_swap)
+  AO_INLINE int
+  AO_compare_and_swap(volatile AO_t *addr, AO_t old_val,
+                            AO_t new_val)
+  {
+    return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
+  }
+# define AO_HAVE_compare_and_swap
+#endif
+
+/* fetch_and_add */
+/* We first try to implement fetch_and_add variants in terms of the     */
+/* corresponding compare_and_swap variants to minimize adding barriers. */
+#if defined(AO_HAVE_compare_and_swap_full) \
+    && !defined(AO_HAVE_fetch_and_add_full)
+  AO_INLINE AO_t
+  AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
+                                                           old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add_full
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_fetch_and_add_acquire)
+  AO_INLINE AO_t
+  AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add_acquire
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_release) \
+    && !defined(AO_HAVE_fetch_and_add_release)
+  AO_INLINE AO_t
+  AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add_release
+#endif
+
+#if defined(AO_HAVE_compare_and_swap) \
+    && !defined(AO_HAVE_fetch_and_add)
+  AO_INLINE AO_t
+  AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old,
+                                                      old + incr)));
+    return old;
+  }
+# define AO_HAVE_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_fetch_and_add_full)
+# if !defined(AO_HAVE_fetch_and_add_release)
+#   define AO_fetch_and_add_release(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_release
+# endif
+# if !defined(AO_HAVE_fetch_and_add_acquire)
+#   define AO_fetch_and_add_acquire(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_acquire
+# endif
+# if !defined(AO_HAVE_fetch_and_add_write)
+#   define AO_fetch_and_add_write(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_write
+# endif
+# if !defined(AO_HAVE_fetch_and_add_read)
+#   define AO_fetch_and_add_read(addr, val) \
+                                AO_fetch_and_add_full(addr, val)
+#   define AO_HAVE_fetch_and_add_read
+# endif
+#endif /* AO_HAVE_fetch_and_add_full */
+
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_release)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_release(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_acquire)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_write)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_write(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_fetch_and_add) \
+    && defined(AO_HAVE_fetch_and_add_read)
+# define AO_fetch_and_add(addr, val) \
+                                AO_fetch_and_add_read(addr, val)
+# define AO_HAVE_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_fetch_and_add_acquire) \
+    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full)
+# define AO_fetch_and_add_full(addr, val) \
+                (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
+# define AO_HAVE_fetch_and_add_full
+#endif
+
+#if !defined(AO_HAVE_fetch_and_add_release_write) \
+    && defined(AO_HAVE_fetch_and_add_write)
+# define AO_fetch_and_add_release_write(addr, val) \
+                                AO_fetch_and_add_write(addr, val)
+# define AO_HAVE_fetch_and_add_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_add_release_write) \
+    && defined(AO_HAVE_fetch_and_add_release)
+# define AO_fetch_and_add_release_write(addr, val) \
+                                AO_fetch_and_add_release(addr, val)
+# define AO_HAVE_fetch_and_add_release_write
+#endif
+
+#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add_read)
+# define AO_fetch_and_add_acquire_read(addr, val) \
+                                AO_fetch_and_add_read(addr, val)
+# define AO_HAVE_fetch_and_add_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add_acquire)
+# define AO_fetch_and_add_acquire_read(addr, val) \
+                                AO_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_fetch_and_add_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_and_add_acquire_read)
+#   define AO_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_fetch_and_add_acquire_read(addr, val)
+#   define AO_HAVE_fetch_and_add_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_and_add)
+#   define AO_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_fetch_and_add(addr, val)
+#   define AO_HAVE_fetch_and_add_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* fetch_and_add1 */
+#if defined(AO_HAVE_fetch_and_add_full) \
+    && !defined(AO_HAVE_fetch_and_add1_full)
+# define AO_fetch_and_add1_full(addr) \
+                                AO_fetch_and_add_full(addr, 1)
+# define AO_HAVE_fetch_and_add1_full
+#endif
+#if defined(AO_HAVE_fetch_and_add_release) \
+    && !defined(AO_HAVE_fetch_and_add1_release)
+# define AO_fetch_and_add1_release(addr) \
+                                AO_fetch_and_add_release(addr, 1)
+# define AO_HAVE_fetch_and_add1_release
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_fetch_and_add1_acquire)
+# define AO_fetch_and_add1_acquire(addr) \
+                                AO_fetch_and_add_acquire(addr, 1)
+# define AO_HAVE_fetch_and_add1_acquire
+#endif
+#if defined(AO_HAVE_fetch_and_add_write) \
+    && !defined(AO_HAVE_fetch_and_add1_write)
+# define AO_fetch_and_add1_write(addr) \
+                                AO_fetch_and_add_write(addr, 1)
+# define AO_HAVE_fetch_and_add1_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_read) \
+    && !defined(AO_HAVE_fetch_and_add1_read)
+# define AO_fetch_and_add1_read(addr) \
+                                AO_fetch_and_add_read(addr, 1)
+# define AO_HAVE_fetch_and_add1_read
+#endif
+#if defined(AO_HAVE_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_fetch_and_add1_release_write)
+# define AO_fetch_and_add1_release_write(addr) \
+                                AO_fetch_and_add_release_write(addr, 1)
+# define AO_HAVE_fetch_and_add1_release_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_fetch_and_add1_acquire_read)
+# define AO_fetch_and_add1_acquire_read(addr) \
+                                AO_fetch_and_add_acquire_read(addr, 1)
+# define AO_HAVE_fetch_and_add1_acquire_read
+#endif
+#if defined(AO_HAVE_fetch_and_add) \
+    && !defined(AO_HAVE_fetch_and_add1)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add(addr, 1)
+# define AO_HAVE_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_fetch_and_add1_full)
+# if !defined(AO_HAVE_fetch_and_add1_release)
+#   define AO_fetch_and_add1_release(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_release
+# endif
+# if !defined(AO_HAVE_fetch_and_add1_acquire)
+#   define AO_fetch_and_add1_acquire(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_acquire
+# endif
+# if !defined(AO_HAVE_fetch_and_add1_write)
+#   define AO_fetch_and_add1_write(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_write
+# endif
+# if !defined(AO_HAVE_fetch_and_add1_read)
+#   define AO_fetch_and_add1_read(addr) \
+                                AO_fetch_and_add1_full(addr)
+#   define AO_HAVE_fetch_and_add1_read
+# endif
+#endif /* AO_HAVE_fetch_and_add1_full */
+
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_release)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_acquire)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_write)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_fetch_and_add1) \
+    && defined(AO_HAVE_fetch_and_add1_read)
+# define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr)
+# define AO_HAVE_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_fetch_and_add1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_and_add1_full)
+# define AO_fetch_and_add1_full(addr) \
+                        (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
+# define AO_HAVE_fetch_and_add1_full
+#endif
+
+#if !defined(AO_HAVE_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_fetch_and_add1_write)
+# define AO_fetch_and_add1_release_write(addr) \
+                                AO_fetch_and_add1_write(addr)
+# define AO_HAVE_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_fetch_and_add1_release)
+# define AO_fetch_and_add1_release_write(addr) \
+                                AO_fetch_and_add1_release(addr)
+# define AO_HAVE_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add1_read)
+# define AO_fetch_and_add1_acquire_read(addr) \
+                                AO_fetch_and_add1_read(addr)
+# define AO_HAVE_fetch_and_add1_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_add1_acquire)
+# define AO_fetch_and_add1_acquire_read(addr) \
+                                AO_fetch_and_add1_acquire(addr)
+# define AO_HAVE_fetch_and_add1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_and_add1_acquire_read)
+#   define AO_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_fetch_and_add1_acquire_read(addr)
+#   define AO_HAVE_fetch_and_add1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_and_add1)
+#   define AO_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_fetch_and_add1(addr)
+#   define AO_HAVE_fetch_and_add1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* fetch_and_sub1 */
+#if defined(AO_HAVE_fetch_and_add_full) \
+    && !defined(AO_HAVE_fetch_and_sub1_full)
+# define AO_fetch_and_sub1_full(addr) \
+                AO_fetch_and_add_full(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_full
+#endif
+#if defined(AO_HAVE_fetch_and_add_release) \
+    && !defined(AO_HAVE_fetch_and_sub1_release)
+# define AO_fetch_and_sub1_release(addr) \
+                AO_fetch_and_add_release(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_release
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_fetch_and_sub1_acquire)
+# define AO_fetch_and_sub1_acquire(addr) \
+                AO_fetch_and_add_acquire(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_acquire
+#endif
+#if defined(AO_HAVE_fetch_and_add_write) \
+    && !defined(AO_HAVE_fetch_and_sub1_write)
+# define AO_fetch_and_sub1_write(addr) \
+                AO_fetch_and_add_write(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_read) \
+    && !defined(AO_HAVE_fetch_and_sub1_read)
+# define AO_fetch_and_sub1_read(addr) \
+                AO_fetch_and_add_read(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_read
+#endif
+#if defined(AO_HAVE_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_fetch_and_sub1_release_write)
+# define AO_fetch_and_sub1_release_write(addr) \
+                AO_fetch_and_add_release_write(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_release_write
+#endif
+#if defined(AO_HAVE_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_fetch_and_sub1_acquire_read)
+# define AO_fetch_and_sub1_acquire_read(addr) \
+                AO_fetch_and_add_acquire_read(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1_acquire_read
+#endif
+#if defined(AO_HAVE_fetch_and_add) \
+    && !defined(AO_HAVE_fetch_and_sub1)
+# define AO_fetch_and_sub1(addr) \
+                AO_fetch_and_add(addr, (AO_t)(-1))
+# define AO_HAVE_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_fetch_and_sub1_full)
+# if !defined(AO_HAVE_fetch_and_sub1_release)
+#   define AO_fetch_and_sub1_release(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_release
+# endif
+# if !defined(AO_HAVE_fetch_and_sub1_acquire)
+#   define AO_fetch_and_sub1_acquire(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_acquire
+# endif
+# if !defined(AO_HAVE_fetch_and_sub1_write)
+#   define AO_fetch_and_sub1_write(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_write
+# endif
+# if !defined(AO_HAVE_fetch_and_sub1_read)
+#   define AO_fetch_and_sub1_read(addr) \
+                                AO_fetch_and_sub1_full(addr)
+#   define AO_HAVE_fetch_and_sub1_read
+# endif
+#endif /* AO_HAVE_fetch_and_sub1_full */
+
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_release)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_acquire)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_write)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1) \
+    && defined(AO_HAVE_fetch_and_sub1_read)
+# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr)
+# define AO_HAVE_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_fetch_and_sub1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_and_sub1_full)
+# define AO_fetch_and_sub1_full(addr) \
+                        (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
+# define AO_HAVE_fetch_and_sub1_full
+#endif
+
+#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_fetch_and_sub1_write)
+# define AO_fetch_and_sub1_release_write(addr) \
+                                AO_fetch_and_sub1_write(addr)
+# define AO_HAVE_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_fetch_and_sub1_release)
+# define AO_fetch_and_sub1_release_write(addr) \
+                                AO_fetch_and_sub1_release(addr)
+# define AO_HAVE_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_sub1_read)
+# define AO_fetch_and_sub1_acquire_read(addr) \
+                                AO_fetch_and_sub1_read(addr)
+# define AO_HAVE_fetch_and_sub1_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_fetch_and_sub1_acquire)
+# define AO_fetch_and_sub1_acquire_read(addr) \
+                                AO_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_fetch_and_sub1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_and_sub1_acquire_read)
+#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_fetch_and_sub1_acquire_read(addr)
+#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_and_sub1)
+#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_fetch_and_sub1(addr)
+#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* and */
+#if defined(AO_HAVE_compare_and_swap_full) \
+    && !defined(AO_HAVE_and_full)
+  AO_INLINE void
+  AO_and_full(volatile AO_t *addr, AO_t value)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
+                                                           old & value)));
+  }
+# define AO_HAVE_and_full
+#endif
+
+#if defined(AO_HAVE_and_full)
+# if !defined(AO_HAVE_and_release)
+#   define AO_and_release(addr, val) AO_and_full(addr, val)
+#   define AO_HAVE_and_release
+# endif
+# if !defined(AO_HAVE_and_acquire)
+#   define AO_and_acquire(addr, val) AO_and_full(addr, val)
+#   define AO_HAVE_and_acquire
+# endif
+# if !defined(AO_HAVE_and_write)
+#   define AO_and_write(addr, val) AO_and_full(addr, val)
+#   define AO_HAVE_and_write
+# endif
+# if !defined(AO_HAVE_and_read)
+#   define AO_and_read(addr, val) AO_and_full(addr, val)
+#   define AO_HAVE_and_read
+# endif
+#endif /* AO_HAVE_and_full */
+
+#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_release)
+# define AO_and(addr, val) AO_and_release(addr, val)
+# define AO_HAVE_and
+#endif
+#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_acquire)
+# define AO_and(addr, val) AO_and_acquire(addr, val)
+# define AO_HAVE_and
+#endif
+#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_write)
+# define AO_and(addr, val) AO_and_write(addr, val)
+# define AO_HAVE_and
+#endif
+#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_read)
+# define AO_and(addr, val) AO_and_read(addr, val)
+# define AO_HAVE_and
+#endif
+
+#if defined(AO_HAVE_and_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_and_full)
+# define AO_and_full(addr, val) \
+                        (AO_nop_full(), AO_and_acquire(addr, val))
+# define AO_HAVE_and_full
+#endif
+
+#if !defined(AO_HAVE_and_release_write) \
+    && defined(AO_HAVE_and_write)
+# define AO_and_release_write(addr, val) AO_and_write(addr, val)
+# define AO_HAVE_and_release_write
+#endif
+#if !defined(AO_HAVE_and_release_write) \
+    && defined(AO_HAVE_and_release)
+# define AO_and_release_write(addr, val) AO_and_release(addr, val)
+# define AO_HAVE_and_release_write
+#endif
+#if !defined(AO_HAVE_and_acquire_read) \
+    && defined(AO_HAVE_and_read)
+# define AO_and_acquire_read(addr, val) AO_and_read(addr, val)
+# define AO_HAVE_and_acquire_read
+#endif
+#if !defined(AO_HAVE_and_acquire_read) \
+    && defined(AO_HAVE_and_acquire)
+# define AO_and_acquire_read(addr, val) AO_and_acquire(addr, val)
+# define AO_HAVE_and_acquire_read
+#endif
+
+/* or */
+#if defined(AO_HAVE_compare_and_swap_full) \
+    && !defined(AO_HAVE_or_full)
+  AO_INLINE void
+  AO_or_full(volatile AO_t *addr, AO_t value)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
+                                                           old | value)));
+  }
+# define AO_HAVE_or_full
+#endif
+
+#if defined(AO_HAVE_or_full)
+# if !defined(AO_HAVE_or_release)
+#   define AO_or_release(addr, val) AO_or_full(addr, val)
+#   define AO_HAVE_or_release
+# endif
+# if !defined(AO_HAVE_or_acquire)
+#   define AO_or_acquire(addr, val) AO_or_full(addr, val)
+#   define AO_HAVE_or_acquire
+# endif
+# if !defined(AO_HAVE_or_write)
+#   define AO_or_write(addr, val) AO_or_full(addr, val)
+#   define AO_HAVE_or_write
+# endif
+# if !defined(AO_HAVE_or_read)
+#   define AO_or_read(addr, val) AO_or_full(addr, val)
+#   define AO_HAVE_or_read
+# endif
+#endif /* AO_HAVE_or_full */
+
+#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_release)
+# define AO_or(addr, val) AO_or_release(addr, val)
+# define AO_HAVE_or
+#endif
+#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_acquire)
+# define AO_or(addr, val) AO_or_acquire(addr, val)
+# define AO_HAVE_or
+#endif
+#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_write)
+# define AO_or(addr, val) AO_or_write(addr, val)
+# define AO_HAVE_or
+#endif
+#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_read)
+# define AO_or(addr, val) AO_or_read(addr, val)
+# define AO_HAVE_or
+#endif
+
+#if defined(AO_HAVE_or_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_or_full)
+# define AO_or_full(addr, val) \
+                        (AO_nop_full(), AO_or_acquire(addr, val))
+# define AO_HAVE_or_full
+#endif
+
+#if !defined(AO_HAVE_or_release_write) \
+    && defined(AO_HAVE_or_write)
+# define AO_or_release_write(addr, val) AO_or_write(addr, val)
+# define AO_HAVE_or_release_write
+#endif
+#if !defined(AO_HAVE_or_release_write) \
+    && defined(AO_HAVE_or_release)
+# define AO_or_release_write(addr, val) AO_or_release(addr, val)
+# define AO_HAVE_or_release_write
+#endif
+#if !defined(AO_HAVE_or_acquire_read) && defined(AO_HAVE_or_read)
+# define AO_or_acquire_read(addr, val) AO_or_read(addr, val)
+# define AO_HAVE_or_acquire_read
+#endif
+#if !defined(AO_HAVE_or_acquire_read) \
+    && defined(AO_HAVE_or_acquire)
+# define AO_or_acquire_read(addr, val) AO_or_acquire(addr, val)
+# define AO_HAVE_or_acquire_read
+#endif
+
+/* xor */
+#if defined(AO_HAVE_compare_and_swap_full) \
+    && !defined(AO_HAVE_xor_full)
+  AO_INLINE void
+  AO_xor_full(volatile AO_t *addr, AO_t value)
+  {
+    AO_t old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
+                                                           old ^ value)));
+  }
+# define AO_HAVE_xor_full
+#endif
+
+#if defined(AO_HAVE_xor_full)
+# if !defined(AO_HAVE_xor_release)
+#   define AO_xor_release(addr, val) AO_xor_full(addr, val)
+#   define AO_HAVE_xor_release
+# endif
+# if !defined(AO_HAVE_xor_acquire)
+#   define AO_xor_acquire(addr, val) AO_xor_full(addr, val)
+#   define AO_HAVE_xor_acquire
+# endif
+# if !defined(AO_HAVE_xor_write)
+#   define AO_xor_write(addr, val) AO_xor_full(addr, val)
+#   define AO_HAVE_xor_write
+# endif
+# if !defined(AO_HAVE_xor_read)
+#   define AO_xor_read(addr, val) AO_xor_full(addr, val)
+#   define AO_HAVE_xor_read
+# endif
+#endif /* AO_HAVE_xor_full */
+
+#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_release)
+# define AO_xor(addr, val) AO_xor_release(addr, val)
+# define AO_HAVE_xor
+#endif
+#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_acquire)
+# define AO_xor(addr, val) AO_xor_acquire(addr, val)
+# define AO_HAVE_xor
+#endif
+#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_write)
+# define AO_xor(addr, val) AO_xor_write(addr, val)
+# define AO_HAVE_xor
+#endif
+#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_read)
+# define AO_xor(addr, val) AO_xor_read(addr, val)
+# define AO_HAVE_xor
+#endif
+
+#if defined(AO_HAVE_xor_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_xor_full)
+# define AO_xor_full(addr, val) \
+                        (AO_nop_full(), AO_xor_acquire(addr, val))
+# define AO_HAVE_xor_full
+#endif
+
+#if !defined(AO_HAVE_xor_release_write) \
+    && defined(AO_HAVE_xor_write)
+# define AO_xor_release_write(addr, val) AO_xor_write(addr, val)
+# define AO_HAVE_xor_release_write
+#endif
+#if !defined(AO_HAVE_xor_release_write) \
+    && defined(AO_HAVE_xor_release)
+# define AO_xor_release_write(addr, val) AO_xor_release(addr, val)
+# define AO_HAVE_xor_release_write
+#endif
+#if !defined(AO_HAVE_xor_acquire_read) \
+    && defined(AO_HAVE_xor_read)
+# define AO_xor_acquire_read(addr, val) AO_xor_read(addr, val)
+# define AO_HAVE_xor_acquire_read
+#endif
+#if !defined(AO_HAVE_xor_acquire_read) \
+    && defined(AO_HAVE_xor_acquire)
+# define AO_xor_acquire_read(addr, val) AO_xor_acquire(addr, val)
+# define AO_HAVE_xor_acquire_read
+#endif
+
+/* and/or/xor_dd_aquire_read are meaningless.     */

diff --git a/src/atomic_ops/generalize-arithm.template b/src/atomic_ops/generalize-arithm.template
new file mode 100644 (file)
index 0000000..b7c5c7d
--- /dev/null
+++ b/src/atomic_ops/generalize-arithm.template
@@ -0,0 +1,767 @@
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* XSIZE_compare_and_swap (based on fetch_compare_and_swap) */
+#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_XSIZE_compare_and_swap_full)
+  AO_INLINE int
+  AO_XSIZE_compare_and_swap_full(volatile XCTYPE *addr, XCTYPE old_val,
+                                 XCTYPE new_val)
+  {
+    return AO_XSIZE_fetch_compare_and_swap_full(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_XSIZE_compare_and_swap_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire)
+  AO_INLINE int
+  AO_XSIZE_compare_and_swap_acquire(volatile XCTYPE *addr, XCTYPE old_val,
+                                    XCTYPE new_val)
+  {
+    return AO_XSIZE_fetch_compare_and_swap_acquire(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_XSIZE_compare_and_swap_acquire
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_XSIZE_compare_and_swap_release)
+  AO_INLINE int
+  AO_XSIZE_compare_and_swap_release(volatile XCTYPE *addr, XCTYPE old_val,
+                                    XCTYPE new_val)
+  {
+    return AO_XSIZE_fetch_compare_and_swap_release(addr, old_val, new_val)
+             == old_val;
+  }
+# define AO_HAVE_XSIZE_compare_and_swap_release
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_XSIZE_compare_and_swap)
+  AO_INLINE int
+  AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, XCTYPE old_val,
+                            XCTYPE new_val)
+  {
+    return AO_XSIZE_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
+  }
+# define AO_HAVE_XSIZE_compare_and_swap
+#endif
+
+/* XSIZE_fetch_and_add */
+/* We first try to implement fetch_and_add variants in terms of the     */
+/* corresponding compare_and_swap variants to minimize adding barriers. */
+#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add_full)
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_and_add_full(volatile XCTYPE *addr, XCTYPE incr)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
+                                                           old + incr)));
+    return old;
+  }
+# define AO_HAVE_XSIZE_fetch_and_add_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_XSIZE_fetch_and_add_acquire
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap_release) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add_release)
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_and_add_release(volatile XCTYPE *addr, XCTYPE incr)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_release(addr, old,
+                                                              old + incr)));
+    return old;
+  }
+# define AO_HAVE_XSIZE_fetch_and_add_release
+#endif
+
+#if defined(AO_HAVE_XSIZE_compare_and_swap) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add)
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_and_add(volatile XCTYPE *addr, XCTYPE incr)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap(addr, old,
+                                                      old + incr)));
+    return old;
+  }
+# define AO_HAVE_XSIZE_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_and_add_full)
+# if !defined(AO_HAVE_XSIZE_fetch_and_add_release)
+#   define AO_XSIZE_fetch_and_add_release(addr, val) \
+                                AO_XSIZE_fetch_and_add_full(addr, val)
+#   define AO_HAVE_XSIZE_fetch_and_add_release
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
+#   define AO_XSIZE_fetch_and_add_acquire(addr, val) \
+                                AO_XSIZE_fetch_and_add_full(addr, val)
+#   define AO_HAVE_XSIZE_fetch_and_add_acquire
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_add_write)
+#   define AO_XSIZE_fetch_and_add_write(addr, val) \
+                                AO_XSIZE_fetch_and_add_full(addr, val)
+#   define AO_HAVE_XSIZE_fetch_and_add_write
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_add_read)
+#   define AO_XSIZE_fetch_and_add_read(addr, val) \
+                                AO_XSIZE_fetch_and_add_full(addr, val)
+#   define AO_HAVE_XSIZE_fetch_and_add_read
+# endif
+#endif /* AO_HAVE_XSIZE_fetch_and_add_full */
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_release)
+# define AO_XSIZE_fetch_and_add(addr, val) \
+                                AO_XSIZE_fetch_and_add_release(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
+# define AO_XSIZE_fetch_and_add(addr, val) \
+                                AO_XSIZE_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_write)
+# define AO_XSIZE_fetch_and_add(addr, val) \
+                                AO_XSIZE_fetch_and_add_write(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_read)
+# define AO_XSIZE_fetch_and_add(addr, val) \
+                                AO_XSIZE_fetch_and_add_read(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
+    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_XSIZE_fetch_and_add_full)
+# define AO_XSIZE_fetch_and_add_full(addr, val) \
+                (AO_nop_full(), AO_XSIZE_fetch_and_add_acquire(addr, val))
+# define AO_HAVE_XSIZE_fetch_and_add_full
+#endif
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_write)
+# define AO_XSIZE_fetch_and_add_release_write(addr, val) \
+                                AO_XSIZE_fetch_and_add_write(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_release)
+# define AO_XSIZE_fetch_and_add_release_write(addr, val) \
+                                AO_XSIZE_fetch_and_add_release(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add_release_write
+#endif
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_read)
+# define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \
+                                AO_XSIZE_fetch_and_add_read(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add_acquire_read
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
+# define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \
+                                AO_XSIZE_fetch_and_add_acquire(addr, val)
+# define AO_HAVE_XSIZE_fetch_and_add_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read)
+#   define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_XSIZE_fetch_and_add_acquire_read(addr, val)
+#   define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_XSIZE_fetch_and_add)
+#   define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \
+                                AO_XSIZE_fetch_and_add(addr, val)
+#   define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* XSIZE_fetch_and_add1 */
+#if defined(AO_HAVE_XSIZE_fetch_and_add_full) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_full)
+# define AO_XSIZE_fetch_and_add1_full(addr) \
+                                AO_XSIZE_fetch_and_add_full(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_full
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_release) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_release)
+# define AO_XSIZE_fetch_and_add1_release(addr) \
+                                AO_XSIZE_fetch_and_add_release(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_release
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
+# define AO_XSIZE_fetch_and_add1_acquire(addr) \
+                                AO_XSIZE_fetch_and_add_acquire(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_acquire
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_write) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_write)
+# define AO_XSIZE_fetch_and_add1_write(addr) \
+                                AO_XSIZE_fetch_and_add_write(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_write
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_read) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_read)
+# define AO_XSIZE_fetch_and_add1_read(addr) \
+                                AO_XSIZE_fetch_and_add_read(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_read
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write)
+# define AO_XSIZE_fetch_and_add1_release_write(addr) \
+                                AO_XSIZE_fetch_and_add_release_write(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_release_write
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read)
+# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_add_acquire_read(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1)
+# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add(addr, 1)
+# define AO_HAVE_XSIZE_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_and_add1_full)
+# if !defined(AO_HAVE_XSIZE_fetch_and_add1_release)
+#   define AO_XSIZE_fetch_and_add1_release(addr) \
+                                AO_XSIZE_fetch_and_add1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_add1_release
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
+#   define AO_XSIZE_fetch_and_add1_acquire(addr) \
+                                AO_XSIZE_fetch_and_add1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_add1_acquire
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_add1_write)
+#   define AO_XSIZE_fetch_and_add1_write(addr) \
+                                AO_XSIZE_fetch_and_add1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_add1_write
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_add1_read)
+#   define AO_XSIZE_fetch_and_add1_read(addr) \
+                                AO_XSIZE_fetch_and_add1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_add1_read
+# endif
+#endif /* AO_HAVE_XSIZE_fetch_and_add1_full */
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_release)
+# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_release(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
+# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_acquire(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_write)
+# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_write(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_read)
+# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_read(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_add1_full)
+# define AO_XSIZE_fetch_and_add1_full(addr) \
+                        (AO_nop_full(), AO_XSIZE_fetch_and_add1_acquire(addr))
+# define AO_HAVE_XSIZE_fetch_and_add1_full
+#endif
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_write)
+# define AO_XSIZE_fetch_and_add1_release_write(addr) \
+                                AO_XSIZE_fetch_and_add1_write(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_release)
+# define AO_XSIZE_fetch_and_add1_release_write(addr) \
+                                AO_XSIZE_fetch_and_add1_release(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_read)
+# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_add1_read(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \
+    && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
+# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_add1_acquire(addr)
+# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read)
+#   define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_add1_acquire_read(addr)
+#   define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_XSIZE_fetch_and_add1)
+#   define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_add1(addr)
+#   define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* XSIZE_fetch_and_sub1 */
+#if defined(AO_HAVE_XSIZE_fetch_and_add_full) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
+# define AO_XSIZE_fetch_and_sub1_full(addr) \
+                AO_XSIZE_fetch_and_add_full(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_full
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_release) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
+# define AO_XSIZE_fetch_and_sub1_release(addr) \
+                AO_XSIZE_fetch_and_add_release(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_release
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
+# define AO_XSIZE_fetch_and_sub1_acquire(addr) \
+                AO_XSIZE_fetch_and_add_acquire(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_acquire
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_write) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
+# define AO_XSIZE_fetch_and_sub1_write(addr) \
+                AO_XSIZE_fetch_and_add_write(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_write
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_read) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
+# define AO_XSIZE_fetch_and_sub1_read(addr) \
+                AO_XSIZE_fetch_and_add_read(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_read
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write)
+# define AO_XSIZE_fetch_and_sub1_release_write(addr) \
+                AO_XSIZE_fetch_and_add_release_write(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_release_write
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read)
+# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
+                AO_XSIZE_fetch_and_add_acquire_read(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
+#endif
+#if defined(AO_HAVE_XSIZE_fetch_and_add) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1)
+# define AO_XSIZE_fetch_and_sub1(addr) \
+                AO_XSIZE_fetch_and_add(addr, (XCTYPE)(-1))
+# define AO_HAVE_XSIZE_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
+# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
+#   define AO_XSIZE_fetch_and_sub1_release(addr) \
+                                AO_XSIZE_fetch_and_sub1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_sub1_release
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
+#   define AO_XSIZE_fetch_and_sub1_acquire(addr) \
+                                AO_XSIZE_fetch_and_sub1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_sub1_acquire
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
+#   define AO_XSIZE_fetch_and_sub1_write(addr) \
+                                AO_XSIZE_fetch_and_sub1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_sub1_write
+# endif
+# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
+#   define AO_XSIZE_fetch_and_sub1_read(addr) \
+                                AO_XSIZE_fetch_and_sub1_full(addr)
+#   define AO_HAVE_XSIZE_fetch_and_sub1_read
+# endif
+#endif /* AO_HAVE_XSIZE_fetch_and_sub1_full */
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
+# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_release(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
+# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
+# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_write(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
+# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_read(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1
+#endif
+
+#if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
+# define AO_XSIZE_fetch_and_sub1_full(addr) \
+                        (AO_nop_full(), AO_XSIZE_fetch_and_sub1_acquire(addr))
+# define AO_HAVE_XSIZE_fetch_and_sub1_full
+#endif
+
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
+# define AO_XSIZE_fetch_and_sub1_release_write(addr) \
+                                AO_XSIZE_fetch_and_sub1_write(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
+# define AO_XSIZE_fetch_and_sub1_release_write(addr) \
+                                AO_XSIZE_fetch_and_sub1_release(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
+# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_sub1_read(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
+#endif
+#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \
+    && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
+# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_sub1_acquire(addr)
+# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read)
+#   define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_sub1_acquire_read(addr)
+#   define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_XSIZE_fetch_and_sub1)
+#   define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \
+                                AO_XSIZE_fetch_and_sub1(addr)
+#   define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
+/* XSIZE_and */
+#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
+    && !defined(AO_HAVE_XSIZE_and_full)
+  AO_INLINE void
+  AO_XSIZE_and_full(volatile XCTYPE *addr, XCTYPE value)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
+                                                           old & value)));
+  }
+# define AO_HAVE_XSIZE_and_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_and_full)
+# if !defined(AO_HAVE_XSIZE_and_release)
+#   define AO_XSIZE_and_release(addr, val) AO_XSIZE_and_full(addr, val)
+#   define AO_HAVE_XSIZE_and_release
+# endif
+# if !defined(AO_HAVE_XSIZE_and_acquire)
+#   define AO_XSIZE_and_acquire(addr, val) AO_XSIZE_and_full(addr, val)
+#   define AO_HAVE_XSIZE_and_acquire
+# endif
+# if !defined(AO_HAVE_XSIZE_and_write)
+#   define AO_XSIZE_and_write(addr, val) AO_XSIZE_and_full(addr, val)
+#   define AO_HAVE_XSIZE_and_write
+# endif
+# if !defined(AO_HAVE_XSIZE_and_read)
+#   define AO_XSIZE_and_read(addr, val) AO_XSIZE_and_full(addr, val)
+#   define AO_HAVE_XSIZE_and_read
+# endif
+#endif /* AO_HAVE_XSIZE_and_full */
+
+#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_release)
+# define AO_XSIZE_and(addr, val) AO_XSIZE_and_release(addr, val)
+# define AO_HAVE_XSIZE_and
+#endif
+#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_acquire)
+# define AO_XSIZE_and(addr, val) AO_XSIZE_and_acquire(addr, val)
+# define AO_HAVE_XSIZE_and
+#endif
+#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_write)
+# define AO_XSIZE_and(addr, val) AO_XSIZE_and_write(addr, val)
+# define AO_HAVE_XSIZE_and
+#endif
+#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_read)
+# define AO_XSIZE_and(addr, val) AO_XSIZE_and_read(addr, val)
+# define AO_HAVE_XSIZE_and
+#endif
+
+#if defined(AO_HAVE_XSIZE_and_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_and_full)
+# define AO_XSIZE_and_full(addr, val) \
+                        (AO_nop_full(), AO_XSIZE_and_acquire(addr, val))
+# define AO_HAVE_XSIZE_and_full
+#endif
+
+#if !defined(AO_HAVE_XSIZE_and_release_write) \
+    && defined(AO_HAVE_XSIZE_and_write)
+# define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_write(addr, val)
+# define AO_HAVE_XSIZE_and_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_and_release_write) \
+    && defined(AO_HAVE_XSIZE_and_release)
+# define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_release(addr, val)
+# define AO_HAVE_XSIZE_and_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_and_acquire_read) \
+    && defined(AO_HAVE_XSIZE_and_read)
+# define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_read(addr, val)
+# define AO_HAVE_XSIZE_and_acquire_read
+#endif
+#if !defined(AO_HAVE_XSIZE_and_acquire_read) \
+    && defined(AO_HAVE_XSIZE_and_acquire)
+# define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_acquire(addr, val)
+# define AO_HAVE_XSIZE_and_acquire_read
+#endif
+
+/* XSIZE_or */
+#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
+    && !defined(AO_HAVE_XSIZE_or_full)
+  AO_INLINE void
+  AO_XSIZE_or_full(volatile XCTYPE *addr, XCTYPE value)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
+                                                           old | value)));
+  }
+# define AO_HAVE_XSIZE_or_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_or_full)
+# if !defined(AO_HAVE_XSIZE_or_release)
+#   define AO_XSIZE_or_release(addr, val) AO_XSIZE_or_full(addr, val)
+#   define AO_HAVE_XSIZE_or_release
+# endif
+# if !defined(AO_HAVE_XSIZE_or_acquire)
+#   define AO_XSIZE_or_acquire(addr, val) AO_XSIZE_or_full(addr, val)
+#   define AO_HAVE_XSIZE_or_acquire
+# endif
+# if !defined(AO_HAVE_XSIZE_or_write)
+#   define AO_XSIZE_or_write(addr, val) AO_XSIZE_or_full(addr, val)
+#   define AO_HAVE_XSIZE_or_write
+# endif
+# if !defined(AO_HAVE_XSIZE_or_read)
+#   define AO_XSIZE_or_read(addr, val) AO_XSIZE_or_full(addr, val)
+#   define AO_HAVE_XSIZE_or_read
+# endif
+#endif /* AO_HAVE_XSIZE_or_full */
+
+#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_release)
+# define AO_XSIZE_or(addr, val) AO_XSIZE_or_release(addr, val)
+# define AO_HAVE_XSIZE_or
+#endif
+#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_acquire)
+# define AO_XSIZE_or(addr, val) AO_XSIZE_or_acquire(addr, val)
+# define AO_HAVE_XSIZE_or
+#endif
+#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_write)
+# define AO_XSIZE_or(addr, val) AO_XSIZE_or_write(addr, val)
+# define AO_HAVE_XSIZE_or
+#endif
+#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_read)
+# define AO_XSIZE_or(addr, val) AO_XSIZE_or_read(addr, val)
+# define AO_HAVE_XSIZE_or
+#endif
+
+#if defined(AO_HAVE_XSIZE_or_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_or_full)
+# define AO_XSIZE_or_full(addr, val) \
+                        (AO_nop_full(), AO_XSIZE_or_acquire(addr, val))
+# define AO_HAVE_XSIZE_or_full
+#endif
+
+#if !defined(AO_HAVE_XSIZE_or_release_write) \
+    && defined(AO_HAVE_XSIZE_or_write)
+# define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_write(addr, val)
+# define AO_HAVE_XSIZE_or_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_or_release_write) \
+    && defined(AO_HAVE_XSIZE_or_release)
+# define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_release(addr, val)
+# define AO_HAVE_XSIZE_or_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_or_acquire_read) && defined(AO_HAVE_XSIZE_or_read)
+# define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_read(addr, val)
+# define AO_HAVE_XSIZE_or_acquire_read
+#endif
+#if !defined(AO_HAVE_XSIZE_or_acquire_read) \
+    && defined(AO_HAVE_XSIZE_or_acquire)
+# define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_acquire(addr, val)
+# define AO_HAVE_XSIZE_or_acquire_read
+#endif
+
+/* XSIZE_xor */
+#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
+    && !defined(AO_HAVE_XSIZE_xor_full)
+  AO_INLINE void
+  AO_XSIZE_xor_full(volatile XCTYPE *addr, XCTYPE value)
+  {
+    XCTYPE old;
+
+    do
+      {
+        old = *addr;
+      }
+    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
+                                                           old ^ value)));
+  }
+# define AO_HAVE_XSIZE_xor_full
+#endif
+
+#if defined(AO_HAVE_XSIZE_xor_full)
+# if !defined(AO_HAVE_XSIZE_xor_release)
+#   define AO_XSIZE_xor_release(addr, val) AO_XSIZE_xor_full(addr, val)
+#   define AO_HAVE_XSIZE_xor_release
+# endif
+# if !defined(AO_HAVE_XSIZE_xor_acquire)
+#   define AO_XSIZE_xor_acquire(addr, val) AO_XSIZE_xor_full(addr, val)
+#   define AO_HAVE_XSIZE_xor_acquire
+# endif
+# if !defined(AO_HAVE_XSIZE_xor_write)
+#   define AO_XSIZE_xor_write(addr, val) AO_XSIZE_xor_full(addr, val)
+#   define AO_HAVE_XSIZE_xor_write
+# endif
+# if !defined(AO_HAVE_XSIZE_xor_read)
+#   define AO_XSIZE_xor_read(addr, val) AO_XSIZE_xor_full(addr, val)
+#   define AO_HAVE_XSIZE_xor_read
+# endif
+#endif /* AO_HAVE_XSIZE_xor_full */
+
+#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_release)
+# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_release(addr, val)
+# define AO_HAVE_XSIZE_xor
+#endif
+#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_acquire)
+# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_acquire(addr, val)
+# define AO_HAVE_XSIZE_xor
+#endif
+#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_write)
+# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_write(addr, val)
+# define AO_HAVE_XSIZE_xor
+#endif
+#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_read)
+# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_read(addr, val)
+# define AO_HAVE_XSIZE_xor
+#endif
+
+#if defined(AO_HAVE_XSIZE_xor_acquire) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_XSIZE_xor_full)
+# define AO_XSIZE_xor_full(addr, val) \
+                        (AO_nop_full(), AO_XSIZE_xor_acquire(addr, val))
+# define AO_HAVE_XSIZE_xor_full
+#endif
+
+#if !defined(AO_HAVE_XSIZE_xor_release_write) \
+    && defined(AO_HAVE_XSIZE_xor_write)
+# define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_write(addr, val)
+# define AO_HAVE_XSIZE_xor_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_xor_release_write) \
+    && defined(AO_HAVE_XSIZE_xor_release)
+# define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_release(addr, val)
+# define AO_HAVE_XSIZE_xor_release_write
+#endif
+#if !defined(AO_HAVE_XSIZE_xor_acquire_read) \
+    && defined(AO_HAVE_XSIZE_xor_read)
+# define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_read(addr, val)
+# define AO_HAVE_XSIZE_xor_acquire_read
+#endif
+#if !defined(AO_HAVE_XSIZE_xor_acquire_read) \
+    && defined(AO_HAVE_XSIZE_xor_acquire)
+# define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_acquire(addr, val)
+# define AO_HAVE_XSIZE_xor_acquire_read
+#endif
+
+/* XSIZE_and/or/xor_dd_aquire_read are meaningless.     */

diff --git a/src/atomic_ops/generalize-small.h b/src/atomic_ops/generalize-small.h
index 9bd3a93..30339de 100644 (file)
--- a/src/atomic_ops/generalize-small.h
+++ b/src/atomic_ops/generalize-small.h
@@ -388,751 +388,6 @@
 #   define AO_HAVE_char_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
-
-#if defined(AO_HAVE_char_fetch_compare_and_swap_full) \
-    && !defined(AO_HAVE_char_compare_and_swap_full)
-  AO_INLINE int
-  AO_char_compare_and_swap_full(volatile unsigned/**/char *addr, unsigned/**/char old_val,
-                                 unsigned/**/char new_val)
-  {
-    return AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_char_compare_and_swap_full
-#endif
-
-#if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_char_compare_and_swap_acquire)
-  AO_INLINE int
-  AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old_val,
-                                    unsigned/**/char new_val)
-  {
-    return AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_char_compare_and_swap_acquire
-#endif
-
-#if defined(AO_HAVE_char_fetch_compare_and_swap_release) \
-    && !defined(AO_HAVE_char_compare_and_swap_release)
-  AO_INLINE int
-  AO_char_compare_and_swap_release(volatile unsigned/**/char *addr, unsigned/**/char old_val,
-                                    unsigned/**/char new_val)
-  {
-    return AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_char_compare_and_swap_release
-#endif
-
-#if defined(AO_HAVE_char_fetch_compare_and_swap) \
-    && !defined(AO_HAVE_char_compare_and_swap)
-  AO_INLINE int
-  AO_char_compare_and_swap(volatile unsigned/**/char *addr, unsigned/**/char old_val,
-                            unsigned/**/char new_val)
-  {
-    return AO_char_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
-  }
-# define AO_HAVE_char_compare_and_swap
-#endif
-
-/* char_fetch_and_add */
-/* We first try to implement fetch_and_add variants in terms of the     */
-/* corresponding compare_and_swap variants to minimize adding barriers. */
-#if defined(AO_HAVE_char_compare_and_swap_full) \
-    && !defined(AO_HAVE_char_fetch_and_add_full)
-  AO_INLINE unsigned/**/char
-  AO_char_fetch_and_add_full(volatile unsigned/**/char *addr, unsigned/**/char incr)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
-                                                           old + incr)));
-    return old;
-  }
-# define AO_HAVE_char_fetch_and_add_full
-#endif
-
-#if defined(AO_HAVE_char_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_char_fetch_and_add_acquire)
-  AO_INLINE unsigned/**/char
-  AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_char_fetch_and_add_acquire
-#endif
-
-#if defined(AO_HAVE_char_compare_and_swap_release) \
-    && !defined(AO_HAVE_char_fetch_and_add_release)
-  AO_INLINE unsigned/**/char
-  AO_char_fetch_and_add_release(volatile unsigned/**/char *addr, unsigned/**/char incr)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_char_fetch_and_add_release
-#endif
-
-#if defined(AO_HAVE_char_compare_and_swap) \
-    && !defined(AO_HAVE_char_fetch_and_add)
-  AO_INLINE unsigned/**/char
-  AO_char_fetch_and_add(volatile unsigned/**/char *addr, unsigned/**/char incr)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr, old,
-                                                      old + incr)));
-    return old;
-  }
-# define AO_HAVE_char_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_char_fetch_and_add_full)
-# if !defined(AO_HAVE_char_fetch_and_add_release)
-#   define AO_char_fetch_and_add_release(addr, val) \
-                                AO_char_fetch_and_add_full(addr, val)
-#   define AO_HAVE_char_fetch_and_add_release
-# endif
-# if !defined(AO_HAVE_char_fetch_and_add_acquire)
-#   define AO_char_fetch_and_add_acquire(addr, val) \
-                                AO_char_fetch_and_add_full(addr, val)
-#   define AO_HAVE_char_fetch_and_add_acquire
-# endif
-# if !defined(AO_HAVE_char_fetch_and_add_write)
-#   define AO_char_fetch_and_add_write(addr, val) \
-                                AO_char_fetch_and_add_full(addr, val)
-#   define AO_HAVE_char_fetch_and_add_write
-# endif
-# if !defined(AO_HAVE_char_fetch_and_add_read)
-#   define AO_char_fetch_and_add_read(addr, val) \
-                                AO_char_fetch_and_add_full(addr, val)
-#   define AO_HAVE_char_fetch_and_add_read
-# endif
-#endif /* AO_HAVE_char_fetch_and_add_full */
-
-#if !defined(AO_HAVE_char_fetch_and_add) \
-    && defined(AO_HAVE_char_fetch_and_add_release)
-# define AO_char_fetch_and_add(addr, val) \
-                                AO_char_fetch_and_add_release(addr, val)
-# define AO_HAVE_char_fetch_and_add
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add) \
-    && defined(AO_HAVE_char_fetch_and_add_acquire)
-# define AO_char_fetch_and_add(addr, val) \
-                                AO_char_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_char_fetch_and_add
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add) \
-    && defined(AO_HAVE_char_fetch_and_add_write)
-# define AO_char_fetch_and_add(addr, val) \
-                                AO_char_fetch_and_add_write(addr, val)
-# define AO_HAVE_char_fetch_and_add
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add) \
-    && defined(AO_HAVE_char_fetch_and_add_read)
-# define AO_char_fetch_and_add(addr, val) \
-                                AO_char_fetch_and_add_read(addr, val)
-# define AO_HAVE_char_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_char_fetch_and_add_acquire) \
-    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_char_fetch_and_add_full)
-# define AO_char_fetch_and_add_full(addr, val) \
-                (AO_nop_full(), AO_char_fetch_and_add_acquire(addr, val))
-# define AO_HAVE_char_fetch_and_add_full
-#endif
-
-#if !defined(AO_HAVE_char_fetch_and_add_release_write) \
-    && defined(AO_HAVE_char_fetch_and_add_write)
-# define AO_char_fetch_and_add_release_write(addr, val) \
-                                AO_char_fetch_and_add_write(addr, val)
-# define AO_HAVE_char_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add_release_write) \
-    && defined(AO_HAVE_char_fetch_and_add_release)
-# define AO_char_fetch_and_add_release_write(addr, val) \
-                                AO_char_fetch_and_add_release(addr, val)
-# define AO_HAVE_char_fetch_and_add_release_write
-#endif
-
-#if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_char_fetch_and_add_read)
-# define AO_char_fetch_and_add_acquire_read(addr, val) \
-                                AO_char_fetch_and_add_read(addr, val)
-# define AO_HAVE_char_fetch_and_add_acquire_read
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_char_fetch_and_add_acquire)
-# define AO_char_fetch_and_add_acquire_read(addr, val) \
-                                AO_char_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_char_fetch_and_add_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_char_fetch_and_add_acquire_read)
-#   define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_char_fetch_and_add_acquire_read(addr, val)
-#   define AO_HAVE_char_fetch_and_add_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_char_fetch_and_add)
-#   define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_char_fetch_and_add(addr, val)
-#   define AO_HAVE_char_fetch_and_add_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* char_fetch_and_add1 */
-#if defined(AO_HAVE_char_fetch_and_add_full) \
-    && !defined(AO_HAVE_char_fetch_and_add1_full)
-# define AO_char_fetch_and_add1_full(addr) \
-                                AO_char_fetch_and_add_full(addr,1)
-# define AO_HAVE_char_fetch_and_add1_full
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_release) \
-    && !defined(AO_HAVE_char_fetch_and_add1_release)
-# define AO_char_fetch_and_add1_release(addr) \
-                                AO_char_fetch_and_add_release(addr,1)
-# define AO_HAVE_char_fetch_and_add1_release
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_char_fetch_and_add1_acquire)
-# define AO_char_fetch_and_add1_acquire(addr) \
-                                AO_char_fetch_and_add_acquire(addr,1)
-# define AO_HAVE_char_fetch_and_add1_acquire
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_write) \
-    && !defined(AO_HAVE_char_fetch_and_add1_write)
-# define AO_char_fetch_and_add1_write(addr) \
-                                AO_char_fetch_and_add_write(addr,1)
-# define AO_HAVE_char_fetch_and_add1_write
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_read) \
-    && !defined(AO_HAVE_char_fetch_and_add1_read)
-# define AO_char_fetch_and_add1_read(addr) \
-                                AO_char_fetch_and_add_read(addr,1)
-# define AO_HAVE_char_fetch_and_add1_read
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_char_fetch_and_add1_release_write)
-# define AO_char_fetch_and_add1_release_write(addr) \
-                                AO_char_fetch_and_add_release_write(addr,1)
-# define AO_HAVE_char_fetch_and_add1_release_write
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_char_fetch_and_add1_acquire_read)
-# define AO_char_fetch_and_add1_acquire_read(addr) \
-                                AO_char_fetch_and_add_acquire_read(addr,1)
-# define AO_HAVE_char_fetch_and_add1_acquire_read
-#endif
-#if defined(AO_HAVE_char_fetch_and_add) \
-    && !defined(AO_HAVE_char_fetch_and_add1)
-# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add(addr,1)
-# define AO_HAVE_char_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_char_fetch_and_add1_full)
-# if !defined(AO_HAVE_char_fetch_and_add1_release)
-#   define AO_char_fetch_and_add1_release(addr) \
-                                AO_char_fetch_and_add1_full(addr)
-#   define AO_HAVE_char_fetch_and_add1_release
-# endif
-# if !defined(AO_HAVE_char_fetch_and_add1_acquire)
-#   define AO_char_fetch_and_add1_acquire(addr) \
-                                AO_char_fetch_and_add1_full(addr)
-#   define AO_HAVE_char_fetch_and_add1_acquire
-# endif
-# if !defined(AO_HAVE_char_fetch_and_add1_write)
-#   define AO_char_fetch_and_add1_write(addr) \
-                                AO_char_fetch_and_add1_full(addr)
-#   define AO_HAVE_char_fetch_and_add1_write
-# endif
-# if !defined(AO_HAVE_char_fetch_and_add1_read)
-#   define AO_char_fetch_and_add1_read(addr) \
-                                AO_char_fetch_and_add1_full(addr)
-#   define AO_HAVE_char_fetch_and_add1_read
-# endif
-#endif /* AO_HAVE_char_fetch_and_add1_full */
-
-#if !defined(AO_HAVE_char_fetch_and_add1) \
-    && defined(AO_HAVE_char_fetch_and_add1_release)
-# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_release(addr)
-# define AO_HAVE_char_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add1) \
-    && defined(AO_HAVE_char_fetch_and_add1_acquire)
-# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_acquire(addr)
-# define AO_HAVE_char_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add1) \
-    && defined(AO_HAVE_char_fetch_and_add1_write)
-# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_write(addr)
-# define AO_HAVE_char_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add1) \
-    && defined(AO_HAVE_char_fetch_and_add1_read)
-# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_read(addr)
-# define AO_HAVE_char_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_char_fetch_and_add1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_fetch_and_add1_full)
-# define AO_char_fetch_and_add1_full(addr) \
-                        (AO_nop_full(), AO_char_fetch_and_add1_acquire(addr))
-# define AO_HAVE_char_fetch_and_add1_full
-#endif
-
-#if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_char_fetch_and_add1_write)
-# define AO_char_fetch_and_add1_release_write(addr) \
-                                AO_char_fetch_and_add1_write(addr)
-# define AO_HAVE_char_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_char_fetch_and_add1_release)
-# define AO_char_fetch_and_add1_release_write(addr) \
-                                AO_char_fetch_and_add1_release(addr)
-# define AO_HAVE_char_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_char_fetch_and_add1_read)
-# define AO_char_fetch_and_add1_acquire_read(addr) \
-                                AO_char_fetch_and_add1_read(addr)
-# define AO_HAVE_char_fetch_and_add1_acquire_read
-#endif
-#if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_char_fetch_and_add1_acquire)
-# define AO_char_fetch_and_add1_acquire_read(addr) \
-                                AO_char_fetch_and_add1_acquire(addr)
-# define AO_HAVE_char_fetch_and_add1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_char_fetch_and_add1_acquire_read)
-#   define AO_char_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_char_fetch_and_add1_acquire_read(addr)
-#   define AO_HAVE_char_fetch_and_add1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_char_fetch_and_add1)
-#   define AO_char_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_char_fetch_and_add1(addr)
-#   define AO_HAVE_char_fetch_and_add1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* char_fetch_and_sub1 */
-#if defined(AO_HAVE_char_fetch_and_add_full) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_full)
-# define AO_char_fetch_and_sub1_full(addr) \
-                AO_char_fetch_and_add_full(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_full
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_release) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_release)
-# define AO_char_fetch_and_sub1_release(addr) \
-                AO_char_fetch_and_add_release(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_release
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_acquire)
-# define AO_char_fetch_and_sub1_acquire(addr) \
-                AO_char_fetch_and_add_acquire(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_acquire
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_write) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_write)
-# define AO_char_fetch_and_sub1_write(addr) \
-                AO_char_fetch_and_add_write(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_write
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_read) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_read)
-# define AO_char_fetch_and_sub1_read(addr) \
-                AO_char_fetch_and_add_read(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_read
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_release_write)
-# define AO_char_fetch_and_sub1_release_write(addr) \
-                AO_char_fetch_and_add_release_write(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_release_write
-#endif
-#if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
-# define AO_char_fetch_and_sub1_acquire_read(addr) \
-                AO_char_fetch_and_add_acquire_read(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1_acquire_read
-#endif
-#if defined(AO_HAVE_char_fetch_and_add) \
-    && !defined(AO_HAVE_char_fetch_and_sub1)
-# define AO_char_fetch_and_sub1(addr) \
-                AO_char_fetch_and_add(addr, (unsigned/**/char)(-1))
-# define AO_HAVE_char_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_char_fetch_and_sub1_full)
-# if !defined(AO_HAVE_char_fetch_and_sub1_release)
-#   define AO_char_fetch_and_sub1_release(addr) \
-                                AO_char_fetch_and_sub1_full(addr)
-#   define AO_HAVE_char_fetch_and_sub1_release
-# endif
-# if !defined(AO_HAVE_char_fetch_and_sub1_acquire)
-#   define AO_char_fetch_and_sub1_acquire(addr) \
-                                AO_char_fetch_and_sub1_full(addr)
-#   define AO_HAVE_char_fetch_and_sub1_acquire
-# endif
-# if !defined(AO_HAVE_char_fetch_and_sub1_write)
-#   define AO_char_fetch_and_sub1_write(addr) \
-                                AO_char_fetch_and_sub1_full(addr)
-#   define AO_HAVE_char_fetch_and_sub1_write
-# endif
-# if !defined(AO_HAVE_char_fetch_and_sub1_read)
-#   define AO_char_fetch_and_sub1_read(addr) \
-                                AO_char_fetch_and_sub1_full(addr)
-#   define AO_HAVE_char_fetch_and_sub1_read
-# endif
-#endif /* AO_HAVE_char_fetch_and_sub1_full */
-
-#if !defined(AO_HAVE_char_fetch_and_sub1) \
-    && defined(AO_HAVE_char_fetch_and_sub1_release)
-# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_release(addr)
-# define AO_HAVE_char_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_char_fetch_and_sub1) \
-    && defined(AO_HAVE_char_fetch_and_sub1_acquire)
-# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_char_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_char_fetch_and_sub1) \
-    && defined(AO_HAVE_char_fetch_and_sub1_write)
-# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_write(addr)
-# define AO_HAVE_char_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_char_fetch_and_sub1) \
-    && defined(AO_HAVE_char_fetch_and_sub1_read)
-# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_read(addr)
-# define AO_HAVE_char_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_char_fetch_and_sub1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_fetch_and_sub1_full)
-# define AO_char_fetch_and_sub1_full(addr) \
-                        (AO_nop_full(), AO_char_fetch_and_sub1_acquire(addr))
-# define AO_HAVE_char_fetch_and_sub1_full
-#endif
-
-#if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_char_fetch_and_sub1_write)
-# define AO_char_fetch_and_sub1_release_write(addr) \
-                                AO_char_fetch_and_sub1_write(addr)
-# define AO_HAVE_char_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_char_fetch_and_sub1_release)
-# define AO_char_fetch_and_sub1_release_write(addr) \
-                                AO_char_fetch_and_sub1_release(addr)
-# define AO_HAVE_char_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_char_fetch_and_sub1_read)
-# define AO_char_fetch_and_sub1_acquire_read(addr) \
-                                AO_char_fetch_and_sub1_read(addr)
-# define AO_HAVE_char_fetch_and_sub1_acquire_read
-#endif
-#if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_char_fetch_and_sub1_acquire)
-# define AO_char_fetch_and_sub1_acquire_read(addr) \
-                                AO_char_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_char_fetch_and_sub1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
-#   define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_char_fetch_and_sub1_acquire_read(addr)
-#   define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_char_fetch_and_sub1)
-#   define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_char_fetch_and_sub1(addr)
-#   define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* char_and */
-#if defined(AO_HAVE_char_compare_and_swap_full) \
-    && !defined(AO_HAVE_char_and_full)
-  AO_INLINE void
-  AO_char_and_full(volatile unsigned/**/char *addr, unsigned/**/char value)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
-                                                           old & value)));
-  }
-# define AO_HAVE_char_and_full
-#endif
-
-#if defined(AO_HAVE_char_and_full)
-# if !defined(AO_HAVE_char_and_release)
-#   define AO_char_and_release(addr, val) AO_char_and_full(addr, val)
-#   define AO_HAVE_char_and_release
-# endif
-# if !defined(AO_HAVE_char_and_acquire)
-#   define AO_char_and_acquire(addr, val) AO_char_and_full(addr, val)
-#   define AO_HAVE_char_and_acquire
-# endif
-# if !defined(AO_HAVE_char_and_write)
-#   define AO_char_and_write(addr, val) AO_char_and_full(addr, val)
-#   define AO_HAVE_char_and_write
-# endif
-# if !defined(AO_HAVE_char_and_read)
-#   define AO_char_and_read(addr, val) AO_char_and_full(addr, val)
-#   define AO_HAVE_char_and_read
-# endif
-#endif /* AO_HAVE_char_and_full */
-
-#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_release)
-# define AO_char_and(addr, val) AO_char_and_release(addr, val)
-# define AO_HAVE_char_and
-#endif
-#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_acquire)
-# define AO_char_and(addr, val) AO_char_and_acquire(addr, val)
-# define AO_HAVE_char_and
-#endif
-#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_write)
-# define AO_char_and(addr, val) AO_char_and_write(addr, val)
-# define AO_HAVE_char_and
-#endif
-#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_read)
-# define AO_char_and(addr, val) AO_char_and_read(addr, val)
-# define AO_HAVE_char_and
-#endif
-
-#if defined(AO_HAVE_char_and_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_and_full)
-# define AO_char_and_full(addr, val) \
-                        (AO_nop_full(), AO_char_and_acquire(addr, val))
-# define AO_HAVE_char_and_full
-#endif
-
-#if !defined(AO_HAVE_char_and_release_write) \
-    && defined(AO_HAVE_char_and_write)
-# define AO_char_and_release_write(addr, val) AO_char_and_write(addr, val)
-# define AO_HAVE_char_and_release_write
-#endif
-#if !defined(AO_HAVE_char_and_release_write) \
-    && defined(AO_HAVE_char_and_release)
-# define AO_char_and_release_write(addr, val) AO_char_and_release(addr, val)
-# define AO_HAVE_char_and_release_write
-#endif
-#if !defined(AO_HAVE_char_and_acquire_read) \
-    && defined(AO_HAVE_char_and_read)
-# define AO_char_and_acquire_read(addr, val) AO_char_and_read(addr, val)
-# define AO_HAVE_char_and_acquire_read
-#endif
-#if !defined(AO_HAVE_char_and_acquire_read) \
-    && defined(AO_HAVE_char_and_acquire)
-# define AO_char_and_acquire_read(addr, val) AO_char_and_acquire(addr, val)
-# define AO_HAVE_char_and_acquire_read
-#endif
-
-/* char_or */
-#if defined(AO_HAVE_char_compare_and_swap_full) \
-    && !defined(AO_HAVE_char_or_full)
-  AO_INLINE void
-  AO_char_or_full(volatile unsigned/**/char *addr, unsigned/**/char value)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
-                                                           old | value)));
-  }
-# define AO_HAVE_char_or_full
-#endif
-
-#if defined(AO_HAVE_char_or_full)
-# if !defined(AO_HAVE_char_or_release)
-#   define AO_char_or_release(addr, val) AO_char_or_full(addr, val)
-#   define AO_HAVE_char_or_release
-# endif
-# if !defined(AO_HAVE_char_or_acquire)
-#   define AO_char_or_acquire(addr, val) AO_char_or_full(addr, val)
-#   define AO_HAVE_char_or_acquire
-# endif
-# if !defined(AO_HAVE_char_or_write)
-#   define AO_char_or_write(addr, val) AO_char_or_full(addr, val)
-#   define AO_HAVE_char_or_write
-# endif
-# if !defined(AO_HAVE_char_or_read)
-#   define AO_char_or_read(addr, val) AO_char_or_full(addr, val)
-#   define AO_HAVE_char_or_read
-# endif
-#endif /* AO_HAVE_char_or_full */
-
-#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_release)
-# define AO_char_or(addr, val) AO_char_or_release(addr, val)
-# define AO_HAVE_char_or
-#endif
-#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_acquire)
-# define AO_char_or(addr, val) AO_char_or_acquire(addr, val)
-# define AO_HAVE_char_or
-#endif
-#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_write)
-# define AO_char_or(addr, val) AO_char_or_write(addr, val)
-# define AO_HAVE_char_or
-#endif
-#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_read)
-# define AO_char_or(addr, val) AO_char_or_read(addr, val)
-# define AO_HAVE_char_or
-#endif
-
-#if defined(AO_HAVE_char_or_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_or_full)
-# define AO_char_or_full(addr, val) \
-                        (AO_nop_full(), AO_char_or_acquire(addr, val))
-# define AO_HAVE_char_or_full
-#endif
-
-#if !defined(AO_HAVE_char_or_release_write) \
-    && defined(AO_HAVE_char_or_write)
-# define AO_char_or_release_write(addr, val) AO_char_or_write(addr, val)
-# define AO_HAVE_char_or_release_write
-#endif
-#if !defined(AO_HAVE_char_or_release_write) \
-    && defined(AO_HAVE_char_or_release)
-# define AO_char_or_release_write(addr, val) AO_char_or_release(addr, val)
-# define AO_HAVE_char_or_release_write
-#endif
-#if !defined(AO_HAVE_char_or_acquire_read) && defined(AO_HAVE_char_or_read)
-# define AO_char_or_acquire_read(addr, val) AO_char_or_read(addr, val)
-# define AO_HAVE_char_or_acquire_read
-#endif
-#if !defined(AO_HAVE_char_or_acquire_read) \
-    && defined(AO_HAVE_char_or_acquire)
-# define AO_char_or_acquire_read(addr, val) AO_char_or_acquire(addr, val)
-# define AO_HAVE_char_or_acquire_read
-#endif
-
-/* char_xor */
-#if defined(AO_HAVE_char_compare_and_swap_full) \
-    && !defined(AO_HAVE_char_xor_full)
-  AO_INLINE void
-  AO_char_xor_full(volatile unsigned/**/char *addr, unsigned/**/char value)
-  {
-    unsigned/**/char old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
-                                                           old ^ value)));
-  }
-# define AO_HAVE_char_xor_full
-#endif
-
-#if defined(AO_HAVE_char_xor_full)
-# if !defined(AO_HAVE_char_xor_release)
-#   define AO_char_xor_release(addr, val) AO_char_xor_full(addr, val)
-#   define AO_HAVE_char_xor_release
-# endif
-# if !defined(AO_HAVE_char_xor_acquire)
-#   define AO_char_xor_acquire(addr, val) AO_char_xor_full(addr, val)
-#   define AO_HAVE_char_xor_acquire
-# endif
-# if !defined(AO_HAVE_char_xor_write)
-#   define AO_char_xor_write(addr, val) AO_char_xor_full(addr, val)
-#   define AO_HAVE_char_xor_write
-# endif
-# if !defined(AO_HAVE_char_xor_read)
-#   define AO_char_xor_read(addr, val) AO_char_xor_full(addr, val)
-#   define AO_HAVE_char_xor_read
-# endif
-#endif /* AO_HAVE_char_xor_full */
-
-#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_release)
-# define AO_char_xor(addr, val) AO_char_xor_release(addr, val)
-# define AO_HAVE_char_xor
-#endif
-#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_acquire)
-# define AO_char_xor(addr, val) AO_char_xor_acquire(addr, val)
-# define AO_HAVE_char_xor
-#endif
-#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_write)
-# define AO_char_xor(addr, val) AO_char_xor_write(addr, val)
-# define AO_HAVE_char_xor
-#endif
-#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_read)
-# define AO_char_xor(addr, val) AO_char_xor_read(addr, val)
-# define AO_HAVE_char_xor
-#endif
-
-#if defined(AO_HAVE_char_xor_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_char_xor_full)
-# define AO_char_xor_full(addr, val) \
-                        (AO_nop_full(), AO_char_xor_acquire(addr, val))
-# define AO_HAVE_char_xor_full
-#endif
-
-#if !defined(AO_HAVE_char_xor_release_write) \
-    && defined(AO_HAVE_char_xor_write)
-# define AO_char_xor_release_write(addr, val) AO_char_xor_write(addr, val)
-# define AO_HAVE_char_xor_release_write
-#endif
-#if !defined(AO_HAVE_char_xor_release_write) \
-    && defined(AO_HAVE_char_xor_release)
-# define AO_char_xor_release_write(addr, val) AO_char_xor_release(addr, val)
-# define AO_HAVE_char_xor_release_write
-#endif
-#if !defined(AO_HAVE_char_xor_acquire_read) \
-    && defined(AO_HAVE_char_xor_read)
-# define AO_char_xor_acquire_read(addr, val) AO_char_xor_read(addr, val)
-# define AO_HAVE_char_xor_acquire_read
-#endif
-#if !defined(AO_HAVE_char_xor_acquire_read) \
-    && defined(AO_HAVE_char_xor_acquire)
-# define AO_char_xor_acquire_read(addr, val) AO_char_xor_acquire(addr, val)
-# define AO_HAVE_char_xor_acquire_read
-#endif
-
-/* char_and/or/xor_dd_aquire_read are meaningless.     */
 /*
  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
  *
@@ -1523,821 +778,76 @@
 #   define AO_HAVE_short_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
+/*
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
 
-#if defined(AO_HAVE_short_fetch_compare_and_swap_full) \
-    && !defined(AO_HAVE_short_compare_and_swap_full)
-  AO_INLINE int
-  AO_short_compare_and_swap_full(volatile unsigned/**/short *addr, unsigned/**/short old_val,
-                                 unsigned/**/short new_val)
-  {
-    return AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_short_compare_and_swap_full
+/* int_load */
+#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
+# define AO_int_load_acquire(addr) AO_int_load_full(addr)
+# define AO_HAVE_int_load_acquire
 #endif
 
-#if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_short_compare_and_swap_acquire)
-  AO_INLINE int
-  AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old_val,
-                                    unsigned/**/short new_val)
-  {
-    return AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_short_compare_and_swap_acquire
+#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
+# define AO_int_load(addr) AO_int_load_acquire(addr)
+# define AO_HAVE_int_load
 #endif
 
-#if defined(AO_HAVE_short_fetch_compare_and_swap_release) \
-    && !defined(AO_HAVE_short_compare_and_swap_release)
-  AO_INLINE int
-  AO_short_compare_and_swap_release(volatile unsigned/**/short *addr, unsigned/**/short old_val,
-                                    unsigned/**/short new_val)
-  {
-    return AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_short_compare_and_swap_release
+#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
+# define AO_int_load_read(addr) AO_int_load_full(addr)
+# define AO_HAVE_int_load_read
 #endif
 
-#if defined(AO_HAVE_short_fetch_compare_and_swap) \
-    && !defined(AO_HAVE_short_compare_and_swap)
-  AO_INLINE int
-  AO_short_compare_and_swap(volatile unsigned/**/short *addr, unsigned/**/short old_val,
-                            unsigned/**/short new_val)
-  {
-    return AO_short_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
-  }
-# define AO_HAVE_short_compare_and_swap
+#if !defined(AO_HAVE_int_load_acquire_read) \
+    && defined(AO_HAVE_int_load_acquire)
+# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
+# define AO_HAVE_int_load_acquire_read
 #endif
 
-/* short_fetch_and_add */
-/* We first try to implement fetch_and_add variants in terms of the     */
-/* corresponding compare_and_swap variants to minimize adding barriers. */
-#if defined(AO_HAVE_short_compare_and_swap_full) \
-    && !defined(AO_HAVE_short_fetch_and_add_full)
-  AO_INLINE unsigned/**/short
-  AO_short_fetch_and_add_full(volatile unsigned/**/short *addr, unsigned/**/short incr)
+#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_load_acquire)
+  AO_INLINE unsigned
+  AO_int_load_acquire(const volatile unsigned *addr)
   {
-    unsigned/**/short old;
+    unsigned result = AO_int_load(addr);
 
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
-                                                           old + incr)));
-    return old;
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_full();
+    return result;
   }
-# define AO_HAVE_short_fetch_and_add_full
+# define AO_HAVE_int_load_acquire
 #endif
 
-#if defined(AO_HAVE_short_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_short_fetch_and_add_acquire)
-  AO_INLINE unsigned/**/short
-  AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr)
+#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
+    && !defined(AO_HAVE_int_load_read)
+  AO_INLINE unsigned
+  AO_int_load_read(const volatile unsigned *addr)
   {
-    unsigned/**/short old;
+    unsigned result = AO_int_load(addr);
 
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_short_fetch_and_add_acquire
-#endif
-
-#if defined(AO_HAVE_short_compare_and_swap_release) \
-    && !defined(AO_HAVE_short_fetch_and_add_release)
-  AO_INLINE unsigned/**/short
-  AO_short_fetch_and_add_release(volatile unsigned/**/short *addr, unsigned/**/short incr)
-  {
-    unsigned/**/short old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_short_fetch_and_add_release
-#endif
-
-#if defined(AO_HAVE_short_compare_and_swap) \
-    && !defined(AO_HAVE_short_fetch_and_add)
-  AO_INLINE unsigned/**/short
-  AO_short_fetch_and_add(volatile unsigned/**/short *addr, unsigned/**/short incr)
-  {
-    unsigned/**/short old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr, old,
-                                                      old + incr)));
-    return old;
-  }
-# define AO_HAVE_short_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_short_fetch_and_add_full)
-# if !defined(AO_HAVE_short_fetch_and_add_release)
-#   define AO_short_fetch_and_add_release(addr, val) \
-                                AO_short_fetch_and_add_full(addr, val)
-#   define AO_HAVE_short_fetch_and_add_release
-# endif
-# if !defined(AO_HAVE_short_fetch_and_add_acquire)
-#   define AO_short_fetch_and_add_acquire(addr, val) \
-                                AO_short_fetch_and_add_full(addr, val)
-#   define AO_HAVE_short_fetch_and_add_acquire
-# endif
-# if !defined(AO_HAVE_short_fetch_and_add_write)
-#   define AO_short_fetch_and_add_write(addr, val) \
-                                AO_short_fetch_and_add_full(addr, val)
-#   define AO_HAVE_short_fetch_and_add_write
-# endif
-# if !defined(AO_HAVE_short_fetch_and_add_read)
-#   define AO_short_fetch_and_add_read(addr, val) \
-                                AO_short_fetch_and_add_full(addr, val)
-#   define AO_HAVE_short_fetch_and_add_read
-# endif
-#endif /* AO_HAVE_short_fetch_and_add_full */
-
-#if !defined(AO_HAVE_short_fetch_and_add) \
-    && defined(AO_HAVE_short_fetch_and_add_release)
-# define AO_short_fetch_and_add(addr, val) \
-                                AO_short_fetch_and_add_release(addr, val)
-# define AO_HAVE_short_fetch_and_add
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add) \
-    && defined(AO_HAVE_short_fetch_and_add_acquire)
-# define AO_short_fetch_and_add(addr, val) \
-                                AO_short_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_short_fetch_and_add
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add) \
-    && defined(AO_HAVE_short_fetch_and_add_write)
-# define AO_short_fetch_and_add(addr, val) \
-                                AO_short_fetch_and_add_write(addr, val)
-# define AO_HAVE_short_fetch_and_add
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add) \
-    && defined(AO_HAVE_short_fetch_and_add_read)
-# define AO_short_fetch_and_add(addr, val) \
-                                AO_short_fetch_and_add_read(addr, val)
-# define AO_HAVE_short_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_short_fetch_and_add_acquire) \
-    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_short_fetch_and_add_full)
-# define AO_short_fetch_and_add_full(addr, val) \
-                (AO_nop_full(), AO_short_fetch_and_add_acquire(addr, val))
-# define AO_HAVE_short_fetch_and_add_full
-#endif
-
-#if !defined(AO_HAVE_short_fetch_and_add_release_write) \
-    && defined(AO_HAVE_short_fetch_and_add_write)
-# define AO_short_fetch_and_add_release_write(addr, val) \
-                                AO_short_fetch_and_add_write(addr, val)
-# define AO_HAVE_short_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add_release_write) \
-    && defined(AO_HAVE_short_fetch_and_add_release)
-# define AO_short_fetch_and_add_release_write(addr, val) \
-                                AO_short_fetch_and_add_release(addr, val)
-# define AO_HAVE_short_fetch_and_add_release_write
-#endif
-
-#if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_short_fetch_and_add_read)
-# define AO_short_fetch_and_add_acquire_read(addr, val) \
-                                AO_short_fetch_and_add_read(addr, val)
-# define AO_HAVE_short_fetch_and_add_acquire_read
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_short_fetch_and_add_acquire)
-# define AO_short_fetch_and_add_acquire_read(addr, val) \
-                                AO_short_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_short_fetch_and_add_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_short_fetch_and_add_acquire_read)
-#   define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_short_fetch_and_add_acquire_read(addr, val)
-#   define AO_HAVE_short_fetch_and_add_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_short_fetch_and_add)
-#   define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_short_fetch_and_add(addr, val)
-#   define AO_HAVE_short_fetch_and_add_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* short_fetch_and_add1 */
-#if defined(AO_HAVE_short_fetch_and_add_full) \
-    && !defined(AO_HAVE_short_fetch_and_add1_full)
-# define AO_short_fetch_and_add1_full(addr) \
-                                AO_short_fetch_and_add_full(addr,1)
-# define AO_HAVE_short_fetch_and_add1_full
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_release) \
-    && !defined(AO_HAVE_short_fetch_and_add1_release)
-# define AO_short_fetch_and_add1_release(addr) \
-                                AO_short_fetch_and_add_release(addr,1)
-# define AO_HAVE_short_fetch_and_add1_release
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_short_fetch_and_add1_acquire)
-# define AO_short_fetch_and_add1_acquire(addr) \
-                                AO_short_fetch_and_add_acquire(addr,1)
-# define AO_HAVE_short_fetch_and_add1_acquire
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_write) \
-    && !defined(AO_HAVE_short_fetch_and_add1_write)
-# define AO_short_fetch_and_add1_write(addr) \
-                                AO_short_fetch_and_add_write(addr,1)
-# define AO_HAVE_short_fetch_and_add1_write
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_read) \
-    && !defined(AO_HAVE_short_fetch_and_add1_read)
-# define AO_short_fetch_and_add1_read(addr) \
-                                AO_short_fetch_and_add_read(addr,1)
-# define AO_HAVE_short_fetch_and_add1_read
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_short_fetch_and_add1_release_write)
-# define AO_short_fetch_and_add1_release_write(addr) \
-                                AO_short_fetch_and_add_release_write(addr,1)
-# define AO_HAVE_short_fetch_and_add1_release_write
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_short_fetch_and_add1_acquire_read)
-# define AO_short_fetch_and_add1_acquire_read(addr) \
-                                AO_short_fetch_and_add_acquire_read(addr,1)
-# define AO_HAVE_short_fetch_and_add1_acquire_read
-#endif
-#if defined(AO_HAVE_short_fetch_and_add) \
-    && !defined(AO_HAVE_short_fetch_and_add1)
-# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add(addr,1)
-# define AO_HAVE_short_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_short_fetch_and_add1_full)
-# if !defined(AO_HAVE_short_fetch_and_add1_release)
-#   define AO_short_fetch_and_add1_release(addr) \
-                                AO_short_fetch_and_add1_full(addr)
-#   define AO_HAVE_short_fetch_and_add1_release
-# endif
-# if !defined(AO_HAVE_short_fetch_and_add1_acquire)
-#   define AO_short_fetch_and_add1_acquire(addr) \
-                                AO_short_fetch_and_add1_full(addr)
-#   define AO_HAVE_short_fetch_and_add1_acquire
-# endif
-# if !defined(AO_HAVE_short_fetch_and_add1_write)
-#   define AO_short_fetch_and_add1_write(addr) \
-                                AO_short_fetch_and_add1_full(addr)
-#   define AO_HAVE_short_fetch_and_add1_write
-# endif
-# if !defined(AO_HAVE_short_fetch_and_add1_read)
-#   define AO_short_fetch_and_add1_read(addr) \
-                                AO_short_fetch_and_add1_full(addr)
-#   define AO_HAVE_short_fetch_and_add1_read
-# endif
-#endif /* AO_HAVE_short_fetch_and_add1_full */
-
-#if !defined(AO_HAVE_short_fetch_and_add1) \
-    && defined(AO_HAVE_short_fetch_and_add1_release)
-# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_release(addr)
-# define AO_HAVE_short_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add1) \
-    && defined(AO_HAVE_short_fetch_and_add1_acquire)
-# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_acquire(addr)
-# define AO_HAVE_short_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add1) \
-    && defined(AO_HAVE_short_fetch_and_add1_write)
-# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_write(addr)
-# define AO_HAVE_short_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add1) \
-    && defined(AO_HAVE_short_fetch_and_add1_read)
-# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_read(addr)
-# define AO_HAVE_short_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_short_fetch_and_add1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_fetch_and_add1_full)
-# define AO_short_fetch_and_add1_full(addr) \
-                        (AO_nop_full(), AO_short_fetch_and_add1_acquire(addr))
-# define AO_HAVE_short_fetch_and_add1_full
-#endif
-
-#if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_short_fetch_and_add1_write)
-# define AO_short_fetch_and_add1_release_write(addr) \
-                                AO_short_fetch_and_add1_write(addr)
-# define AO_HAVE_short_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_short_fetch_and_add1_release)
-# define AO_short_fetch_and_add1_release_write(addr) \
-                                AO_short_fetch_and_add1_release(addr)
-# define AO_HAVE_short_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_short_fetch_and_add1_read)
-# define AO_short_fetch_and_add1_acquire_read(addr) \
-                                AO_short_fetch_and_add1_read(addr)
-# define AO_HAVE_short_fetch_and_add1_acquire_read
-#endif
-#if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_short_fetch_and_add1_acquire)
-# define AO_short_fetch_and_add1_acquire_read(addr) \
-                                AO_short_fetch_and_add1_acquire(addr)
-# define AO_HAVE_short_fetch_and_add1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_short_fetch_and_add1_acquire_read)
-#   define AO_short_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_short_fetch_and_add1_acquire_read(addr)
-#   define AO_HAVE_short_fetch_and_add1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_short_fetch_and_add1)
-#   define AO_short_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_short_fetch_and_add1(addr)
-#   define AO_HAVE_short_fetch_and_add1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* short_fetch_and_sub1 */
-#if defined(AO_HAVE_short_fetch_and_add_full) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_full)
-# define AO_short_fetch_and_sub1_full(addr) \
-                AO_short_fetch_and_add_full(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_full
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_release) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_release)
-# define AO_short_fetch_and_sub1_release(addr) \
-                AO_short_fetch_and_add_release(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_release
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_acquire)
-# define AO_short_fetch_and_sub1_acquire(addr) \
-                AO_short_fetch_and_add_acquire(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_acquire
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_write) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_write)
-# define AO_short_fetch_and_sub1_write(addr) \
-                AO_short_fetch_and_add_write(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_write
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_read) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_read)
-# define AO_short_fetch_and_sub1_read(addr) \
-                AO_short_fetch_and_add_read(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_read
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_release_write)
-# define AO_short_fetch_and_sub1_release_write(addr) \
-                AO_short_fetch_and_add_release_write(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_release_write
-#endif
-#if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
-# define AO_short_fetch_and_sub1_acquire_read(addr) \
-                AO_short_fetch_and_add_acquire_read(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1_acquire_read
-#endif
-#if defined(AO_HAVE_short_fetch_and_add) \
-    && !defined(AO_HAVE_short_fetch_and_sub1)
-# define AO_short_fetch_and_sub1(addr) \
-                AO_short_fetch_and_add(addr, (unsigned/**/short)(-1))
-# define AO_HAVE_short_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_short_fetch_and_sub1_full)
-# if !defined(AO_HAVE_short_fetch_and_sub1_release)
-#   define AO_short_fetch_and_sub1_release(addr) \
-                                AO_short_fetch_and_sub1_full(addr)
-#   define AO_HAVE_short_fetch_and_sub1_release
-# endif
-# if !defined(AO_HAVE_short_fetch_and_sub1_acquire)
-#   define AO_short_fetch_and_sub1_acquire(addr) \
-                                AO_short_fetch_and_sub1_full(addr)
-#   define AO_HAVE_short_fetch_and_sub1_acquire
-# endif
-# if !defined(AO_HAVE_short_fetch_and_sub1_write)
-#   define AO_short_fetch_and_sub1_write(addr) \
-                                AO_short_fetch_and_sub1_full(addr)
-#   define AO_HAVE_short_fetch_and_sub1_write
-# endif
-# if !defined(AO_HAVE_short_fetch_and_sub1_read)
-#   define AO_short_fetch_and_sub1_read(addr) \
-                                AO_short_fetch_and_sub1_full(addr)
-#   define AO_HAVE_short_fetch_and_sub1_read
-# endif
-#endif /* AO_HAVE_short_fetch_and_sub1_full */
-
-#if !defined(AO_HAVE_short_fetch_and_sub1) \
-    && defined(AO_HAVE_short_fetch_and_sub1_release)
-# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_release(addr)
-# define AO_HAVE_short_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_short_fetch_and_sub1) \
-    && defined(AO_HAVE_short_fetch_and_sub1_acquire)
-# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_short_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_short_fetch_and_sub1) \
-    && defined(AO_HAVE_short_fetch_and_sub1_write)
-# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_write(addr)
-# define AO_HAVE_short_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_short_fetch_and_sub1) \
-    && defined(AO_HAVE_short_fetch_and_sub1_read)
-# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_read(addr)
-# define AO_HAVE_short_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_short_fetch_and_sub1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_fetch_and_sub1_full)
-# define AO_short_fetch_and_sub1_full(addr) \
-                        (AO_nop_full(), AO_short_fetch_and_sub1_acquire(addr))
-# define AO_HAVE_short_fetch_and_sub1_full
-#endif
-
-#if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_short_fetch_and_sub1_write)
-# define AO_short_fetch_and_sub1_release_write(addr) \
-                                AO_short_fetch_and_sub1_write(addr)
-# define AO_HAVE_short_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_short_fetch_and_sub1_release)
-# define AO_short_fetch_and_sub1_release_write(addr) \
-                                AO_short_fetch_and_sub1_release(addr)
-# define AO_HAVE_short_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_short_fetch_and_sub1_read)
-# define AO_short_fetch_and_sub1_acquire_read(addr) \
-                                AO_short_fetch_and_sub1_read(addr)
-# define AO_HAVE_short_fetch_and_sub1_acquire_read
-#endif
-#if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_short_fetch_and_sub1_acquire)
-# define AO_short_fetch_and_sub1_acquire_read(addr) \
-                                AO_short_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_short_fetch_and_sub1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
-#   define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_short_fetch_and_sub1_acquire_read(addr)
-#   define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_short_fetch_and_sub1)
-#   define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_short_fetch_and_sub1(addr)
-#   define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* short_and */
-#if defined(AO_HAVE_short_compare_and_swap_full) \
-    && !defined(AO_HAVE_short_and_full)
-  AO_INLINE void
-  AO_short_and_full(volatile unsigned/**/short *addr, unsigned/**/short value)
-  {
-    unsigned/**/short old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
-                                                           old & value)));
-  }
-# define AO_HAVE_short_and_full
-#endif
-
-#if defined(AO_HAVE_short_and_full)
-# if !defined(AO_HAVE_short_and_release)
-#   define AO_short_and_release(addr, val) AO_short_and_full(addr, val)
-#   define AO_HAVE_short_and_release
-# endif
-# if !defined(AO_HAVE_short_and_acquire)
-#   define AO_short_and_acquire(addr, val) AO_short_and_full(addr, val)
-#   define AO_HAVE_short_and_acquire
-# endif
-# if !defined(AO_HAVE_short_and_write)
-#   define AO_short_and_write(addr, val) AO_short_and_full(addr, val)
-#   define AO_HAVE_short_and_write
-# endif
-# if !defined(AO_HAVE_short_and_read)
-#   define AO_short_and_read(addr, val) AO_short_and_full(addr, val)
-#   define AO_HAVE_short_and_read
-# endif
-#endif /* AO_HAVE_short_and_full */
-
-#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_release)
-# define AO_short_and(addr, val) AO_short_and_release(addr, val)
-# define AO_HAVE_short_and
-#endif
-#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_acquire)
-# define AO_short_and(addr, val) AO_short_and_acquire(addr, val)
-# define AO_HAVE_short_and
-#endif
-#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_write)
-# define AO_short_and(addr, val) AO_short_and_write(addr, val)
-# define AO_HAVE_short_and
-#endif
-#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_read)
-# define AO_short_and(addr, val) AO_short_and_read(addr, val)
-# define AO_HAVE_short_and
-#endif
-
-#if defined(AO_HAVE_short_and_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_and_full)
-# define AO_short_and_full(addr, val) \
-                        (AO_nop_full(), AO_short_and_acquire(addr, val))
-# define AO_HAVE_short_and_full
-#endif
-
-#if !defined(AO_HAVE_short_and_release_write) \
-    && defined(AO_HAVE_short_and_write)
-# define AO_short_and_release_write(addr, val) AO_short_and_write(addr, val)
-# define AO_HAVE_short_and_release_write
-#endif
-#if !defined(AO_HAVE_short_and_release_write) \
-    && defined(AO_HAVE_short_and_release)
-# define AO_short_and_release_write(addr, val) AO_short_and_release(addr, val)
-# define AO_HAVE_short_and_release_write
-#endif
-#if !defined(AO_HAVE_short_and_acquire_read) \
-    && defined(AO_HAVE_short_and_read)
-# define AO_short_and_acquire_read(addr, val) AO_short_and_read(addr, val)
-# define AO_HAVE_short_and_acquire_read
-#endif
-#if !defined(AO_HAVE_short_and_acquire_read) \
-    && defined(AO_HAVE_short_and_acquire)
-# define AO_short_and_acquire_read(addr, val) AO_short_and_acquire(addr, val)
-# define AO_HAVE_short_and_acquire_read
-#endif
-
-/* short_or */
-#if defined(AO_HAVE_short_compare_and_swap_full) \
-    && !defined(AO_HAVE_short_or_full)
-  AO_INLINE void
-  AO_short_or_full(volatile unsigned/**/short *addr, unsigned/**/short value)
-  {
-    unsigned/**/short old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
-                                                           old | value)));
-  }
-# define AO_HAVE_short_or_full
-#endif
-
-#if defined(AO_HAVE_short_or_full)
-# if !defined(AO_HAVE_short_or_release)
-#   define AO_short_or_release(addr, val) AO_short_or_full(addr, val)
-#   define AO_HAVE_short_or_release
-# endif
-# if !defined(AO_HAVE_short_or_acquire)
-#   define AO_short_or_acquire(addr, val) AO_short_or_full(addr, val)
-#   define AO_HAVE_short_or_acquire
-# endif
-# if !defined(AO_HAVE_short_or_write)
-#   define AO_short_or_write(addr, val) AO_short_or_full(addr, val)
-#   define AO_HAVE_short_or_write
-# endif
-# if !defined(AO_HAVE_short_or_read)
-#   define AO_short_or_read(addr, val) AO_short_or_full(addr, val)
-#   define AO_HAVE_short_or_read
-# endif
-#endif /* AO_HAVE_short_or_full */
-
-#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_release)
-# define AO_short_or(addr, val) AO_short_or_release(addr, val)
-# define AO_HAVE_short_or
-#endif
-#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_acquire)
-# define AO_short_or(addr, val) AO_short_or_acquire(addr, val)
-# define AO_HAVE_short_or
-#endif
-#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_write)
-# define AO_short_or(addr, val) AO_short_or_write(addr, val)
-# define AO_HAVE_short_or
-#endif
-#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_read)
-# define AO_short_or(addr, val) AO_short_or_read(addr, val)
-# define AO_HAVE_short_or
-#endif
-
-#if defined(AO_HAVE_short_or_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_or_full)
-# define AO_short_or_full(addr, val) \
-                        (AO_nop_full(), AO_short_or_acquire(addr, val))
-# define AO_HAVE_short_or_full
-#endif
-
-#if !defined(AO_HAVE_short_or_release_write) \
-    && defined(AO_HAVE_short_or_write)
-# define AO_short_or_release_write(addr, val) AO_short_or_write(addr, val)
-# define AO_HAVE_short_or_release_write
-#endif
-#if !defined(AO_HAVE_short_or_release_write) \
-    && defined(AO_HAVE_short_or_release)
-# define AO_short_or_release_write(addr, val) AO_short_or_release(addr, val)
-# define AO_HAVE_short_or_release_write
-#endif
-#if !defined(AO_HAVE_short_or_acquire_read) && defined(AO_HAVE_short_or_read)
-# define AO_short_or_acquire_read(addr, val) AO_short_or_read(addr, val)
-# define AO_HAVE_short_or_acquire_read
-#endif
-#if !defined(AO_HAVE_short_or_acquire_read) \
-    && defined(AO_HAVE_short_or_acquire)
-# define AO_short_or_acquire_read(addr, val) AO_short_or_acquire(addr, val)
-# define AO_HAVE_short_or_acquire_read
-#endif
-
-/* short_xor */
-#if defined(AO_HAVE_short_compare_and_swap_full) \
-    && !defined(AO_HAVE_short_xor_full)
-  AO_INLINE void
-  AO_short_xor_full(volatile unsigned/**/short *addr, unsigned/**/short value)
-  {
-    unsigned/**/short old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
-                                                           old ^ value)));
-  }
-# define AO_HAVE_short_xor_full
-#endif
-
-#if defined(AO_HAVE_short_xor_full)
-# if !defined(AO_HAVE_short_xor_release)
-#   define AO_short_xor_release(addr, val) AO_short_xor_full(addr, val)
-#   define AO_HAVE_short_xor_release
-# endif
-# if !defined(AO_HAVE_short_xor_acquire)
-#   define AO_short_xor_acquire(addr, val) AO_short_xor_full(addr, val)
-#   define AO_HAVE_short_xor_acquire
-# endif
-# if !defined(AO_HAVE_short_xor_write)
-#   define AO_short_xor_write(addr, val) AO_short_xor_full(addr, val)
-#   define AO_HAVE_short_xor_write
-# endif
-# if !defined(AO_HAVE_short_xor_read)
-#   define AO_short_xor_read(addr, val) AO_short_xor_full(addr, val)
-#   define AO_HAVE_short_xor_read
-# endif
-#endif /* AO_HAVE_short_xor_full */
-
-#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_release)
-# define AO_short_xor(addr, val) AO_short_xor_release(addr, val)
-# define AO_HAVE_short_xor
-#endif
-#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_acquire)
-# define AO_short_xor(addr, val) AO_short_xor_acquire(addr, val)
-# define AO_HAVE_short_xor
-#endif
-#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_write)
-# define AO_short_xor(addr, val) AO_short_xor_write(addr, val)
-# define AO_HAVE_short_xor
-#endif
-#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_read)
-# define AO_short_xor(addr, val) AO_short_xor_read(addr, val)
-# define AO_HAVE_short_xor
-#endif
-
-#if defined(AO_HAVE_short_xor_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_short_xor_full)
-# define AO_short_xor_full(addr, val) \
-                        (AO_nop_full(), AO_short_xor_acquire(addr, val))
-# define AO_HAVE_short_xor_full
-#endif
-
-#if !defined(AO_HAVE_short_xor_release_write) \
-    && defined(AO_HAVE_short_xor_write)
-# define AO_short_xor_release_write(addr, val) AO_short_xor_write(addr, val)
-# define AO_HAVE_short_xor_release_write
-#endif
-#if !defined(AO_HAVE_short_xor_release_write) \
-    && defined(AO_HAVE_short_xor_release)
-# define AO_short_xor_release_write(addr, val) AO_short_xor_release(addr, val)
-# define AO_HAVE_short_xor_release_write
-#endif
-#if !defined(AO_HAVE_short_xor_acquire_read) \
-    && defined(AO_HAVE_short_xor_read)
-# define AO_short_xor_acquire_read(addr, val) AO_short_xor_read(addr, val)
-# define AO_HAVE_short_xor_acquire_read
-#endif
-#if !defined(AO_HAVE_short_xor_acquire_read) \
-    && defined(AO_HAVE_short_xor_acquire)
-# define AO_short_xor_acquire_read(addr, val) AO_short_xor_acquire(addr, val)
-# define AO_HAVE_short_xor_acquire_read
-#endif
-
-/* short_and/or/xor_dd_aquire_read are meaningless.     */
-/*
- * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* int_load */
-#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
-# define AO_int_load_acquire(addr) AO_int_load_full(addr)
-# define AO_HAVE_int_load_acquire
-#endif
-
-#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
-# define AO_int_load(addr) AO_int_load_acquire(addr)
-# define AO_HAVE_int_load
-#endif
-
-#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
-# define AO_int_load_read(addr) AO_int_load_full(addr)
-# define AO_HAVE_int_load_read
-#endif
-
-#if !defined(AO_HAVE_int_load_acquire_read) \
-    && defined(AO_HAVE_int_load_acquire)
-# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
-# define AO_HAVE_int_load_acquire_read
-#endif
-
-#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_load_acquire)
-  AO_INLINE unsigned
-  AO_int_load_acquire(const volatile unsigned *addr)
-  {
-    unsigned result = AO_int_load(addr);
-
-    /* Acquire barrier would be useless, since the load could be delayed    */
-    /* beyond it.                                                           */
-    AO_nop_full();
-    return result;
-  }
-# define AO_HAVE_int_load_acquire
-#endif
-
-#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
-    && !defined(AO_HAVE_int_load_read)
-  AO_INLINE unsigned
-  AO_int_load_read(const volatile unsigned *addr)
-  {
-    unsigned result = AO_int_load(addr);
-
-    /* Acquire barrier would be useless, since the load could be delayed    */
-    /* beyond it.                                                           */
-    AO_nop_read();
-    return result;
+    /* Acquire barrier would be useless, since the load could be delayed    */
+    /* beyond it.                                                           */
+    AO_nop_read();
+    return result;
   }
 # define AO_HAVE_int_load_read
 #endif
@@ -2579,830 +1089,85 @@
                 AO_int_compare_and_swap_full(addr, old, new_val)
 #   define AO_HAVE_int_compare_and_swap_write
 # endif
-# if !defined(AO_HAVE_int_compare_and_swap_read)
-#   define AO_int_compare_and_swap_read(addr, old, new_val) \
-                AO_int_compare_and_swap_full(addr, old, new_val)
-#   define AO_HAVE_int_compare_and_swap_read
-# endif
-#endif /* AO_HAVE_int_compare_and_swap_full */
-
-#if !defined(AO_HAVE_int_compare_and_swap) \
-    && defined(AO_HAVE_int_compare_and_swap_release)
-# define AO_int_compare_and_swap(addr, old, new_val) \
-                AO_int_compare_and_swap_release(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap
-#endif
-#if !defined(AO_HAVE_int_compare_and_swap) \
-    && defined(AO_HAVE_int_compare_and_swap_acquire)
-# define AO_int_compare_and_swap(addr, old, new_val) \
-                AO_int_compare_and_swap_acquire(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap
-#endif
-#if !defined(AO_HAVE_int_compare_and_swap) \
-    && defined(AO_HAVE_int_compare_and_swap_write)
-# define AO_int_compare_and_swap(addr, old, new_val) \
-                AO_int_compare_and_swap_write(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap
-#endif
-#if !defined(AO_HAVE_int_compare_and_swap) \
-    && defined(AO_HAVE_int_compare_and_swap_read)
-# define AO_int_compare_and_swap(addr, old, new_val) \
-                AO_int_compare_and_swap_read(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap
-#endif
-
-#if defined(AO_HAVE_int_compare_and_swap_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_compare_and_swap_full)
-# define AO_int_compare_and_swap_full(addr, old, new_val) \
-                (AO_nop_full(), \
-                 AO_int_compare_and_swap_acquire(addr, old, new_val))
-# define AO_HAVE_int_compare_and_swap_full
-#endif
-
-#if !defined(AO_HAVE_int_compare_and_swap_release_write) \
-    && defined(AO_HAVE_int_compare_and_swap_write)
-# define AO_int_compare_and_swap_release_write(addr, old, new_val) \
-                AO_int_compare_and_swap_write(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap_release_write
-#endif
-#if !defined(AO_HAVE_int_compare_and_swap_release_write) \
-    && defined(AO_HAVE_int_compare_and_swap_release)
-# define AO_int_compare_and_swap_release_write(addr, old, new_val) \
-                AO_int_compare_and_swap_release(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap_release_write
-#endif
-#if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
-    && defined(AO_HAVE_int_compare_and_swap_read)
-# define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
-                AO_int_compare_and_swap_read(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap_acquire_read
-#endif
-#if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
-    && defined(AO_HAVE_int_compare_and_swap_acquire)
-# define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
-                AO_int_compare_and_swap_acquire(addr, old, new_val)
-# define AO_HAVE_int_compare_and_swap_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_int_compare_and_swap_acquire_read)
-#   define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
-                AO_int_compare_and_swap_acquire_read(addr, old, new_val)
-#   define AO_HAVE_int_compare_and_swap_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_int_compare_and_swap)
-#   define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
-                AO_int_compare_and_swap(addr, old, new_val)
-#   define AO_HAVE_int_compare_and_swap_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-#if defined(AO_HAVE_int_fetch_compare_and_swap_full) \
-    && !defined(AO_HAVE_int_compare_and_swap_full)
-  AO_INLINE int
-  AO_int_compare_and_swap_full(volatile unsigned *addr, unsigned old_val,
-                                 unsigned new_val)
-  {
-    return AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_int_compare_and_swap_full
-#endif
-
-#if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_int_compare_and_swap_acquire)
-  AO_INLINE int
-  AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old_val,
-                                    unsigned new_val)
-  {
-    return AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_int_compare_and_swap_acquire
-#endif
-
-#if defined(AO_HAVE_int_fetch_compare_and_swap_release) \
-    && !defined(AO_HAVE_int_compare_and_swap_release)
-  AO_INLINE int
-  AO_int_compare_and_swap_release(volatile unsigned *addr, unsigned old_val,
-                                    unsigned new_val)
-  {
-    return AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_int_compare_and_swap_release
-#endif
-
-#if defined(AO_HAVE_int_fetch_compare_and_swap) \
-    && !defined(AO_HAVE_int_compare_and_swap)
-  AO_INLINE int
-  AO_int_compare_and_swap(volatile unsigned *addr, unsigned old_val,
-                            unsigned new_val)
-  {
-    return AO_int_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
-  }
-# define AO_HAVE_int_compare_and_swap
-#endif
-
-/* int_fetch_and_add */
-/* We first try to implement fetch_and_add variants in terms of the     */
-/* corresponding compare_and_swap variants to minimize adding barriers. */
-#if defined(AO_HAVE_int_compare_and_swap_full) \
-    && !defined(AO_HAVE_int_fetch_and_add_full)
-  AO_INLINE unsigned
-  AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
-                                                           old + incr)));
-    return old;
-  }
-# define AO_HAVE_int_fetch_and_add_full
-#endif
-
-#if defined(AO_HAVE_int_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_int_fetch_and_add_acquire)
-  AO_INLINE unsigned
-  AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_int_fetch_and_add_acquire
-#endif
-
-#if defined(AO_HAVE_int_compare_and_swap_release) \
-    && !defined(AO_HAVE_int_fetch_and_add_release)
-  AO_INLINE unsigned
-  AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_int_fetch_and_add_release
-#endif
-
-#if defined(AO_HAVE_int_compare_and_swap) \
-    && !defined(AO_HAVE_int_fetch_and_add)
-  AO_INLINE unsigned
-  AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr, old,
-                                                      old + incr)));
-    return old;
-  }
-# define AO_HAVE_int_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_int_fetch_and_add_full)
-# if !defined(AO_HAVE_int_fetch_and_add_release)
-#   define AO_int_fetch_and_add_release(addr, val) \
-                                AO_int_fetch_and_add_full(addr, val)
-#   define AO_HAVE_int_fetch_and_add_release
-# endif
-# if !defined(AO_HAVE_int_fetch_and_add_acquire)
-#   define AO_int_fetch_and_add_acquire(addr, val) \
-                                AO_int_fetch_and_add_full(addr, val)
-#   define AO_HAVE_int_fetch_and_add_acquire
-# endif
-# if !defined(AO_HAVE_int_fetch_and_add_write)
-#   define AO_int_fetch_and_add_write(addr, val) \
-                                AO_int_fetch_and_add_full(addr, val)
-#   define AO_HAVE_int_fetch_and_add_write
-# endif
-# if !defined(AO_HAVE_int_fetch_and_add_read)
-#   define AO_int_fetch_and_add_read(addr, val) \
-                                AO_int_fetch_and_add_full(addr, val)
-#   define AO_HAVE_int_fetch_and_add_read
-# endif
-#endif /* AO_HAVE_int_fetch_and_add_full */
-
-#if !defined(AO_HAVE_int_fetch_and_add) \
-    && defined(AO_HAVE_int_fetch_and_add_release)
-# define AO_int_fetch_and_add(addr, val) \
-                                AO_int_fetch_and_add_release(addr, val)
-# define AO_HAVE_int_fetch_and_add
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add) \
-    && defined(AO_HAVE_int_fetch_and_add_acquire)
-# define AO_int_fetch_and_add(addr, val) \
-                                AO_int_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_int_fetch_and_add
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add) \
-    && defined(AO_HAVE_int_fetch_and_add_write)
-# define AO_int_fetch_and_add(addr, val) \
-                                AO_int_fetch_and_add_write(addr, val)
-# define AO_HAVE_int_fetch_and_add
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add) \
-    && defined(AO_HAVE_int_fetch_and_add_read)
-# define AO_int_fetch_and_add(addr, val) \
-                                AO_int_fetch_and_add_read(addr, val)
-# define AO_HAVE_int_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_int_fetch_and_add_acquire) \
-    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_int_fetch_and_add_full)
-# define AO_int_fetch_and_add_full(addr, val) \
-                (AO_nop_full(), AO_int_fetch_and_add_acquire(addr, val))
-# define AO_HAVE_int_fetch_and_add_full
-#endif
-
-#if !defined(AO_HAVE_int_fetch_and_add_release_write) \
-    && defined(AO_HAVE_int_fetch_and_add_write)
-# define AO_int_fetch_and_add_release_write(addr, val) \
-                                AO_int_fetch_and_add_write(addr, val)
-# define AO_HAVE_int_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add_release_write) \
-    && defined(AO_HAVE_int_fetch_and_add_release)
-# define AO_int_fetch_and_add_release_write(addr, val) \
-                                AO_int_fetch_and_add_release(addr, val)
-# define AO_HAVE_int_fetch_and_add_release_write
-#endif
-
-#if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_int_fetch_and_add_read)
-# define AO_int_fetch_and_add_acquire_read(addr, val) \
-                                AO_int_fetch_and_add_read(addr, val)
-# define AO_HAVE_int_fetch_and_add_acquire_read
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_int_fetch_and_add_acquire)
-# define AO_int_fetch_and_add_acquire_read(addr, val) \
-                                AO_int_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_int_fetch_and_add_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_int_fetch_and_add_acquire_read)
-#   define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_int_fetch_and_add_acquire_read(addr, val)
-#   define AO_HAVE_int_fetch_and_add_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_int_fetch_and_add)
-#   define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_int_fetch_and_add(addr, val)
-#   define AO_HAVE_int_fetch_and_add_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* int_fetch_and_add1 */
-#if defined(AO_HAVE_int_fetch_and_add_full) \
-    && !defined(AO_HAVE_int_fetch_and_add1_full)
-# define AO_int_fetch_and_add1_full(addr) \
-                                AO_int_fetch_and_add_full(addr,1)
-# define AO_HAVE_int_fetch_and_add1_full
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_release) \
-    && !defined(AO_HAVE_int_fetch_and_add1_release)
-# define AO_int_fetch_and_add1_release(addr) \
-                                AO_int_fetch_and_add_release(addr,1)
-# define AO_HAVE_int_fetch_and_add1_release
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_int_fetch_and_add1_acquire)
-# define AO_int_fetch_and_add1_acquire(addr) \
-                                AO_int_fetch_and_add_acquire(addr,1)
-# define AO_HAVE_int_fetch_and_add1_acquire
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_write) \
-    && !defined(AO_HAVE_int_fetch_and_add1_write)
-# define AO_int_fetch_and_add1_write(addr) \
-                                AO_int_fetch_and_add_write(addr,1)
-# define AO_HAVE_int_fetch_and_add1_write
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_read) \
-    && !defined(AO_HAVE_int_fetch_and_add1_read)
-# define AO_int_fetch_and_add1_read(addr) \
-                                AO_int_fetch_and_add_read(addr,1)
-# define AO_HAVE_int_fetch_and_add1_read
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_int_fetch_and_add1_release_write)
-# define AO_int_fetch_and_add1_release_write(addr) \
-                                AO_int_fetch_and_add_release_write(addr,1)
-# define AO_HAVE_int_fetch_and_add1_release_write
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_int_fetch_and_add1_acquire_read)
-# define AO_int_fetch_and_add1_acquire_read(addr) \
-                                AO_int_fetch_and_add_acquire_read(addr,1)
-# define AO_HAVE_int_fetch_and_add1_acquire_read
-#endif
-#if defined(AO_HAVE_int_fetch_and_add) \
-    && !defined(AO_HAVE_int_fetch_and_add1)
-# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add(addr,1)
-# define AO_HAVE_int_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_int_fetch_and_add1_full)
-# if !defined(AO_HAVE_int_fetch_and_add1_release)
-#   define AO_int_fetch_and_add1_release(addr) \
-                                AO_int_fetch_and_add1_full(addr)
-#   define AO_HAVE_int_fetch_and_add1_release
-# endif
-# if !defined(AO_HAVE_int_fetch_and_add1_acquire)
-#   define AO_int_fetch_and_add1_acquire(addr) \
-                                AO_int_fetch_and_add1_full(addr)
-#   define AO_HAVE_int_fetch_and_add1_acquire
-# endif
-# if !defined(AO_HAVE_int_fetch_and_add1_write)
-#   define AO_int_fetch_and_add1_write(addr) \
-                                AO_int_fetch_and_add1_full(addr)
-#   define AO_HAVE_int_fetch_and_add1_write
-# endif
-# if !defined(AO_HAVE_int_fetch_and_add1_read)
-#   define AO_int_fetch_and_add1_read(addr) \
-                                AO_int_fetch_and_add1_full(addr)
-#   define AO_HAVE_int_fetch_and_add1_read
-# endif
-#endif /* AO_HAVE_int_fetch_and_add1_full */
-
-#if !defined(AO_HAVE_int_fetch_and_add1) \
-    && defined(AO_HAVE_int_fetch_and_add1_release)
-# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_release(addr)
-# define AO_HAVE_int_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add1) \
-    && defined(AO_HAVE_int_fetch_and_add1_acquire)
-# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_acquire(addr)
-# define AO_HAVE_int_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add1) \
-    && defined(AO_HAVE_int_fetch_and_add1_write)
-# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_write(addr)
-# define AO_HAVE_int_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add1) \
-    && defined(AO_HAVE_int_fetch_and_add1_read)
-# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_read(addr)
-# define AO_HAVE_int_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_int_fetch_and_add1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_fetch_and_add1_full)
-# define AO_int_fetch_and_add1_full(addr) \
-                        (AO_nop_full(), AO_int_fetch_and_add1_acquire(addr))
-# define AO_HAVE_int_fetch_and_add1_full
-#endif
-
-#if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_int_fetch_and_add1_write)
-# define AO_int_fetch_and_add1_release_write(addr) \
-                                AO_int_fetch_and_add1_write(addr)
-# define AO_HAVE_int_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_int_fetch_and_add1_release)
-# define AO_int_fetch_and_add1_release_write(addr) \
-                                AO_int_fetch_and_add1_release(addr)
-# define AO_HAVE_int_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_int_fetch_and_add1_read)
-# define AO_int_fetch_and_add1_acquire_read(addr) \
-                                AO_int_fetch_and_add1_read(addr)
-# define AO_HAVE_int_fetch_and_add1_acquire_read
-#endif
-#if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_int_fetch_and_add1_acquire)
-# define AO_int_fetch_and_add1_acquire_read(addr) \
-                                AO_int_fetch_and_add1_acquire(addr)
-# define AO_HAVE_int_fetch_and_add1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_int_fetch_and_add1_acquire_read)
-#   define AO_int_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_int_fetch_and_add1_acquire_read(addr)
-#   define AO_HAVE_int_fetch_and_add1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_int_fetch_and_add1)
-#   define AO_int_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_int_fetch_and_add1(addr)
-#   define AO_HAVE_int_fetch_and_add1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* int_fetch_and_sub1 */
-#if defined(AO_HAVE_int_fetch_and_add_full) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_full)
-# define AO_int_fetch_and_sub1_full(addr) \
-                AO_int_fetch_and_add_full(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_full
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_release) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_release)
-# define AO_int_fetch_and_sub1_release(addr) \
-                AO_int_fetch_and_add_release(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_release
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_acquire)
-# define AO_int_fetch_and_sub1_acquire(addr) \
-                AO_int_fetch_and_add_acquire(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_acquire
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_write) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_write)
-# define AO_int_fetch_and_sub1_write(addr) \
-                AO_int_fetch_and_add_write(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_write
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_read) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_read)
-# define AO_int_fetch_and_sub1_read(addr) \
-                AO_int_fetch_and_add_read(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_read
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_release_write)
-# define AO_int_fetch_and_sub1_release_write(addr) \
-                AO_int_fetch_and_add_release_write(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_release_write
-#endif
-#if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
-# define AO_int_fetch_and_sub1_acquire_read(addr) \
-                AO_int_fetch_and_add_acquire_read(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1_acquire_read
-#endif
-#if defined(AO_HAVE_int_fetch_and_add) \
-    && !defined(AO_HAVE_int_fetch_and_sub1)
-# define AO_int_fetch_and_sub1(addr) \
-                AO_int_fetch_and_add(addr, (unsigned)(-1))
-# define AO_HAVE_int_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_int_fetch_and_sub1_full)
-# if !defined(AO_HAVE_int_fetch_and_sub1_release)
-#   define AO_int_fetch_and_sub1_release(addr) \
-                                AO_int_fetch_and_sub1_full(addr)
-#   define AO_HAVE_int_fetch_and_sub1_release
-# endif
-# if !defined(AO_HAVE_int_fetch_and_sub1_acquire)
-#   define AO_int_fetch_and_sub1_acquire(addr) \
-                                AO_int_fetch_and_sub1_full(addr)
-#   define AO_HAVE_int_fetch_and_sub1_acquire
-# endif
-# if !defined(AO_HAVE_int_fetch_and_sub1_write)
-#   define AO_int_fetch_and_sub1_write(addr) \
-                                AO_int_fetch_and_sub1_full(addr)
-#   define AO_HAVE_int_fetch_and_sub1_write
-# endif
-# if !defined(AO_HAVE_int_fetch_and_sub1_read)
-#   define AO_int_fetch_and_sub1_read(addr) \
-                                AO_int_fetch_and_sub1_full(addr)
-#   define AO_HAVE_int_fetch_and_sub1_read
-# endif
-#endif /* AO_HAVE_int_fetch_and_sub1_full */
-
-#if !defined(AO_HAVE_int_fetch_and_sub1) \
-    && defined(AO_HAVE_int_fetch_and_sub1_release)
-# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_release(addr)
-# define AO_HAVE_int_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_int_fetch_and_sub1) \
-    && defined(AO_HAVE_int_fetch_and_sub1_acquire)
-# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_int_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_int_fetch_and_sub1) \
-    && defined(AO_HAVE_int_fetch_and_sub1_write)
-# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_write(addr)
-# define AO_HAVE_int_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_int_fetch_and_sub1) \
-    && defined(AO_HAVE_int_fetch_and_sub1_read)
-# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_read(addr)
-# define AO_HAVE_int_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_int_fetch_and_sub1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_fetch_and_sub1_full)
-# define AO_int_fetch_and_sub1_full(addr) \
-                        (AO_nop_full(), AO_int_fetch_and_sub1_acquire(addr))
-# define AO_HAVE_int_fetch_and_sub1_full
-#endif
-
-#if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_int_fetch_and_sub1_write)
-# define AO_int_fetch_and_sub1_release_write(addr) \
-                                AO_int_fetch_and_sub1_write(addr)
-# define AO_HAVE_int_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_int_fetch_and_sub1_release)
-# define AO_int_fetch_and_sub1_release_write(addr) \
-                                AO_int_fetch_and_sub1_release(addr)
-# define AO_HAVE_int_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_int_fetch_and_sub1_read)
-# define AO_int_fetch_and_sub1_acquire_read(addr) \
-                                AO_int_fetch_and_sub1_read(addr)
-# define AO_HAVE_int_fetch_and_sub1_acquire_read
-#endif
-#if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_int_fetch_and_sub1_acquire)
-# define AO_int_fetch_and_sub1_acquire_read(addr) \
-                                AO_int_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_int_fetch_and_sub1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
-#   define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_int_fetch_and_sub1_acquire_read(addr)
-#   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_int_fetch_and_sub1)
-#   define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_int_fetch_and_sub1(addr)
-#   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* int_and */
-#if defined(AO_HAVE_int_compare_and_swap_full) \
-    && !defined(AO_HAVE_int_and_full)
-  AO_INLINE void
-  AO_int_and_full(volatile unsigned *addr, unsigned value)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
-                                                           old & value)));
-  }
-# define AO_HAVE_int_and_full
-#endif
-
-#if defined(AO_HAVE_int_and_full)
-# if !defined(AO_HAVE_int_and_release)
-#   define AO_int_and_release(addr, val) AO_int_and_full(addr, val)
-#   define AO_HAVE_int_and_release
-# endif
-# if !defined(AO_HAVE_int_and_acquire)
-#   define AO_int_and_acquire(addr, val) AO_int_and_full(addr, val)
-#   define AO_HAVE_int_and_acquire
-# endif
-# if !defined(AO_HAVE_int_and_write)
-#   define AO_int_and_write(addr, val) AO_int_and_full(addr, val)
-#   define AO_HAVE_int_and_write
-# endif
-# if !defined(AO_HAVE_int_and_read)
-#   define AO_int_and_read(addr, val) AO_int_and_full(addr, val)
-#   define AO_HAVE_int_and_read
-# endif
-#endif /* AO_HAVE_int_and_full */
-
-#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_release)
-# define AO_int_and(addr, val) AO_int_and_release(addr, val)
-# define AO_HAVE_int_and
-#endif
-#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_acquire)
-# define AO_int_and(addr, val) AO_int_and_acquire(addr, val)
-# define AO_HAVE_int_and
-#endif
-#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_write)
-# define AO_int_and(addr, val) AO_int_and_write(addr, val)
-# define AO_HAVE_int_and
-#endif
-#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_read)
-# define AO_int_and(addr, val) AO_int_and_read(addr, val)
-# define AO_HAVE_int_and
-#endif
-
-#if defined(AO_HAVE_int_and_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_and_full)
-# define AO_int_and_full(addr, val) \
-                        (AO_nop_full(), AO_int_and_acquire(addr, val))
-# define AO_HAVE_int_and_full
-#endif
-
-#if !defined(AO_HAVE_int_and_release_write) \
-    && defined(AO_HAVE_int_and_write)
-# define AO_int_and_release_write(addr, val) AO_int_and_write(addr, val)
-# define AO_HAVE_int_and_release_write
-#endif
-#if !defined(AO_HAVE_int_and_release_write) \
-    && defined(AO_HAVE_int_and_release)
-# define AO_int_and_release_write(addr, val) AO_int_and_release(addr, val)
-# define AO_HAVE_int_and_release_write
-#endif
-#if !defined(AO_HAVE_int_and_acquire_read) \
-    && defined(AO_HAVE_int_and_read)
-# define AO_int_and_acquire_read(addr, val) AO_int_and_read(addr, val)
-# define AO_HAVE_int_and_acquire_read
-#endif
-#if !defined(AO_HAVE_int_and_acquire_read) \
-    && defined(AO_HAVE_int_and_acquire)
-# define AO_int_and_acquire_read(addr, val) AO_int_and_acquire(addr, val)
-# define AO_HAVE_int_and_acquire_read
-#endif
-
-/* int_or */
-#if defined(AO_HAVE_int_compare_and_swap_full) \
-    && !defined(AO_HAVE_int_or_full)
-  AO_INLINE void
-  AO_int_or_full(volatile unsigned *addr, unsigned value)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
-                                                           old | value)));
-  }
-# define AO_HAVE_int_or_full
-#endif
-
-#if defined(AO_HAVE_int_or_full)
-# if !defined(AO_HAVE_int_or_release)
-#   define AO_int_or_release(addr, val) AO_int_or_full(addr, val)
-#   define AO_HAVE_int_or_release
-# endif
-# if !defined(AO_HAVE_int_or_acquire)
-#   define AO_int_or_acquire(addr, val) AO_int_or_full(addr, val)
-#   define AO_HAVE_int_or_acquire
-# endif
-# if !defined(AO_HAVE_int_or_write)
-#   define AO_int_or_write(addr, val) AO_int_or_full(addr, val)
-#   define AO_HAVE_int_or_write
-# endif
-# if !defined(AO_HAVE_int_or_read)
-#   define AO_int_or_read(addr, val) AO_int_or_full(addr, val)
-#   define AO_HAVE_int_or_read
+# if !defined(AO_HAVE_int_compare_and_swap_read)
+#   define AO_int_compare_and_swap_read(addr, old, new_val) \
+                AO_int_compare_and_swap_full(addr, old, new_val)
+#   define AO_HAVE_int_compare_and_swap_read
 # endif
-#endif /* AO_HAVE_int_or_full */
+#endif /* AO_HAVE_int_compare_and_swap_full */
 
-#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_release)
-# define AO_int_or(addr, val) AO_int_or_release(addr, val)
-# define AO_HAVE_int_or
+#if !defined(AO_HAVE_int_compare_and_swap) \
+    && defined(AO_HAVE_int_compare_and_swap_release)
+# define AO_int_compare_and_swap(addr, old, new_val) \
+                AO_int_compare_and_swap_release(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap
 #endif
-#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_acquire)
-# define AO_int_or(addr, val) AO_int_or_acquire(addr, val)
-# define AO_HAVE_int_or
+#if !defined(AO_HAVE_int_compare_and_swap) \
+    && defined(AO_HAVE_int_compare_and_swap_acquire)
+# define AO_int_compare_and_swap(addr, old, new_val) \
+                AO_int_compare_and_swap_acquire(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap
 #endif
-#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_write)
-# define AO_int_or(addr, val) AO_int_or_write(addr, val)
-# define AO_HAVE_int_or
+#if !defined(AO_HAVE_int_compare_and_swap) \
+    && defined(AO_HAVE_int_compare_and_swap_write)
+# define AO_int_compare_and_swap(addr, old, new_val) \
+                AO_int_compare_and_swap_write(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap
 #endif
-#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_read)
-# define AO_int_or(addr, val) AO_int_or_read(addr, val)
-# define AO_HAVE_int_or
+#if !defined(AO_HAVE_int_compare_and_swap) \
+    && defined(AO_HAVE_int_compare_and_swap_read)
+# define AO_int_compare_and_swap(addr, old, new_val) \
+                AO_int_compare_and_swap_read(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap
 #endif
 
-#if defined(AO_HAVE_int_or_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_or_full)
-# define AO_int_or_full(addr, val) \
-                        (AO_nop_full(), AO_int_or_acquire(addr, val))
-# define AO_HAVE_int_or_full
+#if defined(AO_HAVE_int_compare_and_swap_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_int_compare_and_swap_full)
+# define AO_int_compare_and_swap_full(addr, old, new_val) \
+                (AO_nop_full(), \
+                 AO_int_compare_and_swap_acquire(addr, old, new_val))
+# define AO_HAVE_int_compare_and_swap_full
 #endif
 
-#if !defined(AO_HAVE_int_or_release_write) \
-    && defined(AO_HAVE_int_or_write)
-# define AO_int_or_release_write(addr, val) AO_int_or_write(addr, val)
-# define AO_HAVE_int_or_release_write
-#endif
-#if !defined(AO_HAVE_int_or_release_write) \
-    && defined(AO_HAVE_int_or_release)
-# define AO_int_or_release_write(addr, val) AO_int_or_release(addr, val)
-# define AO_HAVE_int_or_release_write
+#if !defined(AO_HAVE_int_compare_and_swap_release_write) \
+    && defined(AO_HAVE_int_compare_and_swap_write)
+# define AO_int_compare_and_swap_release_write(addr, old, new_val) \
+                AO_int_compare_and_swap_write(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap_release_write
 #endif
-#if !defined(AO_HAVE_int_or_acquire_read) && defined(AO_HAVE_int_or_read)
-# define AO_int_or_acquire_read(addr, val) AO_int_or_read(addr, val)
-# define AO_HAVE_int_or_acquire_read
+#if !defined(AO_HAVE_int_compare_and_swap_release_write) \
+    && defined(AO_HAVE_int_compare_and_swap_release)
+# define AO_int_compare_and_swap_release_write(addr, old, new_val) \
+                AO_int_compare_and_swap_release(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap_release_write
 #endif
-#if !defined(AO_HAVE_int_or_acquire_read) \
-    && defined(AO_HAVE_int_or_acquire)
-# define AO_int_or_acquire_read(addr, val) AO_int_or_acquire(addr, val)
-# define AO_HAVE_int_or_acquire_read
+#if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
+    && defined(AO_HAVE_int_compare_and_swap_read)
+# define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
+                AO_int_compare_and_swap_read(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap_acquire_read
 #endif
-
-/* int_xor */
-#if defined(AO_HAVE_int_compare_and_swap_full) \
-    && !defined(AO_HAVE_int_xor_full)
-  AO_INLINE void
-  AO_int_xor_full(volatile unsigned *addr, unsigned value)
-  {
-    unsigned old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
-                                                           old ^ value)));
-  }
-# define AO_HAVE_int_xor_full
+#if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
+    && defined(AO_HAVE_int_compare_and_swap_acquire)
+# define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
+                AO_int_compare_and_swap_acquire(addr, old, new_val)
+# define AO_HAVE_int_compare_and_swap_acquire_read
 #endif
 
-#if defined(AO_HAVE_int_xor_full)
-# if !defined(AO_HAVE_int_xor_release)
-#   define AO_int_xor_release(addr, val) AO_int_xor_full(addr, val)
-#   define AO_HAVE_int_xor_release
-# endif
-# if !defined(AO_HAVE_int_xor_acquire)
-#   define AO_int_xor_acquire(addr, val) AO_int_xor_full(addr, val)
-#   define AO_HAVE_int_xor_acquire
-# endif
-# if !defined(AO_HAVE_int_xor_write)
-#   define AO_int_xor_write(addr, val) AO_int_xor_full(addr, val)
-#   define AO_HAVE_int_xor_write
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_int_compare_and_swap_acquire_read)
+#   define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
+                AO_int_compare_and_swap_acquire_read(addr, old, new_val)
+#   define AO_HAVE_int_compare_and_swap_dd_acquire_read
 # endif
-# if !defined(AO_HAVE_int_xor_read)
-#   define AO_int_xor_read(addr, val) AO_int_xor_full(addr, val)
-#   define AO_HAVE_int_xor_read
+#else
+# if defined(AO_HAVE_int_compare_and_swap)
+#   define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
+                AO_int_compare_and_swap(addr, old, new_val)
+#   define AO_HAVE_int_compare_and_swap_dd_acquire_read
 # endif
-#endif /* AO_HAVE_int_xor_full */
-
-#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_release)
-# define AO_int_xor(addr, val) AO_int_xor_release(addr, val)
-# define AO_HAVE_int_xor
-#endif
-#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_acquire)
-# define AO_int_xor(addr, val) AO_int_xor_acquire(addr, val)
-# define AO_HAVE_int_xor
-#endif
-#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_write)
-# define AO_int_xor(addr, val) AO_int_xor_write(addr, val)
-# define AO_HAVE_int_xor
-#endif
-#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_read)
-# define AO_int_xor(addr, val) AO_int_xor_read(addr, val)
-# define AO_HAVE_int_xor
-#endif
-
-#if defined(AO_HAVE_int_xor_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_int_xor_full)
-# define AO_int_xor_full(addr, val) \
-                        (AO_nop_full(), AO_int_xor_acquire(addr, val))
-# define AO_HAVE_int_xor_full
-#endif
-
-#if !defined(AO_HAVE_int_xor_release_write) \
-    && defined(AO_HAVE_int_xor_write)
-# define AO_int_xor_release_write(addr, val) AO_int_xor_write(addr, val)
-# define AO_HAVE_int_xor_release_write
-#endif
-#if !defined(AO_HAVE_int_xor_release_write) \
-    && defined(AO_HAVE_int_xor_release)
-# define AO_int_xor_release_write(addr, val) AO_int_xor_release(addr, val)
-# define AO_HAVE_int_xor_release_write
-#endif
-#if !defined(AO_HAVE_int_xor_acquire_read) \
-    && defined(AO_HAVE_int_xor_read)
-# define AO_int_xor_acquire_read(addr, val) AO_int_xor_read(addr, val)
-# define AO_HAVE_int_xor_acquire_read
-#endif
-#if !defined(AO_HAVE_int_xor_acquire_read) \
-    && defined(AO_HAVE_int_xor_acquire)
-# define AO_int_xor_acquire_read(addr, val) AO_int_xor_acquire(addr, val)
-# define AO_HAVE_int_xor_acquire_read
-#endif
-
-/* int_and/or/xor_dd_aquire_read are meaningless.     */
+#endif /* !AO_NO_DD_ORDERING */
 /*
  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
  *
@@ -3793,748 +1558,3 @@
 #   define AO_HAVE_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
-
-#if defined(AO_HAVE_fetch_compare_and_swap_full) \
-    && !defined(AO_HAVE_compare_and_swap_full)
-  AO_INLINE int
-  AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
-                                 AO_t new_val)
-  {
-    return AO_fetch_compare_and_swap_full(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_compare_and_swap_full
-#endif
-
-#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_compare_and_swap_acquire)
-  AO_INLINE int
-  AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
-                                    AO_t new_val)
-  {
-    return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_compare_and_swap_acquire
-#endif
-
-#if defined(AO_HAVE_fetch_compare_and_swap_release) \
-    && !defined(AO_HAVE_compare_and_swap_release)
-  AO_INLINE int
-  AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val,
-                                    AO_t new_val)
-  {
-    return AO_fetch_compare_and_swap_release(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_compare_and_swap_release
-#endif
-
-#if defined(AO_HAVE_fetch_compare_and_swap) \
-    && !defined(AO_HAVE_compare_and_swap)
-  AO_INLINE int
-  AO_compare_and_swap(volatile AO_t *addr, AO_t old_val,
-                            AO_t new_val)
-  {
-    return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
-  }
-# define AO_HAVE_compare_and_swap
-#endif
-
-/* fetch_and_add */
-/* We first try to implement fetch_and_add variants in terms of the     */
-/* corresponding compare_and_swap variants to minimize adding barriers. */
-#if defined(AO_HAVE_compare_and_swap_full) \
-    && !defined(AO_HAVE_fetch_and_add_full)
-  AO_INLINE AO_t
-  AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
-                                                           old + incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add_full
-#endif
-
-#if defined(AO_HAVE_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_fetch_and_add_acquire)
-  AO_INLINE AO_t
-  AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add_acquire
-#endif
-
-#if defined(AO_HAVE_compare_and_swap_release) \
-    && !defined(AO_HAVE_fetch_and_add_release)
-  AO_INLINE AO_t
-  AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add_release
-#endif
-
-#if defined(AO_HAVE_compare_and_swap) \
-    && !defined(AO_HAVE_fetch_and_add)
-  AO_INLINE AO_t
-  AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old,
-                                                      old + incr)));
-    return old;
-  }
-# define AO_HAVE_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_fetch_and_add_full)
-# if !defined(AO_HAVE_fetch_and_add_release)
-#   define AO_fetch_and_add_release(addr, val) \
-                                AO_fetch_and_add_full(addr, val)
-#   define AO_HAVE_fetch_and_add_release
-# endif
-# if !defined(AO_HAVE_fetch_and_add_acquire)
-#   define AO_fetch_and_add_acquire(addr, val) \
-                                AO_fetch_and_add_full(addr, val)
-#   define AO_HAVE_fetch_and_add_acquire
-# endif
-# if !defined(AO_HAVE_fetch_and_add_write)
-#   define AO_fetch_and_add_write(addr, val) \
-                                AO_fetch_and_add_full(addr, val)
-#   define AO_HAVE_fetch_and_add_write
-# endif
-# if !defined(AO_HAVE_fetch_and_add_read)
-#   define AO_fetch_and_add_read(addr, val) \
-                                AO_fetch_and_add_full(addr, val)
-#   define AO_HAVE_fetch_and_add_read
-# endif
-#endif /* AO_HAVE_fetch_and_add_full */
-
-#if !defined(AO_HAVE_fetch_and_add) \
-    && defined(AO_HAVE_fetch_and_add_release)
-# define AO_fetch_and_add(addr, val) \
-                                AO_fetch_and_add_release(addr, val)
-# define AO_HAVE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_fetch_and_add) \
-    && defined(AO_HAVE_fetch_and_add_acquire)
-# define AO_fetch_and_add(addr, val) \
-                                AO_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_fetch_and_add) \
-    && defined(AO_HAVE_fetch_and_add_write)
-# define AO_fetch_and_add(addr, val) \
-                                AO_fetch_and_add_write(addr, val)
-# define AO_HAVE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_fetch_and_add) \
-    && defined(AO_HAVE_fetch_and_add_read)
-# define AO_fetch_and_add(addr, val) \
-                                AO_fetch_and_add_read(addr, val)
-# define AO_HAVE_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_fetch_and_add_acquire) \
-    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full)
-# define AO_fetch_and_add_full(addr, val) \
-                (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
-# define AO_HAVE_fetch_and_add_full
-#endif
-
-#if !defined(AO_HAVE_fetch_and_add_release_write) \
-    && defined(AO_HAVE_fetch_and_add_write)
-# define AO_fetch_and_add_release_write(addr, val) \
-                                AO_fetch_and_add_write(addr, val)
-# define AO_HAVE_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add_release_write) \
-    && defined(AO_HAVE_fetch_and_add_release)
-# define AO_fetch_and_add_release_write(addr, val) \
-                                AO_fetch_and_add_release(addr, val)
-# define AO_HAVE_fetch_and_add_release_write
-#endif
-
-#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add_read)
-# define AO_fetch_and_add_acquire_read(addr, val) \
-                                AO_fetch_and_add_read(addr, val)
-# define AO_HAVE_fetch_and_add_acquire_read
-#endif
-#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add_acquire)
-# define AO_fetch_and_add_acquire_read(addr, val) \
-                                AO_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_fetch_and_add_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_fetch_and_add_acquire_read)
-#   define AO_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_fetch_and_add_acquire_read(addr, val)
-#   define AO_HAVE_fetch_and_add_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_fetch_and_add)
-#   define AO_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_fetch_and_add(addr, val)
-#   define AO_HAVE_fetch_and_add_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* fetch_and_add1 */
-#if defined(AO_HAVE_fetch_and_add_full) \
-    && !defined(AO_HAVE_fetch_and_add1_full)
-# define AO_fetch_and_add1_full(addr) \
-                                AO_fetch_and_add_full(addr,1)
-# define AO_HAVE_fetch_and_add1_full
-#endif
-#if defined(AO_HAVE_fetch_and_add_release) \
-    && !defined(AO_HAVE_fetch_and_add1_release)
-# define AO_fetch_and_add1_release(addr) \
-                                AO_fetch_and_add_release(addr,1)
-# define AO_HAVE_fetch_and_add1_release
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_fetch_and_add1_acquire)
-# define AO_fetch_and_add1_acquire(addr) \
-                                AO_fetch_and_add_acquire(addr,1)
-# define AO_HAVE_fetch_and_add1_acquire
-#endif
-#if defined(AO_HAVE_fetch_and_add_write) \
-    && !defined(AO_HAVE_fetch_and_add1_write)
-# define AO_fetch_and_add1_write(addr) \
-                                AO_fetch_and_add_write(addr,1)
-# define AO_HAVE_fetch_and_add1_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_read) \
-    && !defined(AO_HAVE_fetch_and_add1_read)
-# define AO_fetch_and_add1_read(addr) \
-                                AO_fetch_and_add_read(addr,1)
-# define AO_HAVE_fetch_and_add1_read
-#endif
-#if defined(AO_HAVE_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_fetch_and_add1_release_write)
-# define AO_fetch_and_add1_release_write(addr) \
-                                AO_fetch_and_add_release_write(addr,1)
-# define AO_HAVE_fetch_and_add1_release_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_fetch_and_add1_acquire_read)
-# define AO_fetch_and_add1_acquire_read(addr) \
-                                AO_fetch_and_add_acquire_read(addr,1)
-# define AO_HAVE_fetch_and_add1_acquire_read
-#endif
-#if defined(AO_HAVE_fetch_and_add) \
-    && !defined(AO_HAVE_fetch_and_add1)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add(addr,1)
-# define AO_HAVE_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_fetch_and_add1_full)
-# if !defined(AO_HAVE_fetch_and_add1_release)
-#   define AO_fetch_and_add1_release(addr) \
-                                AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_release
-# endif
-# if !defined(AO_HAVE_fetch_and_add1_acquire)
-#   define AO_fetch_and_add1_acquire(addr) \
-                                AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_acquire
-# endif
-# if !defined(AO_HAVE_fetch_and_add1_write)
-#   define AO_fetch_and_add1_write(addr) \
-                                AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_write
-# endif
-# if !defined(AO_HAVE_fetch_and_add1_read)
-#   define AO_fetch_and_add1_read(addr) \
-                                AO_fetch_and_add1_full(addr)
-#   define AO_HAVE_fetch_and_add1_read
-# endif
-#endif /* AO_HAVE_fetch_and_add1_full */
-
-#if !defined(AO_HAVE_fetch_and_add1) \
-    && defined(AO_HAVE_fetch_and_add1_release)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_fetch_and_add1) \
-    && defined(AO_HAVE_fetch_and_add1_acquire)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_fetch_and_add1) \
-    && defined(AO_HAVE_fetch_and_add1_write)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_fetch_and_add1) \
-    && defined(AO_HAVE_fetch_and_add1_read)
-# define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr)
-# define AO_HAVE_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_fetch_and_add1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_fetch_and_add1_full)
-# define AO_fetch_and_add1_full(addr) \
-                        (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
-# define AO_HAVE_fetch_and_add1_full
-#endif
-
-#if !defined(AO_HAVE_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_fetch_and_add1_write)
-# define AO_fetch_and_add1_release_write(addr) \
-                                AO_fetch_and_add1_write(addr)
-# define AO_HAVE_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_fetch_and_add1_release)
-# define AO_fetch_and_add1_release_write(addr) \
-                                AO_fetch_and_add1_release(addr)
-# define AO_HAVE_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add1_read)
-# define AO_fetch_and_add1_acquire_read(addr) \
-                                AO_fetch_and_add1_read(addr)
-# define AO_HAVE_fetch_and_add1_acquire_read
-#endif
-#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_add1_acquire)
-# define AO_fetch_and_add1_acquire_read(addr) \
-                                AO_fetch_and_add1_acquire(addr)
-# define AO_HAVE_fetch_and_add1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_fetch_and_add1_acquire_read)
-#   define AO_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_fetch_and_add1_acquire_read(addr)
-#   define AO_HAVE_fetch_and_add1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_fetch_and_add1)
-#   define AO_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_fetch_and_add1(addr)
-#   define AO_HAVE_fetch_and_add1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* fetch_and_sub1 */
-#if defined(AO_HAVE_fetch_and_add_full) \
-    && !defined(AO_HAVE_fetch_and_sub1_full)
-# define AO_fetch_and_sub1_full(addr) \
-                AO_fetch_and_add_full(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_full
-#endif
-#if defined(AO_HAVE_fetch_and_add_release) \
-    && !defined(AO_HAVE_fetch_and_sub1_release)
-# define AO_fetch_and_sub1_release(addr) \
-                AO_fetch_and_add_release(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_release
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_fetch_and_sub1_acquire)
-# define AO_fetch_and_sub1_acquire(addr) \
-                AO_fetch_and_add_acquire(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_acquire
-#endif
-#if defined(AO_HAVE_fetch_and_add_write) \
-    && !defined(AO_HAVE_fetch_and_sub1_write)
-# define AO_fetch_and_sub1_write(addr) \
-                AO_fetch_and_add_write(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_read) \
-    && !defined(AO_HAVE_fetch_and_sub1_read)
-# define AO_fetch_and_sub1_read(addr) \
-                AO_fetch_and_add_read(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_read
-#endif
-#if defined(AO_HAVE_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_fetch_and_sub1_release_write)
-# define AO_fetch_and_sub1_release_write(addr) \
-                AO_fetch_and_add_release_write(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_release_write
-#endif
-#if defined(AO_HAVE_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_fetch_and_sub1_acquire_read)
-# define AO_fetch_and_sub1_acquire_read(addr) \
-                AO_fetch_and_add_acquire_read(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1_acquire_read
-#endif
-#if defined(AO_HAVE_fetch_and_add) \
-    && !defined(AO_HAVE_fetch_and_sub1)
-# define AO_fetch_and_sub1(addr) \
-                AO_fetch_and_add(addr, (AO_t)(-1))
-# define AO_HAVE_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_fetch_and_sub1_full)
-# if !defined(AO_HAVE_fetch_and_sub1_release)
-#   define AO_fetch_and_sub1_release(addr) \
-                                AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_release
-# endif
-# if !defined(AO_HAVE_fetch_and_sub1_acquire)
-#   define AO_fetch_and_sub1_acquire(addr) \
-                                AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_acquire
-# endif
-# if !defined(AO_HAVE_fetch_and_sub1_write)
-#   define AO_fetch_and_sub1_write(addr) \
-                                AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_write
-# endif
-# if !defined(AO_HAVE_fetch_and_sub1_read)
-#   define AO_fetch_and_sub1_read(addr) \
-                                AO_fetch_and_sub1_full(addr)
-#   define AO_HAVE_fetch_and_sub1_read
-# endif
-#endif /* AO_HAVE_fetch_and_sub1_full */
-
-#if !defined(AO_HAVE_fetch_and_sub1) \
-    && defined(AO_HAVE_fetch_and_sub1_release)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1) \
-    && defined(AO_HAVE_fetch_and_sub1_acquire)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1) \
-    && defined(AO_HAVE_fetch_and_sub1_write)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1) \
-    && defined(AO_HAVE_fetch_and_sub1_read)
-# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr)
-# define AO_HAVE_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_fetch_and_sub1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_fetch_and_sub1_full)
-# define AO_fetch_and_sub1_full(addr) \
-                        (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
-# define AO_HAVE_fetch_and_sub1_full
-#endif
-
-#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_fetch_and_sub1_write)
-# define AO_fetch_and_sub1_release_write(addr) \
-                                AO_fetch_and_sub1_write(addr)
-# define AO_HAVE_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_fetch_and_sub1_release)
-# define AO_fetch_and_sub1_release_write(addr) \
-                                AO_fetch_and_sub1_release(addr)
-# define AO_HAVE_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_sub1_read)
-# define AO_fetch_and_sub1_acquire_read(addr) \
-                                AO_fetch_and_sub1_read(addr)
-# define AO_HAVE_fetch_and_sub1_acquire_read
-#endif
-#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_fetch_and_sub1_acquire)
-# define AO_fetch_and_sub1_acquire_read(addr) \
-                                AO_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_fetch_and_sub1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_fetch_and_sub1_acquire_read)
-#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_fetch_and_sub1_acquire_read(addr)
-#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_fetch_and_sub1)
-#   define AO_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_fetch_and_sub1(addr)
-#   define AO_HAVE_fetch_and_sub1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* and */
-#if defined(AO_HAVE_compare_and_swap_full) \
-    && !defined(AO_HAVE_and_full)
-  AO_INLINE void
-  AO_and_full(volatile AO_t *addr, AO_t value)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
-                                                           old & value)));
-  }
-# define AO_HAVE_and_full
-#endif
-
-#if defined(AO_HAVE_and_full)
-# if !defined(AO_HAVE_and_release)
-#   define AO_and_release(addr, val) AO_and_full(addr, val)
-#   define AO_HAVE_and_release
-# endif
-# if !defined(AO_HAVE_and_acquire)
-#   define AO_and_acquire(addr, val) AO_and_full(addr, val)
-#   define AO_HAVE_and_acquire
-# endif
-# if !defined(AO_HAVE_and_write)
-#   define AO_and_write(addr, val) AO_and_full(addr, val)
-#   define AO_HAVE_and_write
-# endif
-# if !defined(AO_HAVE_and_read)
-#   define AO_and_read(addr, val) AO_and_full(addr, val)
-#   define AO_HAVE_and_read
-# endif
-#endif /* AO_HAVE_and_full */
-
-#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_release)
-# define AO_and(addr, val) AO_and_release(addr, val)
-# define AO_HAVE_and
-#endif
-#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_acquire)
-# define AO_and(addr, val) AO_and_acquire(addr, val)
-# define AO_HAVE_and
-#endif
-#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_write)
-# define AO_and(addr, val) AO_and_write(addr, val)
-# define AO_HAVE_and
-#endif
-#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_read)
-# define AO_and(addr, val) AO_and_read(addr, val)
-# define AO_HAVE_and
-#endif
-
-#if defined(AO_HAVE_and_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_and_full)
-# define AO_and_full(addr, val) \
-                        (AO_nop_full(), AO_and_acquire(addr, val))
-# define AO_HAVE_and_full
-#endif
-
-#if !defined(AO_HAVE_and_release_write) \
-    && defined(AO_HAVE_and_write)
-# define AO_and_release_write(addr, val) AO_and_write(addr, val)
-# define AO_HAVE_and_release_write
-#endif
-#if !defined(AO_HAVE_and_release_write) \
-    && defined(AO_HAVE_and_release)
-# define AO_and_release_write(addr, val) AO_and_release(addr, val)
-# define AO_HAVE_and_release_write
-#endif
-#if !defined(AO_HAVE_and_acquire_read) \
-    && defined(AO_HAVE_and_read)
-# define AO_and_acquire_read(addr, val) AO_and_read(addr, val)
-# define AO_HAVE_and_acquire_read
-#endif
-#if !defined(AO_HAVE_and_acquire_read) \
-    && defined(AO_HAVE_and_acquire)
-# define AO_and_acquire_read(addr, val) AO_and_acquire(addr, val)
-# define AO_HAVE_and_acquire_read
-#endif
-
-/* or */
-#if defined(AO_HAVE_compare_and_swap_full) \
-    && !defined(AO_HAVE_or_full)
-  AO_INLINE void
-  AO_or_full(volatile AO_t *addr, AO_t value)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
-                                                           old | value)));
-  }
-# define AO_HAVE_or_full
-#endif
-
-#if defined(AO_HAVE_or_full)
-# if !defined(AO_HAVE_or_release)
-#   define AO_or_release(addr, val) AO_or_full(addr, val)
-#   define AO_HAVE_or_release
-# endif
-# if !defined(AO_HAVE_or_acquire)
-#   define AO_or_acquire(addr, val) AO_or_full(addr, val)
-#   define AO_HAVE_or_acquire
-# endif
-# if !defined(AO_HAVE_or_write)
-#   define AO_or_write(addr, val) AO_or_full(addr, val)
-#   define AO_HAVE_or_write
-# endif
-# if !defined(AO_HAVE_or_read)
-#   define AO_or_read(addr, val) AO_or_full(addr, val)
-#   define AO_HAVE_or_read
-# endif
-#endif /* AO_HAVE_or_full */
-
-#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_release)
-# define AO_or(addr, val) AO_or_release(addr, val)
-# define AO_HAVE_or
-#endif
-#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_acquire)
-# define AO_or(addr, val) AO_or_acquire(addr, val)
-# define AO_HAVE_or
-#endif
-#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_write)
-# define AO_or(addr, val) AO_or_write(addr, val)
-# define AO_HAVE_or
-#endif
-#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_read)
-# define AO_or(addr, val) AO_or_read(addr, val)
-# define AO_HAVE_or
-#endif
-
-#if defined(AO_HAVE_or_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_or_full)
-# define AO_or_full(addr, val) \
-                        (AO_nop_full(), AO_or_acquire(addr, val))
-# define AO_HAVE_or_full
-#endif
-
-#if !defined(AO_HAVE_or_release_write) \
-    && defined(AO_HAVE_or_write)
-# define AO_or_release_write(addr, val) AO_or_write(addr, val)
-# define AO_HAVE_or_release_write
-#endif
-#if !defined(AO_HAVE_or_release_write) \
-    && defined(AO_HAVE_or_release)
-# define AO_or_release_write(addr, val) AO_or_release(addr, val)
-# define AO_HAVE_or_release_write
-#endif
-#if !defined(AO_HAVE_or_acquire_read) && defined(AO_HAVE_or_read)
-# define AO_or_acquire_read(addr, val) AO_or_read(addr, val)
-# define AO_HAVE_or_acquire_read
-#endif
-#if !defined(AO_HAVE_or_acquire_read) \
-    && defined(AO_HAVE_or_acquire)
-# define AO_or_acquire_read(addr, val) AO_or_acquire(addr, val)
-# define AO_HAVE_or_acquire_read
-#endif
-
-/* xor */
-#if defined(AO_HAVE_compare_and_swap_full) \
-    && !defined(AO_HAVE_xor_full)
-  AO_INLINE void
-  AO_xor_full(volatile AO_t *addr, AO_t value)
-  {
-    AO_t old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
-                                                           old ^ value)));
-  }
-# define AO_HAVE_xor_full
-#endif
-
-#if defined(AO_HAVE_xor_full)
-# if !defined(AO_HAVE_xor_release)
-#   define AO_xor_release(addr, val) AO_xor_full(addr, val)
-#   define AO_HAVE_xor_release
-# endif
-# if !defined(AO_HAVE_xor_acquire)
-#   define AO_xor_acquire(addr, val) AO_xor_full(addr, val)
-#   define AO_HAVE_xor_acquire
-# endif
-# if !defined(AO_HAVE_xor_write)
-#   define AO_xor_write(addr, val) AO_xor_full(addr, val)
-#   define AO_HAVE_xor_write
-# endif
-# if !defined(AO_HAVE_xor_read)
-#   define AO_xor_read(addr, val) AO_xor_full(addr, val)
-#   define AO_HAVE_xor_read
-# endif
-#endif /* AO_HAVE_xor_full */
-
-#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_release)
-# define AO_xor(addr, val) AO_xor_release(addr, val)
-# define AO_HAVE_xor
-#endif
-#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_acquire)
-# define AO_xor(addr, val) AO_xor_acquire(addr, val)
-# define AO_HAVE_xor
-#endif
-#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_write)
-# define AO_xor(addr, val) AO_xor_write(addr, val)
-# define AO_HAVE_xor
-#endif
-#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_read)
-# define AO_xor(addr, val) AO_xor_read(addr, val)
-# define AO_HAVE_xor
-#endif
-
-#if defined(AO_HAVE_xor_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_xor_full)
-# define AO_xor_full(addr, val) \
-                        (AO_nop_full(), AO_xor_acquire(addr, val))
-# define AO_HAVE_xor_full
-#endif
-
-#if !defined(AO_HAVE_xor_release_write) \
-    && defined(AO_HAVE_xor_write)
-# define AO_xor_release_write(addr, val) AO_xor_write(addr, val)
-# define AO_HAVE_xor_release_write
-#endif
-#if !defined(AO_HAVE_xor_release_write) \
-    && defined(AO_HAVE_xor_release)
-# define AO_xor_release_write(addr, val) AO_xor_release(addr, val)
-# define AO_HAVE_xor_release_write
-#endif
-#if !defined(AO_HAVE_xor_acquire_read) \
-    && defined(AO_HAVE_xor_read)
-# define AO_xor_acquire_read(addr, val) AO_xor_read(addr, val)
-# define AO_HAVE_xor_acquire_read
-#endif
-#if !defined(AO_HAVE_xor_acquire_read) \
-    && defined(AO_HAVE_xor_acquire)
-# define AO_xor_acquire_read(addr, val) AO_xor_acquire(addr, val)
-# define AO_HAVE_xor_acquire_read
-#endif
-
-/* and/or/xor_dd_aquire_read are meaningless.     */

diff --git a/src/atomic_ops/generalize-small.template b/src/atomic_ops/generalize-small.template
index 27abab1..8ac9cfd 100644 (file)
--- a/src/atomic_ops/generalize-small.template
+++ b/src/atomic_ops/generalize-small.template
@@ -388,748 +388,3 @@
 #   define AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read
 # endif
 #endif /* !AO_NO_DD_ORDERING */
-
-#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_full) \
-    && !defined(AO_HAVE_XSIZE_compare_and_swap_full)
-  AO_INLINE int
-  AO_XSIZE_compare_and_swap_full(volatile XCTYPE *addr, XCTYPE old_val,
-                                 XCTYPE new_val)
-  {
-    return AO_XSIZE_fetch_compare_and_swap_full(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_XSIZE_compare_and_swap_full
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire)
-  AO_INLINE int
-  AO_XSIZE_compare_and_swap_acquire(volatile XCTYPE *addr, XCTYPE old_val,
-                                    XCTYPE new_val)
-  {
-    return AO_XSIZE_fetch_compare_and_swap_acquire(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_XSIZE_compare_and_swap_acquire
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release) \
-    && !defined(AO_HAVE_XSIZE_compare_and_swap_release)
-  AO_INLINE int
-  AO_XSIZE_compare_and_swap_release(volatile XCTYPE *addr, XCTYPE old_val,
-                                    XCTYPE new_val)
-  {
-    return AO_XSIZE_fetch_compare_and_swap_release(addr, old_val, new_val)
-             == old_val;
-  }
-# define AO_HAVE_XSIZE_compare_and_swap_release
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \
-    && !defined(AO_HAVE_XSIZE_compare_and_swap)
-  AO_INLINE int
-  AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, XCTYPE old_val,
-                            XCTYPE new_val)
-  {
-    return AO_XSIZE_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
-  }
-# define AO_HAVE_XSIZE_compare_and_swap
-#endif
-
-/* XSIZE_fetch_and_add */
-/* We first try to implement fetch_and_add variants in terms of the     */
-/* corresponding compare_and_swap variants to minimize adding barriers. */
-#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add_full)
-  AO_INLINE XCTYPE
-  AO_XSIZE_fetch_and_add_full(volatile XCTYPE *addr, XCTYPE incr)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
-                                                           old + incr)));
-    return old;
-  }
-# define AO_HAVE_XSIZE_fetch_and_add_full
-#endif
-
-#if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
-  AO_INLINE XCTYPE
-  AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_XSIZE_fetch_and_add_acquire
-#endif
-
-#if defined(AO_HAVE_XSIZE_compare_and_swap_release) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add_release)
-  AO_INLINE XCTYPE
-  AO_XSIZE_fetch_and_add_release(volatile XCTYPE *addr, XCTYPE incr)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_release(addr, old,
-                                                              old + incr)));
-    return old;
-  }
-# define AO_HAVE_XSIZE_fetch_and_add_release
-#endif
-
-#if defined(AO_HAVE_XSIZE_compare_and_swap) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add)
-  AO_INLINE XCTYPE
-  AO_XSIZE_fetch_and_add(volatile XCTYPE *addr, XCTYPE incr)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap(addr, old,
-                                                      old + incr)));
-    return old;
-  }
-# define AO_HAVE_XSIZE_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_and_add_full)
-# if !defined(AO_HAVE_XSIZE_fetch_and_add_release)
-#   define AO_XSIZE_fetch_and_add_release(addr, val) \
-                                AO_XSIZE_fetch_and_add_full(addr, val)
-#   define AO_HAVE_XSIZE_fetch_and_add_release
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
-#   define AO_XSIZE_fetch_and_add_acquire(addr, val) \
-                                AO_XSIZE_fetch_and_add_full(addr, val)
-#   define AO_HAVE_XSIZE_fetch_and_add_acquire
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_add_write)
-#   define AO_XSIZE_fetch_and_add_write(addr, val) \
-                                AO_XSIZE_fetch_and_add_full(addr, val)
-#   define AO_HAVE_XSIZE_fetch_and_add_write
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_add_read)
-#   define AO_XSIZE_fetch_and_add_read(addr, val) \
-                                AO_XSIZE_fetch_and_add_full(addr, val)
-#   define AO_HAVE_XSIZE_fetch_and_add_read
-# endif
-#endif /* AO_HAVE_XSIZE_fetch_and_add_full */
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_release)
-# define AO_XSIZE_fetch_and_add(addr, val) \
-                                AO_XSIZE_fetch_and_add_release(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
-# define AO_XSIZE_fetch_and_add(addr, val) \
-                                AO_XSIZE_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_write)
-# define AO_XSIZE_fetch_and_add(addr, val) \
-                                AO_XSIZE_fetch_and_add_write(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_read)
-# define AO_XSIZE_fetch_and_add(addr, val) \
-                                AO_XSIZE_fetch_and_add_read(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
-    && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_XSIZE_fetch_and_add_full)
-# define AO_XSIZE_fetch_and_add_full(addr, val) \
-                (AO_nop_full(), AO_XSIZE_fetch_and_add_acquire(addr, val))
-# define AO_HAVE_XSIZE_fetch_and_add_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_write)
-# define AO_XSIZE_fetch_and_add_release_write(addr, val) \
-                                AO_XSIZE_fetch_and_add_write(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_release)
-# define AO_XSIZE_fetch_and_add_release_write(addr, val) \
-                                AO_XSIZE_fetch_and_add_release(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add_release_write
-#endif
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_read)
-# define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \
-                                AO_XSIZE_fetch_and_add_read(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add_acquire_read
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
-# define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \
-                                AO_XSIZE_fetch_and_add_acquire(addr, val)
-# define AO_HAVE_XSIZE_fetch_and_add_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read)
-#   define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_XSIZE_fetch_and_add_acquire_read(addr, val)
-#   define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_XSIZE_fetch_and_add)
-#   define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \
-                                AO_XSIZE_fetch_and_add(addr, val)
-#   define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* XSIZE_fetch_and_add1 */
-#if defined(AO_HAVE_XSIZE_fetch_and_add_full) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_full)
-# define AO_XSIZE_fetch_and_add1_full(addr) \
-                                AO_XSIZE_fetch_and_add_full(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_full
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_release) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_release)
-# define AO_XSIZE_fetch_and_add1_release(addr) \
-                                AO_XSIZE_fetch_and_add_release(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_release
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
-# define AO_XSIZE_fetch_and_add1_acquire(addr) \
-                                AO_XSIZE_fetch_and_add_acquire(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_acquire
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_write) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_write)
-# define AO_XSIZE_fetch_and_add1_write(addr) \
-                                AO_XSIZE_fetch_and_add_write(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_write
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_read) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_read)
-# define AO_XSIZE_fetch_and_add1_read(addr) \
-                                AO_XSIZE_fetch_and_add_read(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_read
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write)
-# define AO_XSIZE_fetch_and_add1_release_write(addr) \
-                                AO_XSIZE_fetch_and_add_release_write(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_release_write
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read)
-# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_add_acquire_read(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1)
-# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add(addr,1)
-# define AO_HAVE_XSIZE_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_and_add1_full)
-# if !defined(AO_HAVE_XSIZE_fetch_and_add1_release)
-#   define AO_XSIZE_fetch_and_add1_release(addr) \
-                                AO_XSIZE_fetch_and_add1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_add1_release
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
-#   define AO_XSIZE_fetch_and_add1_acquire(addr) \
-                                AO_XSIZE_fetch_and_add1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_add1_acquire
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_add1_write)
-#   define AO_XSIZE_fetch_and_add1_write(addr) \
-                                AO_XSIZE_fetch_and_add1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_add1_write
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_add1_read)
-#   define AO_XSIZE_fetch_and_add1_read(addr) \
-                                AO_XSIZE_fetch_and_add1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_add1_read
-# endif
-#endif /* AO_HAVE_XSIZE_fetch_and_add1_full */
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_release)
-# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_release(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
-# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_acquire(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_write)
-# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_write(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_read)
-# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_read(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_add1_full)
-# define AO_XSIZE_fetch_and_add1_full(addr) \
-                        (AO_nop_full(), AO_XSIZE_fetch_and_add1_acquire(addr))
-# define AO_HAVE_XSIZE_fetch_and_add1_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_write)
-# define AO_XSIZE_fetch_and_add1_release_write(addr) \
-                                AO_XSIZE_fetch_and_add1_write(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_release)
-# define AO_XSIZE_fetch_and_add1_release_write(addr) \
-                                AO_XSIZE_fetch_and_add1_release(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_read)
-# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_add1_read(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \
-    && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
-# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_add1_acquire(addr)
-# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read)
-#   define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_add1_acquire_read(addr)
-#   define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_XSIZE_fetch_and_add1)
-#   define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_add1(addr)
-#   define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* XSIZE_fetch_and_sub1 */
-#if defined(AO_HAVE_XSIZE_fetch_and_add_full) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
-# define AO_XSIZE_fetch_and_sub1_full(addr) \
-                AO_XSIZE_fetch_and_add_full(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_full
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_release) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
-# define AO_XSIZE_fetch_and_sub1_release(addr) \
-                AO_XSIZE_fetch_and_add_release(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_release
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
-# define AO_XSIZE_fetch_and_sub1_acquire(addr) \
-                AO_XSIZE_fetch_and_add_acquire(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_acquire
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_write) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
-# define AO_XSIZE_fetch_and_sub1_write(addr) \
-                AO_XSIZE_fetch_and_add_write(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_write
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_read) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
-# define AO_XSIZE_fetch_and_sub1_read(addr) \
-                AO_XSIZE_fetch_and_add_read(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_read
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write)
-# define AO_XSIZE_fetch_and_sub1_release_write(addr) \
-                AO_XSIZE_fetch_and_add_release_write(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_release_write
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read)
-# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
-                AO_XSIZE_fetch_and_add_acquire_read(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
-#endif
-#if defined(AO_HAVE_XSIZE_fetch_and_add) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1)
-# define AO_XSIZE_fetch_and_sub1(addr) \
-                AO_XSIZE_fetch_and_add(addr, (XCTYPE)(-1))
-# define AO_HAVE_XSIZE_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
-# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
-#   define AO_XSIZE_fetch_and_sub1_release(addr) \
-                                AO_XSIZE_fetch_and_sub1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_sub1_release
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
-#   define AO_XSIZE_fetch_and_sub1_acquire(addr) \
-                                AO_XSIZE_fetch_and_sub1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_sub1_acquire
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
-#   define AO_XSIZE_fetch_and_sub1_write(addr) \
-                                AO_XSIZE_fetch_and_sub1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_sub1_write
-# endif
-# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
-#   define AO_XSIZE_fetch_and_sub1_read(addr) \
-                                AO_XSIZE_fetch_and_sub1_full(addr)
-#   define AO_HAVE_XSIZE_fetch_and_sub1_read
-# endif
-#endif /* AO_HAVE_XSIZE_fetch_and_sub1_full */
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
-# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_release(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
-# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
-# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_write(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
-# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_read(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1
-#endif
-
-#if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) \
-    && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
-# define AO_XSIZE_fetch_and_sub1_full(addr) \
-                        (AO_nop_full(), AO_XSIZE_fetch_and_sub1_acquire(addr))
-# define AO_HAVE_XSIZE_fetch_and_sub1_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
-# define AO_XSIZE_fetch_and_sub1_release_write(addr) \
-                                AO_XSIZE_fetch_and_sub1_write(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
-# define AO_XSIZE_fetch_and_sub1_release_write(addr) \
-                                AO_XSIZE_fetch_and_sub1_release(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
-# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_sub1_read(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
-#endif
-#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \
-    && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
-# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_sub1_acquire(addr)
-# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
-#endif
-
-#ifdef AO_NO_DD_ORDERING
-# if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read)
-#   define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_sub1_acquire_read(addr)
-#   define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read
-# endif
-#else
-# if defined(AO_HAVE_XSIZE_fetch_and_sub1)
-#   define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \
-                                AO_XSIZE_fetch_and_sub1(addr)
-#   define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read
-# endif
-#endif /* !AO_NO_DD_ORDERING */
-
-/* XSIZE_and */
-#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
-    && !defined(AO_HAVE_XSIZE_and_full)
-  AO_INLINE void
-  AO_XSIZE_and_full(volatile XCTYPE *addr, XCTYPE value)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
-                                                           old & value)));
-  }
-# define AO_HAVE_XSIZE_and_full
-#endif
-
-#if defined(AO_HAVE_XSIZE_and_full)
-# if !defined(AO_HAVE_XSIZE_and_release)
-#   define AO_XSIZE_and_release(addr, val) AO_XSIZE_and_full(addr, val)
-#   define AO_HAVE_XSIZE_and_release
-# endif
-# if !defined(AO_HAVE_XSIZE_and_acquire)
-#   define AO_XSIZE_and_acquire(addr, val) AO_XSIZE_and_full(addr, val)
-#   define AO_HAVE_XSIZE_and_acquire
-# endif
-# if !defined(AO_HAVE_XSIZE_and_write)
-#   define AO_XSIZE_and_write(addr, val) AO_XSIZE_and_full(addr, val)
-#   define AO_HAVE_XSIZE_and_write
-# endif
-# if !defined(AO_HAVE_XSIZE_and_read)
-#   define AO_XSIZE_and_read(addr, val) AO_XSIZE_and_full(addr, val)
-#   define AO_HAVE_XSIZE_and_read
-# endif
-#endif /* AO_HAVE_XSIZE_and_full */
-
-#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_release)
-# define AO_XSIZE_and(addr, val) AO_XSIZE_and_release(addr, val)
-# define AO_HAVE_XSIZE_and
-#endif
-#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_acquire)
-# define AO_XSIZE_and(addr, val) AO_XSIZE_and_acquire(addr, val)
-# define AO_HAVE_XSIZE_and
-#endif
-#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_write)
-# define AO_XSIZE_and(addr, val) AO_XSIZE_and_write(addr, val)
-# define AO_HAVE_XSIZE_and
-#endif
-#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_read)
-# define AO_XSIZE_and(addr, val) AO_XSIZE_and_read(addr, val)
-# define AO_HAVE_XSIZE_and
-#endif
-
-#if defined(AO_HAVE_XSIZE_and_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_and_full)
-# define AO_XSIZE_and_full(addr, val) \
-                        (AO_nop_full(), AO_XSIZE_and_acquire(addr, val))
-# define AO_HAVE_XSIZE_and_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_and_release_write) \
-    && defined(AO_HAVE_XSIZE_and_write)
-# define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_write(addr, val)
-# define AO_HAVE_XSIZE_and_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_and_release_write) \
-    && defined(AO_HAVE_XSIZE_and_release)
-# define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_release(addr, val)
-# define AO_HAVE_XSIZE_and_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_and_acquire_read) \
-    && defined(AO_HAVE_XSIZE_and_read)
-# define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_read(addr, val)
-# define AO_HAVE_XSIZE_and_acquire_read
-#endif
-#if !defined(AO_HAVE_XSIZE_and_acquire_read) \
-    && defined(AO_HAVE_XSIZE_and_acquire)
-# define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_acquire(addr, val)
-# define AO_HAVE_XSIZE_and_acquire_read
-#endif
-
-/* XSIZE_or */
-#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
-    && !defined(AO_HAVE_XSIZE_or_full)
-  AO_INLINE void
-  AO_XSIZE_or_full(volatile XCTYPE *addr, XCTYPE value)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
-                                                           old | value)));
-  }
-# define AO_HAVE_XSIZE_or_full
-#endif
-
-#if defined(AO_HAVE_XSIZE_or_full)
-# if !defined(AO_HAVE_XSIZE_or_release)
-#   define AO_XSIZE_or_release(addr, val) AO_XSIZE_or_full(addr, val)
-#   define AO_HAVE_XSIZE_or_release
-# endif
-# if !defined(AO_HAVE_XSIZE_or_acquire)
-#   define AO_XSIZE_or_acquire(addr, val) AO_XSIZE_or_full(addr, val)
-#   define AO_HAVE_XSIZE_or_acquire
-# endif
-# if !defined(AO_HAVE_XSIZE_or_write)
-#   define AO_XSIZE_or_write(addr, val) AO_XSIZE_or_full(addr, val)
-#   define AO_HAVE_XSIZE_or_write
-# endif
-# if !defined(AO_HAVE_XSIZE_or_read)
-#   define AO_XSIZE_or_read(addr, val) AO_XSIZE_or_full(addr, val)
-#   define AO_HAVE_XSIZE_or_read
-# endif
-#endif /* AO_HAVE_XSIZE_or_full */
-
-#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_release)
-# define AO_XSIZE_or(addr, val) AO_XSIZE_or_release(addr, val)
-# define AO_HAVE_XSIZE_or
-#endif
-#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_acquire)
-# define AO_XSIZE_or(addr, val) AO_XSIZE_or_acquire(addr, val)
-# define AO_HAVE_XSIZE_or
-#endif
-#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_write)
-# define AO_XSIZE_or(addr, val) AO_XSIZE_or_write(addr, val)
-# define AO_HAVE_XSIZE_or
-#endif
-#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_read)
-# define AO_XSIZE_or(addr, val) AO_XSIZE_or_read(addr, val)
-# define AO_HAVE_XSIZE_or
-#endif
-
-#if defined(AO_HAVE_XSIZE_or_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_or_full)
-# define AO_XSIZE_or_full(addr, val) \
-                        (AO_nop_full(), AO_XSIZE_or_acquire(addr, val))
-# define AO_HAVE_XSIZE_or_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_or_release_write) \
-    && defined(AO_HAVE_XSIZE_or_write)
-# define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_write(addr, val)
-# define AO_HAVE_XSIZE_or_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_or_release_write) \
-    && defined(AO_HAVE_XSIZE_or_release)
-# define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_release(addr, val)
-# define AO_HAVE_XSIZE_or_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_or_acquire_read) && defined(AO_HAVE_XSIZE_or_read)
-# define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_read(addr, val)
-# define AO_HAVE_XSIZE_or_acquire_read
-#endif
-#if !defined(AO_HAVE_XSIZE_or_acquire_read) \
-    && defined(AO_HAVE_XSIZE_or_acquire)
-# define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_acquire(addr, val)
-# define AO_HAVE_XSIZE_or_acquire_read
-#endif
-
-/* XSIZE_xor */
-#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
-    && !defined(AO_HAVE_XSIZE_xor_full)
-  AO_INLINE void
-  AO_XSIZE_xor_full(volatile XCTYPE *addr, XCTYPE value)
-  {
-    XCTYPE old;
-
-    do
-      {
-        old = *addr;
-      }
-    while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
-                                                           old ^ value)));
-  }
-# define AO_HAVE_XSIZE_xor_full
-#endif
-
-#if defined(AO_HAVE_XSIZE_xor_full)
-# if !defined(AO_HAVE_XSIZE_xor_release)
-#   define AO_XSIZE_xor_release(addr, val) AO_XSIZE_xor_full(addr, val)
-#   define AO_HAVE_XSIZE_xor_release
-# endif
-# if !defined(AO_HAVE_XSIZE_xor_acquire)
-#   define AO_XSIZE_xor_acquire(addr, val) AO_XSIZE_xor_full(addr, val)
-#   define AO_HAVE_XSIZE_xor_acquire
-# endif
-# if !defined(AO_HAVE_XSIZE_xor_write)
-#   define AO_XSIZE_xor_write(addr, val) AO_XSIZE_xor_full(addr, val)
-#   define AO_HAVE_XSIZE_xor_write
-# endif
-# if !defined(AO_HAVE_XSIZE_xor_read)
-#   define AO_XSIZE_xor_read(addr, val) AO_XSIZE_xor_full(addr, val)
-#   define AO_HAVE_XSIZE_xor_read
-# endif
-#endif /* AO_HAVE_XSIZE_xor_full */
-
-#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_release)
-# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_release(addr, val)
-# define AO_HAVE_XSIZE_xor
-#endif
-#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_acquire)
-# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_acquire(addr, val)
-# define AO_HAVE_XSIZE_xor
-#endif
-#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_write)
-# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_write(addr, val)
-# define AO_HAVE_XSIZE_xor
-#endif
-#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_read)
-# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_read(addr, val)
-# define AO_HAVE_XSIZE_xor
-#endif
-
-#if defined(AO_HAVE_XSIZE_xor_acquire) && defined(AO_HAVE_nop_full) \
-    && !defined(AO_HAVE_XSIZE_xor_full)
-# define AO_XSIZE_xor_full(addr, val) \
-                        (AO_nop_full(), AO_XSIZE_xor_acquire(addr, val))
-# define AO_HAVE_XSIZE_xor_full
-#endif
-
-#if !defined(AO_HAVE_XSIZE_xor_release_write) \
-    && defined(AO_HAVE_XSIZE_xor_write)
-# define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_write(addr, val)
-# define AO_HAVE_XSIZE_xor_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_xor_release_write) \
-    && defined(AO_HAVE_XSIZE_xor_release)
-# define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_release(addr, val)
-# define AO_HAVE_XSIZE_xor_release_write
-#endif
-#if !defined(AO_HAVE_XSIZE_xor_acquire_read) \
-    && defined(AO_HAVE_XSIZE_xor_read)
-# define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_read(addr, val)
-# define AO_HAVE_XSIZE_xor_acquire_read
-#endif
-#if !defined(AO_HAVE_XSIZE_xor_acquire_read) \
-    && defined(AO_HAVE_XSIZE_xor_acquire)
-# define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_acquire(addr, val)
-# define AO_HAVE_XSIZE_xor_acquire_read
-#endif
-
-/* XSIZE_and/or/xor_dd_aquire_read are meaningless.     */

diff --git a/src/atomic_ops/generalize.h b/src/atomic_ops/generalize.h
index a496028..7755575 100644 (file)
--- a/src/atomic_ops/generalize.h
+++ b/src/atomic_ops/generalize.h
@@ -298,6 +298,8 @@
 
 #include "generalize-small.h"
 
+#include "generalize-arithm.h"
+
 /* Compare_double_and_swap_double based on double_compare_and_swap.     */
 #ifdef AO_HAVE_DOUBLE_PTR_STORAGE
 # if defined(AO_HAVE_double_compare_and_swap) \