glib/gatomic.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * g_atomic_*: atomic operations.
   5  * Copyright (C) 2003 Sebastian Wilhelmi
   6  * Copyright (C) 2007 Nokia Corporation
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the
  20  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  21  * Boston, MA 02111-1307, USA.
  22  */
  23
  24 #include "config.h"
  25
  26 #if defined (G_ATOMIC_ARM)
  27 #include <sched.h>
  28 #endif
  29
  30 #include "gatomic.h"
  31 #include "gthreadprivate.h"
  32
  33 /**
  34  * SECTION:atomic_operations
  35  * @title: Atomic Operations
  36  * @short_description: basic atomic integer and pointer operations
  37  * @see_also: #GMutex
  38  *
  39  * The following functions can be used to atomically access integers and
  40  * pointers. They are implemented as inline assembler function on most
  41  * platforms and use slower fall-backs otherwise. Using them can sometimes
  42  * save you from using a performance-expensive #GMutex to protect the
  43  * integer or pointer.
  44  *
  45  * The most important usage is reference counting. Using
  46  * g_atomic_int_inc() and g_atomic_int_dec_and_test() makes reference
  47  * counting a very fast operation.
  48  *
  49  * <note><para>You must not directly read integers or pointers concurrently
  50  * accessed by multiple threads, but use the atomic accessor functions
  51  * instead. That is, always use g_atomic_int_get() and g_atomic_pointer_get()
  52  * for read outs. They provide the neccessary synchonization mechanisms
  53  * like memory barriers to access memory locations concurrently.
  54  * </para></note>
  55  *
  56  * <note><para>If you are using those functions for anything apart from
  57  * simple reference counting, you should really be aware of the implications
  58  * of doing that. There are literally thousands of ways to shoot yourself
  59  * in the foot. So if in doubt, use a #GMutex. If you don't know, what
  60  * memory barriers are, do not use anything but g_atomic_int_inc() and
  61  * g_atomic_int_dec_and_test().
  62  * </para></note>
  63  *
  64  * <note><para>It is not safe to set an integer or pointer just by assigning
  65  * to it, when it is concurrently accessed by other threads with the following
  66  * functions. Use g_atomic_int_compare_and_exchange() or
  67  * g_atomic_pointer_compare_and_exchange() respectively.
  68  * </para></note>
  69  */
  70
  71 #if defined (__GNUC__)
  72 # if defined (G_ATOMIC_I486)
  73 /* Adapted from CVS version 1.10 of glibc's sysdeps/i386/i486/bits/atomic.h
  74  */
  75 gint
  76 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
  77                                gint           val)
  78 {
  79   gint result;
  80
  81   __asm__ __volatile__ ("lock; xaddl %0,%1"
  82                         : "=r" (result), "=m" (*atomic)
  83                         : "0" (val), "m" (*atomic));
  84   return result;
  85 }
  86
  87 void
  88 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
  89                   gint           val)
  90 {
  91   __asm__ __volatile__ ("lock; addl %1,%0"
  92                         : "=m" (*atomic)
  93                         : "ir" (val), "m" (*atomic));
  94 }
  95
  96 gboolean
  97 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
  98                                    gint           oldval,
  99                                    gint           newval)
 100 {
 101   gint result;
 102
 103   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 104                         : "=a" (result), "=m" (*atomic)
 105                         : "r" (newval), "m" (*atomic), "0" (oldval));
 106
 107   return result == oldval;
 108 }
 109
 110 /* The same code as above, as on i386 gpointer is 32 bit as well.
 111  * Duplicating the code here seems more natural than casting the
 112  * arguments and calling the former function */
 113
 114 gboolean
 115 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 116                                        gpointer           oldval,
 117                                        gpointer           newval)
 118 {
 119   gpointer result;
 120
 121   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 122                         : "=a" (result), "=m" (*atomic)
 123                         : "r" (newval), "m" (*atomic), "0" (oldval));
 124
 125   return result == oldval;
 126 }
 127
 128 # elif defined (G_ATOMIC_SPARCV9)
 129 /* Adapted from CVS version 1.3 of glibc's sysdeps/sparc/sparc64/bits/atomic.h
 130  */
 131 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 132   ({                                                                    \
 133      gint __result;                                                     \
 134      __asm__ __volatile__ ("cas [%4], %2, %0"                           \
 135                            : "=r" (__result), "=m" (*(atomic))          \
 136                            : "r" (oldval), "m" (*(atomic)), "r" (atomic),\
 137                            "0" (newval));                               \
 138      __result == oldval;                                                \
 139   })
 140
 141 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 142 gboolean
 143 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 144                                        gpointer           oldval,
 145                                        gpointer           newval)
 146 {
 147   gpointer result;
 148   __asm__ __volatile__ ("cas [%4], %2, %0"
 149                         : "=r" (result), "=m" (*atomic)
 150                         : "r" (oldval), "m" (*atomic), "r" (atomic),
 151                         "0" (newval));
 152   return result == oldval;
 153 }
 154 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 155 gboolean
 156 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 157                                        gpointer           oldval,
 158                                        gpointer           newval)
 159 {
 160   gpointer result;
 161   gpointer *a = atomic;
 162   __asm__ __volatile__ ("casx [%4], %2, %0"
 163                         : "=r" (result), "=m" (*a)
 164                         : "r" (oldval), "m" (*a), "r" (a),
 165                         "0" (newval));
 166   return result == oldval;
 167 }
 168 #  else /* What's that */
 169 #    error "Your system has an unsupported pointer size"
 170 #  endif /* GLIB_SIZEOF_VOID_P */
 171 #  define G_ATOMIC_MEMORY_BARRIER                                       \
 172   __asm__ __volatile__ ("membar #LoadLoad | #LoadStore"                 \
 173                         " | #StoreLoad | #StoreStore" : : : "memory")
 174
 175 # elif defined (G_ATOMIC_ALPHA)
 176 /* Adapted from CVS version 1.3 of glibc's sysdeps/alpha/bits/atomic.h
 177  */
 178 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 179   ({                                                                    \
 180      gint __result;                                                     \
 181      gint __prev;                                                       \
 182      __asm__ __volatile__ (                                             \
 183         "       mb\n"                                                   \
 184         "1:     ldl_l   %0,%2\n"                                        \
 185         "       cmpeq   %0,%3,%1\n"                                     \
 186         "       beq     %1,2f\n"                                        \
 187         "       mov     %4,%1\n"                                        \
 188         "       stl_c   %1,%2\n"                                        \
 189         "       beq     %1,1b\n"                                        \
 190         "       mb\n"                                                   \
 191         "2:"                                                            \
 192         : "=&r" (__prev),                                               \
 193           "=&r" (__result)                                              \
 194         : "m" (*(atomic)),                                              \
 195           "Ir" (oldval),                                                \
 196           "Ir" (newval)                                                 \
 197         : "memory");                                                    \
 198      __result != 0;                                                     \
 199   })
 200 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 201 gboolean
 202 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 203                                        gpointer           oldval,
 204                                        gpointer           newval)
 205 {
 206   gint result;
 207   gpointer prev;
 208   __asm__ __volatile__ (
 209         "       mb\n"
 210         "1:     ldl_l   %0,%2\n"
 211         "       cmpeq   %0,%3,%1\n"
 212         "       beq     %1,2f\n"
 213         "       mov     %4,%1\n"
 214         "       stl_c   %1,%2\n"
 215         "       beq     %1,1b\n"
 216         "       mb\n"
 217         "2:"
 218         : "=&r" (prev),
 219           "=&r" (result)
 220         : "m" (*atomic),
 221           "Ir" (oldval),
 222           "Ir" (newval)
 223         : "memory");
 224   return result != 0;
 225 }
 226 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 227 gboolean
 228 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 229                                        gpointer           oldval,
 230                                        gpointer           newval)
 231 {
 232   gint result;
 233   gpointer prev;
 234   __asm__ __volatile__ (
 235         "       mb\n"
 236         "1:     ldq_l   %0,%2\n"
 237         "       cmpeq   %0,%3,%1\n"
 238         "       beq     %1,2f\n"
 239         "       mov     %4,%1\n"
 240         "       stq_c   %1,%2\n"
 241         "       beq     %1,1b\n"
 242         "       mb\n"
 243         "2:"
 244         : "=&r" (prev),
 245           "=&r" (result)
 246         : "m" (*atomic),
 247           "Ir" (oldval),
 248           "Ir" (newval)
 249         : "memory");
 250   return result != 0;
 251 }
 252 #  else /* What's that */
 253 #   error "Your system has an unsupported pointer size"
 254 #  endif /* GLIB_SIZEOF_VOID_P */
 255 #  define G_ATOMIC_MEMORY_BARRIER  __asm__ ("mb" : : : "memory")
 256 # elif defined (G_ATOMIC_X86_64)
 257 /* Adapted from CVS version 1.9 of glibc's sysdeps/x86_64/bits/atomic.h
 258  */
 259 gint
 260 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 261                                gint           val)
 262 {
 263   gint result;
 264
 265   __asm__ __volatile__ ("lock; xaddl %0,%1"
 266                         : "=r" (result), "=m" (*atomic)
 267                         : "0" (val), "m" (*atomic));
 268   return result;
 269 }
 270
 271 void
 272 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 273                   gint           val)
 274 {
 275   __asm__ __volatile__ ("lock; addl %1,%0"
 276                         : "=m" (*atomic)
 277                         : "ir" (val), "m" (*atomic));
 278 }
 279
 280 gboolean
 281 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 282                                    gint           oldval,
 283                                    gint           newval)
 284 {
 285   gint result;
 286
 287   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 288                         : "=a" (result), "=m" (*atomic)
 289                         : "r" (newval), "m" (*atomic), "0" (oldval));
 290
 291   return result == oldval;
 292 }
 293
 294 gboolean
 295 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 296                                        gpointer           oldval,
 297                                        gpointer           newval)
 298 {
 299   gpointer result;
 300
 301   __asm__ __volatile__ ("lock; cmpxchgq %q2, %1"
 302                         : "=a" (result), "=m" (*atomic)
 303                         : "r" (newval), "m" (*atomic), "0" (oldval));
 304
 305   return result == oldval;
 306 }
 307
 308 # elif defined (G_ATOMIC_POWERPC)
 309 /* Adapted from CVS version 1.16 of glibc's sysdeps/powerpc/bits/atomic.h
 310  * and CVS version 1.4 of glibc's sysdeps/powerpc/powerpc32/bits/atomic.h
 311  * and CVS version 1.7 of glibc's sysdeps/powerpc/powerpc64/bits/atomic.h
 312  */
 313 #   ifdef __OPTIMIZE__
 314 /* Non-optimizing compile bails on the following two asm statements
 315  * for reasons unknown to the author */
 316 gint
 317 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 318                                gint           val)
 319 {
 320   gint result, temp;
 321 #if ASM_NUMERIC_LABELS
 322   __asm__ __volatile__ ("1:       lwarx   %0,0,%3\n"
 323                         "         add     %1,%0,%4\n"
 324                         "         stwcx.  %1,0,%3\n"
 325                         "         bne-    1b"
 326                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 327                         : "b" (atomic), "r" (val), "m" (*atomic)
 328                         : "cr0", "memory");
 329 #else
 330   __asm__ __volatile__ (".Lieaa%=:       lwarx   %0,0,%3\n"
 331                         "         add     %1,%0,%4\n"
 332                         "         stwcx.  %1,0,%3\n"
 333                         "         bne-    .Lieaa%="
 334                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 335                         : "b" (atomic), "r" (val), "m" (*atomic)
 336                         : "cr0", "memory");
 337 #endif
 338   return result;
 339 }
 340
 341 /* The same as above, to save a function call repeated here */
 342 void
 343 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 344                   gint           val)
 345 {
 346   gint result, temp;
 347 #if ASM_NUMERIC_LABELS
 348   __asm__ __volatile__ ("1:       lwarx   %0,0,%3\n"
 349                         "         add     %1,%0,%4\n"
 350                         "         stwcx.  %1,0,%3\n"
 351                         "         bne-    1b"
 352                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 353                         : "b" (atomic), "r" (val), "m" (*atomic)
 354                         : "cr0", "memory");
 355 #else
 356   __asm__ __volatile__ (".Lia%=:       lwarx   %0,0,%3\n"
 357                         "         add     %1,%0,%4\n"
 358                         "         stwcx.  %1,0,%3\n"
 359                         "         bne-    .Lia%="
 360                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 361                         : "b" (atomic), "r" (val), "m" (*atomic)
 362                         : "cr0", "memory");
 363 #endif
 364 }
 365 #   else /* !__OPTIMIZE__ */
 366 gint
 367 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 368                                gint           val)
 369 {
 370   gint result;
 371   do
 372     result = *atomic;
 373   while (!g_atomic_int_compare_and_exchange (atomic, result, result + val));
 374
 375   return result;
 376 }
 377
 378 void
 379 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 380                   gint           val)
 381 {
 382   gint result;
 383   do
 384     result = *atomic;
 385   while (!g_atomic_int_compare_and_exchange (atomic, result, result + val));
 386 }
 387 #   endif /* !__OPTIMIZE__ */
 388
 389 #   if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 390 gboolean
 391 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 392                                    gint           oldval,
 393                                    gint           newval)
 394 {
 395   gint result;
 396 #if ASM_NUMERIC_LABELS
 397   __asm__ __volatile__ ("sync\n"
 398                         "1: lwarx   %0,0,%1\n"
 399                         "   subf.   %0,%2,%0\n"
 400                         "   bne     2f\n"
 401                         "   stwcx.  %3,0,%1\n"
 402                         "   bne-    1b\n"
 403                         "2: isync"
 404                         : "=&r" (result)
 405                         : "b" (atomic), "r" (oldval), "r" (newval)
 406                         : "cr0", "memory");
 407 #else
 408   __asm__ __volatile__ ("sync\n"
 409                         ".L1icae%=: lwarx   %0,0,%1\n"
 410                         "   subf.   %0,%2,%0\n"
 411                         "   bne     .L2icae%=\n"
 412                         "   stwcx.  %3,0,%1\n"
 413                         "   bne-    .L1icae%=\n"
 414                         ".L2icae%=: isync"
 415                         : "=&r" (result)
 416                         : "b" (atomic), "r" (oldval), "r" (newval)
 417                         : "cr0", "memory");
 418 #endif
 419   return result == 0;
 420 }
 421
 422 gboolean
 423 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 424                                        gpointer           oldval,
 425                                        gpointer           newval)
 426 {
 427   gpointer result;
 428 #if ASM_NUMERIC_LABELS
 429   __asm__ __volatile__ ("sync\n"
 430                         "1: lwarx   %0,0,%1\n"
 431                         "   subf.   %0,%2,%0\n"
 432                         "   bne     2f\n"
 433                         "   stwcx.  %3,0,%1\n"
 434                         "   bne-    1b\n"
 435                         "2: isync"
 436                         : "=&r" (result)
 437                         : "b" (atomic), "r" (oldval), "r" (newval)
 438                         : "cr0", "memory");
 439 #else
 440   __asm__ __volatile__ ("sync\n"
 441                         ".L1pcae%=: lwarx   %0,0,%1\n"
 442                         "   subf.   %0,%2,%0\n"
 443                         "   bne     .L2pcae%=\n"
 444                         "   stwcx.  %3,0,%1\n"
 445                         "   bne-    .L1pcae%=\n"
 446                         ".L2pcae%=: isync"
 447                         : "=&r" (result)
 448                         : "b" (atomic), "r" (oldval), "r" (newval)
 449                         : "cr0", "memory");
 450 #endif
 451   return result == 0;
 452 }
 453 #   elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 454 gboolean
 455 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 456                                    gint           oldval,
 457                                    gint           newval)
 458 {
 459   gpointer result;
 460 #if ASM_NUMERIC_LABELS
 461   __asm__ __volatile__ ("sync\n"
 462                         "1: lwarx   %0,0,%1\n"
 463                         "   extsw   %0,%0\n"
 464                         "   subf.   %0,%2,%0\n"
 465                         "   bne     2f\n"
 466                         "   stwcx.  %3,0,%1\n"
 467                         "   bne-    1b\n"
 468                         "2: isync"
 469                         : "=&r" (result)
 470                         : "b" (atomic), "r" (oldval), "r" (newval)
 471                         : "cr0", "memory");
 472 #else
 473   __asm__ __volatile__ ("sync\n"
 474                         ".L1icae%=: lwarx   %0,0,%1\n"
 475                         "   extsw   %0,%0\n"
 476                         "   subf.   %0,%2,%0\n"
 477                         "   bne     .L2icae%=\n"
 478                         "   stwcx.  %3,0,%1\n"
 479                         "   bne-    .L1icae%=\n"
 480                         ".L2icae%=: isync"
 481                         : "=&r" (result)
 482                         : "b" (atomic), "r" (oldval), "r" (newval)
 483                         : "cr0", "memory");
 484 #endif
 485   return result == 0;
 486 }
 487
 488 gboolean
 489 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 490                                        gpointer           oldval,
 491                                        gpointer           newval)
 492 {
 493   gpointer result;
 494 #if ASM_NUMERIC_LABELS
 495   __asm__ __volatile__ ("sync\n"
 496                         "1: ldarx   %0,0,%1\n"
 497                         "   subf.   %0,%2,%0\n"
 498                         "   bne     2f\n"
 499                         "   stdcx.  %3,0,%1\n"
 500                         "   bne-    1b\n"
 501                         "2: isync"
 502                         : "=&r" (result)
 503                         : "b" (atomic), "r" (oldval), "r" (newval)
 504                         : "cr0", "memory");
 505 #else
 506   __asm__ __volatile__ ("sync\n"
 507                         ".L1pcae%=: ldarx   %0,0,%1\n"
 508                         "   subf.   %0,%2,%0\n"
 509                         "   bne     .L2pcae%=\n"
 510                         "   stdcx.  %3,0,%1\n"
 511                         "   bne-    .L1pcae%=\n"
 512                         ".L2pcae%=: isync"
 513                         : "=&r" (result)
 514                         : "b" (atomic), "r" (oldval), "r" (newval)
 515                         : "cr0", "memory");
 516 #endif
 517   return result == 0;
 518 }
 519 #  else /* What's that */
 520 #   error "Your system has an unsupported pointer size"
 521 #  endif /* GLIB_SIZEOF_VOID_P */
 522
 523 #  define G_ATOMIC_MEMORY_BARRIER __asm__ ("sync" : : : "memory")
 524
 525 # elif defined (G_ATOMIC_IA64)
 526 /* Adapted from CVS version 1.8 of glibc's sysdeps/ia64/bits/atomic.h
 527  */
 528 gint
 529 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 530                                gint           val)
 531 {
 532   return __sync_fetch_and_add (atomic, val);
 533 }
 534
 535 void
 536 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 537                   gint val)
 538 {
 539   __sync_fetch_and_add (atomic, val);
 540 }
 541
 542 gboolean
 543 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 544                                    gint           oldval,
 545                                    gint           newval)
 546 {
 547   return __sync_bool_compare_and_swap (atomic, oldval, newval);
 548 }
 549
 550 gboolean
 551 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 552                                        gpointer           oldval,
 553                                        gpointer           newval)
 554 {
 555   return __sync_bool_compare_and_swap ((long *)atomic,
 556                                        (long)oldval, (long)newval);
 557 }
 558
 559 #  define G_ATOMIC_MEMORY_BARRIER __sync_synchronize ()
 560 # elif defined (G_ATOMIC_S390)
 561 /* Adapted from glibc's sysdeps/s390/bits/atomic.h
 562  */
 563 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 564   ({                                                                    \
 565      gint __result = oldval;                                    \
 566      __asm__ __volatile__ ("cs %0, %2, %1"                              \
 567                            : "+d" (__result), "=Q" (*(atomic))          \
 568                            : "d" (newval), "m" (*(atomic)) : "cc" );    \
 569      __result == oldval;                                                \
 570   })
 571
 572 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 573 gboolean
 574 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 575                                        gpointer           oldval,
 576                                        gpointer           newval)
 577 {
 578   gpointer result = oldval;
 579   __asm__ __volatile__ ("cs %0, %2, %1"
 580                         : "+d" (result), "=Q" (*(atomic))
 581                         : "d" (newval), "m" (*(atomic)) : "cc" );
 582   return result == oldval;
 583 }
 584 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 585 gboolean
 586 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 587                                        gpointer           oldval,
 588                                        gpointer           newval)
 589 {
 590   gpointer result = oldval;
 591   gpointer *a = atomic;
 592   __asm__ __volatile__ ("csg %0, %2, %1"
 593                         : "+d" (result), "=Q" (*a)
 594                         : "d" ((long)(newval)), "m" (*a) : "cc" );
 595   return result == oldval;
 596 }
 597 #  else /* What's that */
 598 #    error "Your system has an unsupported pointer size"
 599 #  endif /* GLIB_SIZEOF_VOID_P */
 600 # elif defined (G_ATOMIC_ARM)
 601 static volatile int atomic_spin = 0;
 602
 603 static int atomic_spin_trylock (void)
 604 {
 605   int result;
 606
 607   asm volatile (
 608     "swp %0, %1, [%2]\n"
 609     : "=&r,&r" (result)
 610     : "r,0" (1), "r,r" (&atomic_spin)
 611     : "memory");
 612   if (result == 0)
 613     return 0;
 614   else
 615     return -1;
 616 }
 617
 618 static void atomic_spin_lock (void)
 619 {
 620   while (atomic_spin_trylock())
 621     sched_yield();
 622 }
 623
 624 static void atomic_spin_unlock (void)
 625 {
 626   atomic_spin = 0;
 627 }
 628
 629 gint
 630 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 631                                gint           val)
 632 {
 633   gint result;
 634
 635   atomic_spin_lock();
 636   result = *atomic;
 637   *atomic += val;
 638   atomic_spin_unlock();
 639
 640   return result;
 641 }
 642
 643 void
 644 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 645                   gint           val)
 646 {
 647   atomic_spin_lock();
 648   *atomic += val;
 649   atomic_spin_unlock();
 650 }
 651
 652 gboolean
 653 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 654                                    gint           oldval,
 655                                    gint           newval)
 656 {
 657   gboolean result;
 658
 659   atomic_spin_lock();
 660   if (*atomic == oldval)
 661     {
 662       result = TRUE;
 663       *atomic = newval;
 664     }
 665   else
 666     result = FALSE;
 667   atomic_spin_unlock();
 668
 669   return result;
 670 }
 671
 672 gboolean
 673 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 674                                        gpointer           oldval,
 675                                        gpointer           newval)
 676 {
 677   gboolean result;
 678
 679   atomic_spin_lock();
 680   if (*atomic == oldval)
 681     {
 682       result = TRUE;
 683       *atomic = newval;
 684     }
 685   else
 686     result = FALSE;
 687   atomic_spin_unlock();
 688
 689   return result;
 690 }
 691 # elif defined (G_ATOMIC_CRIS) || defined (G_ATOMIC_CRISV32)
 692 #  ifdef G_ATOMIC_CRIS
 693 #   define CRIS_ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)             \
 694   ({                                                                    \
 695      gboolean __result;                                                 \
 696      __asm__ __volatile__ ("\n"                                         \
 697                            "0:\tclearf\n\t"                             \
 698                            "cmp.d [%[Atomic]], %[OldVal]\n\t"           \
 699                            "bne 1f\n\t"                                 \
 700                            "ax\n\t"                                     \
 701                            "move.d %[NewVal], [%[Atomic]]\n\t"          \
 702                            "bwf 0b\n"                                   \
 703                            "1:\tseq %[Result]"                          \
 704                            : [Result] "=&r" (__result),                 \
 705                                       "=m" (*(atomic))                  \
 706                            : [Atomic] "r" (atomic),                     \
 707                              [OldVal] "r" (oldval),                     \
 708                              [NewVal] "r" (newval),                     \
 709                                       "g" (*(gpointer*) (atomic))       \
 710                            : "memory");                                 \
 711      __result;                                                          \
 712   })
 713 #  else
 714 #   define CRIS_ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)             \
 715   ({                                                                    \
 716      gboolean __result;                                                 \
 717      __asm__ __volatile__ ("\n"                                         \
 718                            "0:\tclearf p\n\t"                           \
 719                            "cmp.d [%[Atomic]], %[OldVal]\n\t"           \
 720                            "bne 1f\n\t"                                 \
 721                            "ax\n\t"                                     \
 722                            "move.d %[NewVal], [%[Atomic]]\n\t"          \
 723                            "bcs 0b\n"                                   \
 724                            "1:\tseq %[Result]"                          \
 725                            : [Result] "=&r" (__result),                 \
 726                                       "=m" (*(atomic))                  \
 727                            : [Atomic] "r" (atomic),                     \
 728                              [OldVal] "r" (oldval),                     \
 729                              [NewVal] "r" (newval),                     \
 730                                       "g" (*(gpointer*) (atomic))       \
 731                            : "memory");                                 \
 732      __result;                                                          \
 733   })
 734 #  endif
 735
 736 #define CRIS_CACHELINE_SIZE 32
 737 #define CRIS_ATOMIC_BREAKS_CACHELINE(atomic) \
 738   (((gulong)(atomic) & (CRIS_CACHELINE_SIZE - 1)) > (CRIS_CACHELINE_SIZE - sizeof (atomic)))
 739
 740 gint     __g_atomic_int_exchange_and_add         (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 741                                                   gint             val);
 742 void     __g_atomic_int_add                      (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 743                                                   gint             val);
 744 gboolean __g_atomic_int_compare_and_exchange     (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 745                                                   gint             oldval,
 746                                                   gint             newval);
 747 gboolean __g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 748                                                   gpointer         oldval,
 749                                                   gpointer         newval);
 750
 751 gboolean
 752 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 753                                        gpointer           oldval,
 754                                        gpointer           newval)
 755 {
 756   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 757     return __g_atomic_pointer_compare_and_exchange (atomic, oldval, newval);
 758
 759   return CRIS_ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 760 }
 761
 762 gboolean
 763 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 764                                    gint           oldval,
 765                                    gint           newval)
 766 {
 767   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 768     return __g_atomic_int_compare_and_exchange (atomic, oldval, newval);
 769
 770   return CRIS_ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 771 }
 772
 773 gint
 774 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 775                                gint           val)
 776 {
 777   gint result;
 778
 779   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 780     return __g_atomic_int_exchange_and_add (atomic, val);
 781
 782   do
 783     result = *atomic;
 784   while (!CRIS_ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
 785
 786   return result;
 787 }
 788
 789 void
 790 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 791                   gint           val)
 792 {
 793   gint result;
 794
 795   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 796     return __g_atomic_int_add (atomic, val);
 797
 798   do
 799     result = *atomic;
 800   while (!CRIS_ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
 801 }
 802
 803 /* We need the atomic mutex for atomic operations where the atomic variable
 804  * breaks the 32 byte cache line since the CRIS architecture does not support
 805  * atomic operations on such variables. Fortunately this should be rare.
 806  */
 807 #  define DEFINE_WITH_MUTEXES
 808 #  define g_atomic_int_exchange_and_add __g_atomic_int_exchange_and_add
 809 #  define g_atomic_int_add __g_atomic_int_add
 810 #  define g_atomic_int_compare_and_exchange __g_atomic_int_compare_and_exchange
 811 #  define g_atomic_pointer_compare_and_exchange __g_atomic_pointer_compare_and_exchange
 812
 813 # else /* !G_ATOMIC_* */
 814 #  define DEFINE_WITH_MUTEXES
 815 # endif /* G_ATOMIC_* */
 816 #else /* !__GNUC__ */
 817 # ifdef G_PLATFORM_WIN32
 818 #  define DEFINE_WITH_WIN32_INTERLOCKED
 819 # else
 820 #  define DEFINE_WITH_MUTEXES
 821 # endif
 822 #endif /* __GNUC__ */
 823
 824 #ifdef DEFINE_WITH_WIN32_INTERLOCKED
 825 # include <windows.h>
 826 /* Following indicates that InterlockedCompareExchangePointer is
 827  * declared in winbase.h (included by windows.h) and needs to be
 828  * commented out if not true. It is defined iff WINVER > 0x0400,
 829  * which is usually correct but can be wrong if WINVER is set before
 830  * windows.h is included.
 831  */
 832 # if WINVER > 0x0400
 833 #  define HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 834 # endif
 835
 836 gint32
 837 g_atomic_int_exchange_and_add (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 838                                gint32           val)
 839 {
 840   return InterlockedExchangeAdd (atomic, val);
 841 }
 842
 843 void
 844 g_atomic_int_add (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 845                   gint32           val)
 846 {
 847   InterlockedExchangeAdd (atomic, val);
 848 }
 849
 850 gboolean
 851 g_atomic_int_compare_and_exchange (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 852                                    gint32           oldval,
 853                                    gint32           newval)
 854 {
 855 #ifndef HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 856   return (guint32) InterlockedCompareExchange ((PVOID*)atomic,
 857                                                (PVOID)newval,
 858                                                (PVOID)oldval) == oldval;
 859 #else
 860   return InterlockedCompareExchange (atomic,
 861                                      newval,
 862                                      oldval) == oldval;
 863 #endif
 864 }
 865
 866 gboolean
 867 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 868                                        gpointer           oldval,
 869                                        gpointer           newval)
 870 {
 871 # ifdef HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 872   return InterlockedCompareExchangePointer (atomic, newval, oldval) == oldval;
 873 # else
 874 #  if GLIB_SIZEOF_VOID_P != 4 /* no 32-bit system */
 875 #   error "InterlockedCompareExchangePointer needed"
 876 #  else
 877    return InterlockedCompareExchange (atomic, newval, oldval) == oldval;
 878 #  endif
 879 # endif
 880 }
 881 #endif /* DEFINE_WITH_WIN32_INTERLOCKED */
 882
 883 #ifdef DEFINE_WITH_MUTEXES
 884 /* We have to use the slow, but safe locking method */
 885 static GMutex *g_atomic_mutex;
 886
 887 /**
 888  * g_atomic_int_exchange_and_add:
 889  * @atomic: a pointer to an integer
 890  * @val: the value to add to *@atomic
 891  *
 892  * Atomically adds @val to the integer pointed to by @atomic.
 893  * It returns the value of *@atomic just before the addition
 894  * took place. Also acts as a memory barrier.
 895  *
 896  * Returns: the value of *@atomic before the addition.
 897  *
 898  * Since: 2.4
 899  */
 900 gint
 901 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 902                                gint           val)
 903 {
 904   gint result;
 905
 906   g_mutex_lock (g_atomic_mutex);
 907   result = *atomic;
 908   *atomic += val;
 909   g_mutex_unlock (g_atomic_mutex);
 910
 911   return result;
 912 }
 913
 914 /**
 915  * g_atomic_int_add:
 916  * @atomic: a pointer to an integer
 917  * @val: the value to add to *@atomic
 918  *
 919  * Atomically adds @val to the integer pointed to by @atomic.
 920  * Also acts as a memory barrier.
 921  *
 922  * Since: 2.4
 923  */
 924 void
 925 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 926                   gint           val)
 927 {
 928   g_mutex_lock (g_atomic_mutex);
 929   *atomic += val;
 930   g_mutex_unlock (g_atomic_mutex);
 931 }
 932
 933 /**
 934  * g_atomic_int_compare_and_exchange:
 935  * @atomic: a pointer to an integer
 936  * @oldval: the assumed old value of *@atomic
 937  * @newval: the new value of *@atomic
 938  *
 939  * Compares @oldval with the integer pointed to by @atomic and
 940  * if they are equal, atomically exchanges *@atomic with @newval.
 941  * Also acts as a memory barrier.
 942  *
 943  * Returns: %TRUE, if *@atomic was equal @oldval. %FALSE otherwise.
 944  *
 945  * Since: 2.4
 946  */
 947 gboolean
 948 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 949                                    gint           oldval,
 950                                    gint           newval)
 951 {
 952   gboolean result;
 953
 954   g_mutex_lock (g_atomic_mutex);
 955   if (*atomic == oldval)
 956     {
 957       result = TRUE;
 958       *atomic = newval;
 959     }
 960   else
 961     result = FALSE;
 962   g_mutex_unlock (g_atomic_mutex);
 963
 964   return result;
 965 }
 966
 967 /**
 968  * g_atomic_pointer_compare_and_exchange:
 969  * @atomic: a pointer to a #gpointer
 970  * @oldval: the assumed old value of *@atomic
 971  * @newval: the new value of *@atomic
 972  *
 973  * Compares @oldval with the pointer pointed to by @atomic and
 974  * if they are equal, atomically exchanges *@atomic with @newval.
 975  * Also acts as a memory barrier.
 976  *
 977  * Returns: %TRUE, if *@atomic was equal @oldval. %FALSE otherwise.
 978  *
 979  * Since: 2.4
 980  */
 981 gboolean
 982 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 983                                        gpointer           oldval,
 984                                        gpointer           newval)
 985 {
 986   gboolean result;
 987
 988   g_mutex_lock (g_atomic_mutex);
 989   if (*atomic == oldval)
 990     {
 991       result = TRUE;
 992       *atomic = newval;
 993     }
 994   else
 995     result = FALSE;
 996   g_mutex_unlock (g_atomic_mutex);
 997
 998   return result;
 999 }
1000
1001 #ifdef G_ATOMIC_OP_MEMORY_BARRIER_NEEDED
1002
1003 /**
1004  * g_atomic_int_get:
1005  * @atomic: a pointer to an integer
1006  *
1007  * Reads the value of the integer pointed to by @atomic.
1008  * Also acts as a memory barrier.
1009  *
1010  * Returns: the value of *@atomic
1011  *
1012  * Since: 2.4
1013  */
1014 gint
1015 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1016 {
1017   gint result;
1018
1019   g_mutex_lock (g_atomic_mutex);
1020   result = *atomic;
1021   g_mutex_unlock (g_atomic_mutex);
1022
1023   return result;
1024 }
1025
1026 /**
1027  * g_atomic_int_set:
1028  * @atomic: a pointer to an integer
1029  * @newval: the new value
1030  *
1031  * Sets the value of the integer pointed to by @atomic.
1032  * Also acts as a memory barrier.
1033  *
1034  * Since: 2.10
1035  */
1036 void
1037 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1038                   gint           newval)
1039 {
1040   g_mutex_lock (g_atomic_mutex);
1041   *atomic = newval;
1042   g_mutex_unlock (g_atomic_mutex);
1043 }
1044
1045 /**
1046  * g_atomic_pointer_get:
1047  * @atomic: a pointer to a #gpointer.
1048  *
1049  * Reads the value of the pointer pointed to by @atomic.
1050  * Also acts as a memory barrier.
1051  *
1052  * Returns: the value to add to *@atomic.
1053  *
1054  * Since: 2.4
1055  */
1056 gpointer
1057 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1058 {
1059   gpointer result;
1060
1061   g_mutex_lock (g_atomic_mutex);
1062   result = *atomic;
1063   g_mutex_unlock (g_atomic_mutex);
1064
1065   return result;
1066 }
1067
1068 /**
1069  * g_atomic_pointer_set:
1070  * @atomic: a pointer to a #gpointer
1071  * @newval: the new value
1072  *
1073  * Sets the value of the pointer pointed to by @atomic.
1074  * Also acts as a memory barrier.
1075  *
1076  * Since: 2.10
1077  */
1078 void
1079 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1080                       gpointer           newval)
1081 {
1082   g_mutex_lock (g_atomic_mutex);
1083   *atomic = newval;
1084   g_mutex_unlock (g_atomic_mutex);
1085 }
1086 #endif /* G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */
1087 #elif defined (G_ATOMIC_OP_MEMORY_BARRIER_NEEDED)
1088 gint
1089 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1090 {
1091   G_ATOMIC_MEMORY_BARRIER;
1092   return *atomic;
1093 }
1094
1095 void
1096 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1097                   gint           newval)
1098 {
1099   *atomic = newval;
1100   G_ATOMIC_MEMORY_BARRIER;
1101 }
1102
1103 gpointer
1104 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1105 {
1106   G_ATOMIC_MEMORY_BARRIER;
1107   return *atomic;
1108 }
1109
1110 void
1111 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1112                       gpointer           newval)
1113 {
1114   *atomic = newval;
1115   G_ATOMIC_MEMORY_BARRIER;
1116 }
1117 #endif /* DEFINE_WITH_MUTEXES || G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */
1118
1119 #ifdef ATOMIC_INT_CMP_XCHG
1120 gboolean
1121 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
1122                                    gint           oldval,
1123                                    gint           newval)
1124 {
1125   return ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
1126 }
1127
1128 gint
1129 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
1130                                gint           val)
1131 {
1132   gint result;
1133   do
1134     result = *atomic;
1135   while (!ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
1136
1137   return result;
1138 }
1139
1140 void
1141 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
1142                   gint           val)
1143 {
1144   gint result;
1145   do
1146     result = *atomic;
1147   while (!ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
1148 }
1149 #endif /* ATOMIC_INT_CMP_XCHG */
1150
1151 void
1152 _g_atomic_thread_init (void)
1153 {
1154 #ifdef DEFINE_WITH_MUTEXES
1155   g_atomic_mutex = g_mutex_new ();
1156 #endif /* DEFINE_WITH_MUTEXES */
1157 }
1158
1159 #ifndef G_ATOMIC_OP_MEMORY_BARRIER_NEEDED
1160 gint
1161 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1162 {
1163   return g_atomic_int_get (atomic);
1164 }
1165
1166 void
1167 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1168                     gint           newval)
1169 {
1170   g_atomic_int_set (atomic, newval);
1171 }
1172
1173 gpointer
1174 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1175 {
1176   return g_atomic_pointer_get (atomic);
1177 }
1178
1179 void
1180 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1181                         gpointer           newval)
1182 {
1183   g_atomic_pointer_set (atomic, newval);
1184 }
1185 #endif /* G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */