glib/gatomic.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * g_atomic_*: atomic operations.
   5  * Copyright (C) 2003 Sebastian Wilhelmi
   6  * Copyright (C) 2007 Nokia Corporation
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the
  20  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  21  * Boston, MA 02111-1307, USA.
  22  */
  23
  24 #include "config.h"
  25
  26 #if defined (G_ATOMIC_ARM)
  27 #include <sched.h>
  28 #endif
  29
  30 #include "gatomic.h"
  31 #include "gthread.h"
  32 #include "gthreadprivate.h"
  33
  34 /**
  35  * SECTION:atomic_operations
  36  * @title: Atomic Operations
  37  * @short_description: basic atomic integer and pointer operations
  38  * @see_also: #GMutex
  39  *
  40  * The following functions can be used to atomically access integers and
  41  * pointers. They are implemented as inline assembler function on most
  42  * platforms and use slower fall-backs otherwise. Using them can sometimes
  43  * save you from using a performance-expensive #GMutex to protect the
  44  * integer or pointer.
  45  *
  46  * The most important usage is reference counting. Using
  47  * g_atomic_int_inc() and g_atomic_int_dec_and_test() makes reference
  48  * counting a very fast operation.
  49  *
  50  * <note><para>You must not directly read integers or pointers concurrently
  51  * accessed by multiple threads, but use the atomic accessor functions
  52  * instead. That is, always use g_atomic_int_get() and g_atomic_pointer_get()
  53  * for read outs. They provide the neccessary synchonization mechanisms
  54  * like memory barriers to access memory locations concurrently.
  55  * </para></note>
  56  *
  57  * <note><para>If you are using those functions for anything apart from
  58  * simple reference counting, you should really be aware of the implications
  59  * of doing that. There are literally thousands of ways to shoot yourself
  60  * in the foot. So if in doubt, use a #GMutex. If you don't know, what
  61  * memory barriers are, do not use anything but g_atomic_int_inc() and
  62  * g_atomic_int_dec_and_test().
  63  * </para></note>
  64  *
  65  * <note><para>It is not safe to set an integer or pointer just by assigning
  66  * to it, when it is concurrently accessed by other threads with the following
  67  * functions. Use g_atomic_int_compare_and_exchange() or
  68  * g_atomic_pointer_compare_and_exchange() respectively.
  69  * </para></note>
  70  */
  71
  72 #if defined (__GNUC__)
  73 # if defined (G_ATOMIC_I486)
  74 /* Adapted from CVS version 1.10 of glibc's sysdeps/i386/i486/bits/atomic.h
  75  */
  76 gint
  77 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
  78                                gint           val)
  79 {
  80   gint result;
  81
  82   __asm__ __volatile__ ("lock; xaddl %0,%1"
  83                         : "=r" (result), "=m" (*atomic)
  84                         : "0" (val), "m" (*atomic));
  85   return result;
  86 }
  87
  88 void
  89 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
  90                   gint           val)
  91 {
  92   __asm__ __volatile__ ("lock; addl %1,%0"
  93                         : "=m" (*atomic)
  94                         : "ir" (val), "m" (*atomic));
  95 }
  96
  97 gboolean
  98 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
  99                                    gint           oldval,
 100                                    gint           newval)
 101 {
 102   gint result;
 103
 104   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 105                         : "=a" (result), "=m" (*atomic)
 106                         : "r" (newval), "m" (*atomic), "0" (oldval));
 107
 108   return result == oldval;
 109 }
 110
 111 /* The same code as above, as on i386 gpointer is 32 bit as well.
 112  * Duplicating the code here seems more natural than casting the
 113  * arguments and calling the former function */
 114
 115 gboolean
 116 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 117                                        gpointer           oldval,
 118                                        gpointer           newval)
 119 {
 120   gpointer result;
 121
 122   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 123                         : "=a" (result), "=m" (*atomic)
 124                         : "r" (newval), "m" (*atomic), "0" (oldval));
 125
 126   return result == oldval;
 127 }
 128
 129 # elif defined (G_ATOMIC_SPARCV9)
 130 /* Adapted from CVS version 1.3 of glibc's sysdeps/sparc/sparc64/bits/atomic.h
 131  */
 132 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 133   ({                                                                    \
 134      gint __result;                                                     \
 135      __asm__ __volatile__ ("cas [%4], %2, %0"                           \
 136                            : "=r" (__result), "=m" (*(atomic))          \
 137                            : "r" (oldval), "m" (*(atomic)), "r" (atomic),\
 138                            "0" (newval));                               \
 139      __result == oldval;                                                \
 140   })
 141
 142 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 143 gboolean
 144 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 145                                        gpointer           oldval,
 146                                        gpointer           newval)
 147 {
 148   gpointer result;
 149   __asm__ __volatile__ ("cas [%4], %2, %0"
 150                         : "=r" (result), "=m" (*atomic)
 151                         : "r" (oldval), "m" (*atomic), "r" (atomic),
 152                         "0" (newval));
 153   return result == oldval;
 154 }
 155 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 156 gboolean
 157 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 158                                        gpointer           oldval,
 159                                        gpointer           newval)
 160 {
 161   gpointer result;
 162   gpointer *a = atomic;
 163   __asm__ __volatile__ ("casx [%4], %2, %0"
 164                         : "=r" (result), "=m" (*a)
 165                         : "r" (oldval), "m" (*a), "r" (a),
 166                         "0" (newval));
 167   return result == oldval;
 168 }
 169 #  else /* What's that */
 170 #    error "Your system has an unsupported pointer size"
 171 #  endif /* GLIB_SIZEOF_VOID_P */
 172 #  define G_ATOMIC_MEMORY_BARRIER                                       \
 173   __asm__ __volatile__ ("membar #LoadLoad | #LoadStore"                 \
 174                         " | #StoreLoad | #StoreStore" : : : "memory")
 175
 176 # elif defined (G_ATOMIC_ALPHA)
 177 /* Adapted from CVS version 1.3 of glibc's sysdeps/alpha/bits/atomic.h
 178  */
 179 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 180   ({                                                                    \
 181      gint __result;                                                     \
 182      gint __prev;                                                       \
 183      __asm__ __volatile__ (                                             \
 184         "       mb\n"                                                   \
 185         "1:     ldl_l   %0,%2\n"                                        \
 186         "       cmpeq   %0,%3,%1\n"                                     \
 187         "       beq     %1,2f\n"                                        \
 188         "       mov     %4,%1\n"                                        \
 189         "       stl_c   %1,%2\n"                                        \
 190         "       beq     %1,1b\n"                                        \
 191         "       mb\n"                                                   \
 192         "2:"                                                            \
 193         : "=&r" (__prev),                                               \
 194           "=&r" (__result)                                              \
 195         : "m" (*(atomic)),                                              \
 196           "Ir" (oldval),                                                \
 197           "Ir" (newval)                                                 \
 198         : "memory");                                                    \
 199      __result != 0;                                                     \
 200   })
 201 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 202 gboolean
 203 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 204                                        gpointer           oldval,
 205                                        gpointer           newval)
 206 {
 207   gint result;
 208   gpointer prev;
 209   __asm__ __volatile__ (
 210         "       mb\n"
 211         "1:     ldl_l   %0,%2\n"
 212         "       cmpeq   %0,%3,%1\n"
 213         "       beq     %1,2f\n"
 214         "       mov     %4,%1\n"
 215         "       stl_c   %1,%2\n"
 216         "       beq     %1,1b\n"
 217         "       mb\n"
 218         "2:"
 219         : "=&r" (prev),
 220           "=&r" (result)
 221         : "m" (*atomic),
 222           "Ir" (oldval),
 223           "Ir" (newval)
 224         : "memory");
 225   return result != 0;
 226 }
 227 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 228 gboolean
 229 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 230                                        gpointer           oldval,
 231                                        gpointer           newval)
 232 {
 233   gint result;
 234   gpointer prev;
 235   __asm__ __volatile__ (
 236         "       mb\n"
 237         "1:     ldq_l   %0,%2\n"
 238         "       cmpeq   %0,%3,%1\n"
 239         "       beq     %1,2f\n"
 240         "       mov     %4,%1\n"
 241         "       stq_c   %1,%2\n"
 242         "       beq     %1,1b\n"
 243         "       mb\n"
 244         "2:"
 245         : "=&r" (prev),
 246           "=&r" (result)
 247         : "m" (*atomic),
 248           "Ir" (oldval),
 249           "Ir" (newval)
 250         : "memory");
 251   return result != 0;
 252 }
 253 #  else /* What's that */
 254 #   error "Your system has an unsupported pointer size"
 255 #  endif /* GLIB_SIZEOF_VOID_P */
 256 #  define G_ATOMIC_MEMORY_BARRIER  __asm__ ("mb" : : : "memory")
 257 # elif defined (G_ATOMIC_X86_64)
 258 /* Adapted from CVS version 1.9 of glibc's sysdeps/x86_64/bits/atomic.h
 259  */
 260 gint
 261 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 262                                gint           val)
 263 {
 264   gint result;
 265
 266   __asm__ __volatile__ ("lock; xaddl %0,%1"
 267                         : "=r" (result), "=m" (*atomic)
 268                         : "0" (val), "m" (*atomic));
 269   return result;
 270 }
 271
 272 void
 273 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 274                   gint           val)
 275 {
 276   __asm__ __volatile__ ("lock; addl %1,%0"
 277                         : "=m" (*atomic)
 278                         : "ir" (val), "m" (*atomic));
 279 }
 280
 281 gboolean
 282 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 283                                    gint           oldval,
 284                                    gint           newval)
 285 {
 286   gint result;
 287
 288   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 289                         : "=a" (result), "=m" (*atomic)
 290                         : "r" (newval), "m" (*atomic), "0" (oldval));
 291
 292   return result == oldval;
 293 }
 294
 295 gboolean
 296 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 297                                        gpointer           oldval,
 298                                        gpointer           newval)
 299 {
 300   gpointer result;
 301
 302   __asm__ __volatile__ ("lock; cmpxchgq %q2, %1"
 303                         : "=a" (result), "=m" (*atomic)
 304                         : "r" (newval), "m" (*atomic), "0" (oldval));
 305
 306   return result == oldval;
 307 }
 308
 309 # elif defined (G_ATOMIC_POWERPC)
 310 /* Adapted from CVS version 1.16 of glibc's sysdeps/powerpc/bits/atomic.h
 311  * and CVS version 1.4 of glibc's sysdeps/powerpc/powerpc32/bits/atomic.h
 312  * and CVS version 1.7 of glibc's sysdeps/powerpc/powerpc64/bits/atomic.h
 313  */
 314 #   ifdef __OPTIMIZE__
 315 /* Non-optimizing compile bails on the following two asm statements
 316  * for reasons unknown to the author */
 317 gint
 318 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 319                                gint           val)
 320 {
 321   gint result, temp;
 322 #if ASM_NUMERIC_LABELS
 323   __asm__ __volatile__ ("1:       lwarx   %0,0,%3\n"
 324                         "         add     %1,%0,%4\n"
 325                         "         stwcx.  %1,0,%3\n"
 326                         "         bne-    1b"
 327                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 328                         : "b" (atomic), "r" (val), "m" (*atomic)
 329                         : "cr0", "memory");
 330 #else
 331   __asm__ __volatile__ (".Lieaa%=:       lwarx   %0,0,%3\n"
 332                         "         add     %1,%0,%4\n"
 333                         "         stwcx.  %1,0,%3\n"
 334                         "         bne-    .Lieaa%="
 335                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 336                         : "b" (atomic), "r" (val), "m" (*atomic)
 337                         : "cr0", "memory");
 338 #endif
 339   return result;
 340 }
 341
 342 /* The same as above, to save a function call repeated here */
 343 void
 344 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 345                   gint           val)
 346 {
 347   gint result, temp;
 348 #if ASM_NUMERIC_LABELS
 349   __asm__ __volatile__ ("1:       lwarx   %0,0,%3\n"
 350                         "         add     %1,%0,%4\n"
 351                         "         stwcx.  %1,0,%3\n"
 352                         "         bne-    1b"
 353                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 354                         : "b" (atomic), "r" (val), "m" (*atomic)
 355                         : "cr0", "memory");
 356 #else
 357   __asm__ __volatile__ (".Lia%=:       lwarx   %0,0,%3\n"
 358                         "         add     %1,%0,%4\n"
 359                         "         stwcx.  %1,0,%3\n"
 360                         "         bne-    .Lia%="
 361                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 362                         : "b" (atomic), "r" (val), "m" (*atomic)
 363                         : "cr0", "memory");
 364 #endif
 365 }
 366 #   else /* !__OPTIMIZE__ */
 367 gint
 368 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 369                                gint           val)
 370 {
 371   gint result;
 372   do
 373     result = *atomic;
 374   while (!g_atomic_int_compare_and_exchange (atomic, result, result + val));
 375
 376   return result;
 377 }
 378
 379 void
 380 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 381                   gint           val)
 382 {
 383   gint result;
 384   do
 385     result = *atomic;
 386   while (!g_atomic_int_compare_and_exchange (atomic, result, result + val));
 387 }
 388 #   endif /* !__OPTIMIZE__ */
 389
 390 #   if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 391 gboolean
 392 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 393                                    gint           oldval,
 394                                    gint           newval)
 395 {
 396   gint result;
 397 #if ASM_NUMERIC_LABELS
 398   __asm__ __volatile__ ("sync\n"
 399                         "1: lwarx   %0,0,%1\n"
 400                         "   subf.   %0,%2,%0\n"
 401                         "   bne     2f\n"
 402                         "   stwcx.  %3,0,%1\n"
 403                         "   bne-    1b\n"
 404                         "2: isync"
 405                         : "=&r" (result)
 406                         : "b" (atomic), "r" (oldval), "r" (newval)
 407                         : "cr0", "memory");
 408 #else
 409   __asm__ __volatile__ ("sync\n"
 410                         ".L1icae%=: lwarx   %0,0,%1\n"
 411                         "   subf.   %0,%2,%0\n"
 412                         "   bne     .L2icae%=\n"
 413                         "   stwcx.  %3,0,%1\n"
 414                         "   bne-    .L1icae%=\n"
 415                         ".L2icae%=: isync"
 416                         : "=&r" (result)
 417                         : "b" (atomic), "r" (oldval), "r" (newval)
 418                         : "cr0", "memory");
 419 #endif
 420   return result == 0;
 421 }
 422
 423 gboolean
 424 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 425                                        gpointer           oldval,
 426                                        gpointer           newval)
 427 {
 428   gpointer result;
 429 #if ASM_NUMERIC_LABELS
 430   __asm__ __volatile__ ("sync\n"
 431                         "1: lwarx   %0,0,%1\n"
 432                         "   subf.   %0,%2,%0\n"
 433                         "   bne     2f\n"
 434                         "   stwcx.  %3,0,%1\n"
 435                         "   bne-    1b\n"
 436                         "2: isync"
 437                         : "=&r" (result)
 438                         : "b" (atomic), "r" (oldval), "r" (newval)
 439                         : "cr0", "memory");
 440 #else
 441   __asm__ __volatile__ ("sync\n"
 442                         ".L1pcae%=: lwarx   %0,0,%1\n"
 443                         "   subf.   %0,%2,%0\n"
 444                         "   bne     .L2pcae%=\n"
 445                         "   stwcx.  %3,0,%1\n"
 446                         "   bne-    .L1pcae%=\n"
 447                         ".L2pcae%=: isync"
 448                         : "=&r" (result)
 449                         : "b" (atomic), "r" (oldval), "r" (newval)
 450                         : "cr0", "memory");
 451 #endif
 452   return result == 0;
 453 }
 454 #   elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 455 gboolean
 456 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 457                                    gint           oldval,
 458                                    gint           newval)
 459 {
 460   gpointer result;
 461 #if ASM_NUMERIC_LABELS
 462   __asm__ __volatile__ ("sync\n"
 463                         "1: lwarx   %0,0,%1\n"
 464                         "   extsw   %0,%0\n"
 465                         "   subf.   %0,%2,%0\n"
 466                         "   bne     2f\n"
 467                         "   stwcx.  %3,0,%1\n"
 468                         "   bne-    1b\n"
 469                         "2: isync"
 470                         : "=&r" (result)
 471                         : "b" (atomic), "r" (oldval), "r" (newval)
 472                         : "cr0", "memory");
 473 #else
 474   __asm__ __volatile__ ("sync\n"
 475                         ".L1icae%=: lwarx   %0,0,%1\n"
 476                         "   extsw   %0,%0\n"
 477                         "   subf.   %0,%2,%0\n"
 478                         "   bne     .L2icae%=\n"
 479                         "   stwcx.  %3,0,%1\n"
 480                         "   bne-    .L1icae%=\n"
 481                         ".L2icae%=: isync"
 482                         : "=&r" (result)
 483                         : "b" (atomic), "r" (oldval), "r" (newval)
 484                         : "cr0", "memory");
 485 #endif
 486   return result == 0;
 487 }
 488
 489 gboolean
 490 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 491                                        gpointer           oldval,
 492                                        gpointer           newval)
 493 {
 494   gpointer result;
 495 #if ASM_NUMERIC_LABELS
 496   __asm__ __volatile__ ("sync\n"
 497                         "1: ldarx   %0,0,%1\n"
 498                         "   subf.   %0,%2,%0\n"
 499                         "   bne     2f\n"
 500                         "   stdcx.  %3,0,%1\n"
 501                         "   bne-    1b\n"
 502                         "2: isync"
 503                         : "=&r" (result)
 504                         : "b" (atomic), "r" (oldval), "r" (newval)
 505                         : "cr0", "memory");
 506 #else
 507   __asm__ __volatile__ ("sync\n"
 508                         ".L1pcae%=: ldarx   %0,0,%1\n"
 509                         "   subf.   %0,%2,%0\n"
 510                         "   bne     .L2pcae%=\n"
 511                         "   stdcx.  %3,0,%1\n"
 512                         "   bne-    .L1pcae%=\n"
 513                         ".L2pcae%=: isync"
 514                         : "=&r" (result)
 515                         : "b" (atomic), "r" (oldval), "r" (newval)
 516                         : "cr0", "memory");
 517 #endif
 518   return result == 0;
 519 }
 520 #  else /* What's that */
 521 #   error "Your system has an unsupported pointer size"
 522 #  endif /* GLIB_SIZEOF_VOID_P */
 523
 524 #  define G_ATOMIC_MEMORY_BARRIER __asm__ ("sync" : : : "memory")
 525
 526 # elif defined (G_ATOMIC_IA64)
 527 /* Adapted from CVS version 1.8 of glibc's sysdeps/ia64/bits/atomic.h
 528  */
 529 gint
 530 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 531                                gint           val)
 532 {
 533   return __sync_fetch_and_add (atomic, val);
 534 }
 535
 536 void
 537 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 538                   gint val)
 539 {
 540   __sync_fetch_and_add (atomic, val);
 541 }
 542
 543 gboolean
 544 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 545                                    gint           oldval,
 546                                    gint           newval)
 547 {
 548   return __sync_bool_compare_and_swap (atomic, oldval, newval);
 549 }
 550
 551 gboolean
 552 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 553                                        gpointer           oldval,
 554                                        gpointer           newval)
 555 {
 556   return __sync_bool_compare_and_swap ((long *)atomic,
 557                                        (long)oldval, (long)newval);
 558 }
 559
 560 #  define G_ATOMIC_MEMORY_BARRIER __sync_synchronize ()
 561 # elif defined (G_ATOMIC_S390)
 562 /* Adapted from glibc's sysdeps/s390/bits/atomic.h
 563  */
 564 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 565   ({                                                                    \
 566      gint __result = oldval;                                    \
 567      __asm__ __volatile__ ("cs %0, %2, %1"                              \
 568                            : "+d" (__result), "=Q" (*(atomic))          \
 569                            : "d" (newval), "m" (*(atomic)) : "cc" );    \
 570      __result == oldval;                                                \
 571   })
 572
 573 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 574 gboolean
 575 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 576                                        gpointer           oldval,
 577                                        gpointer           newval)
 578 {
 579   gpointer result = oldval;
 580   __asm__ __volatile__ ("cs %0, %2, %1"
 581                         : "+d" (result), "=Q" (*(atomic))
 582                         : "d" (newval), "m" (*(atomic)) : "cc" );
 583   return result == oldval;
 584 }
 585 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 586 gboolean
 587 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 588                                        gpointer           oldval,
 589                                        gpointer           newval)
 590 {
 591   gpointer result = oldval;
 592   gpointer *a = atomic;
 593   __asm__ __volatile__ ("csg %0, %2, %1"
 594                         : "+d" (result), "=Q" (*a)
 595                         : "d" ((long)(newval)), "m" (*a) : "cc" );
 596   return result == oldval;
 597 }
 598 #  else /* What's that */
 599 #    error "Your system has an unsupported pointer size"
 600 #  endif /* GLIB_SIZEOF_VOID_P */
 601 # elif defined (G_ATOMIC_ARM)
 602 static volatile int atomic_spin = 0;
 603
 604 static int atomic_spin_trylock (void)
 605 {
 606   int result;
 607
 608   asm volatile (
 609     "swp %0, %1, [%2]\n"
 610     : "=&r,&r" (result)
 611     : "r,0" (1), "r,r" (&atomic_spin)
 612     : "memory");
 613   if (result == 0)
 614     return 0;
 615   else
 616     return -1;
 617 }
 618
 619 static void atomic_spin_lock (void)
 620 {
 621   while (atomic_spin_trylock())
 622     sched_yield();
 623 }
 624
 625 static void atomic_spin_unlock (void)
 626 {
 627   atomic_spin = 0;
 628 }
 629
 630 gint
 631 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 632                                gint           val)
 633 {
 634   gint result;
 635
 636   atomic_spin_lock();
 637   result = *atomic;
 638   *atomic += val;
 639   atomic_spin_unlock();
 640
 641   return result;
 642 }
 643
 644 void
 645 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 646                   gint           val)
 647 {
 648   atomic_spin_lock();
 649   *atomic += val;
 650   atomic_spin_unlock();
 651 }
 652
 653 gboolean
 654 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 655                                    gint           oldval,
 656                                    gint           newval)
 657 {
 658   gboolean result;
 659
 660   atomic_spin_lock();
 661   if (*atomic == oldval)
 662     {
 663       result = TRUE;
 664       *atomic = newval;
 665     }
 666   else
 667     result = FALSE;
 668   atomic_spin_unlock();
 669
 670   return result;
 671 }
 672
 673 gboolean
 674 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 675                                        gpointer           oldval,
 676                                        gpointer           newval)
 677 {
 678   gboolean result;
 679
 680   atomic_spin_lock();
 681   if (*atomic == oldval)
 682     {
 683       result = TRUE;
 684       *atomic = newval;
 685     }
 686   else
 687     result = FALSE;
 688   atomic_spin_unlock();
 689
 690   return result;
 691 }
 692 # elif defined (G_ATOMIC_CRIS) || defined (G_ATOMIC_CRISV32)
 693 #  ifdef G_ATOMIC_CRIS
 694 #   define CRIS_ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)             \
 695   ({                                                                    \
 696      gboolean __result;                                                 \
 697      __asm__ __volatile__ ("\n"                                         \
 698                            "0:\tclearf\n\t"                             \
 699                            "cmp.d [%[Atomic]], %[OldVal]\n\t"           \
 700                            "bne 1f\n\t"                                 \
 701                            "ax\n\t"                                     \
 702                            "move.d %[NewVal], [%[Atomic]]\n\t"          \
 703                            "bwf 0b\n"                                   \
 704                            "1:\tseq %[Result]"                          \
 705                            : [Result] "=&r" (__result),                 \
 706                                       "=m" (*(atomic))                  \
 707                            : [Atomic] "r" (atomic),                     \
 708                              [OldVal] "r" (oldval),                     \
 709                              [NewVal] "r" (newval),                     \
 710                                       "g" (*(gpointer*) (atomic))       \
 711                            : "memory");                                 \
 712      __result;                                                          \
 713   })
 714 #  else
 715 #   define CRIS_ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)             \
 716   ({                                                                    \
 717      gboolean __result;                                                 \
 718      __asm__ __volatile__ ("\n"                                         \
 719                            "0:\tclearf p\n\t"                           \
 720                            "cmp.d [%[Atomic]], %[OldVal]\n\t"           \
 721                            "bne 1f\n\t"                                 \
 722                            "ax\n\t"                                     \
 723                            "move.d %[NewVal], [%[Atomic]]\n\t"          \
 724                            "bcs 0b\n"                                   \
 725                            "1:\tseq %[Result]"                          \
 726                            : [Result] "=&r" (__result),                 \
 727                                       "=m" (*(atomic))                  \
 728                            : [Atomic] "r" (atomic),                     \
 729                              [OldVal] "r" (oldval),                     \
 730                              [NewVal] "r" (newval),                     \
 731                                       "g" (*(gpointer*) (atomic))       \
 732                            : "memory");                                 \
 733      __result;                                                          \
 734   })
 735 #  endif
 736
 737 #define CRIS_CACHELINE_SIZE 32
 738 #define CRIS_ATOMIC_BREAKS_CACHELINE(atomic) \
 739   (((gulong)(atomic) & (CRIS_CACHELINE_SIZE - 1)) > (CRIS_CACHELINE_SIZE - sizeof (atomic)))
 740
 741 gint     __g_atomic_int_exchange_and_add         (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 742                                                   gint             val);
 743 void     __g_atomic_int_add                      (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 744                                                   gint             val);
 745 gboolean __g_atomic_int_compare_and_exchange     (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 746                                                   gint             oldval,
 747                                                   gint             newval);
 748 gboolean __g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 749                                                   gpointer         oldval,
 750                                                   gpointer         newval);
 751
 752 gboolean
 753 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 754                                        gpointer           oldval,
 755                                        gpointer           newval)
 756 {
 757   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 758     return __g_atomic_pointer_compare_and_exchange (atomic, oldval, newval);
 759
 760   return CRIS_ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 761 }
 762
 763 gboolean
 764 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 765                                    gint           oldval,
 766                                    gint           newval)
 767 {
 768   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 769     return __g_atomic_int_compare_and_exchange (atomic, oldval, newval);
 770
 771   return CRIS_ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 772 }
 773
 774 gint
 775 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 776                                gint           val)
 777 {
 778   gint result;
 779
 780   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 781     return __g_atomic_int_exchange_and_add (atomic, val);
 782
 783   do
 784     result = *atomic;
 785   while (!CRIS_ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
 786
 787   return result;
 788 }
 789
 790 void
 791 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 792                   gint           val)
 793 {
 794   gint result;
 795
 796   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 797     return __g_atomic_int_add (atomic, val);
 798
 799   do
 800     result = *atomic;
 801   while (!CRIS_ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
 802 }
 803
 804 /* We need the atomic mutex for atomic operations where the atomic variable
 805  * breaks the 32 byte cache line since the CRIS architecture does not support
 806  * atomic operations on such variables. Fortunately this should be rare.
 807  */
 808 #  define DEFINE_WITH_MUTEXES
 809 #  define g_atomic_int_exchange_and_add __g_atomic_int_exchange_and_add
 810 #  define g_atomic_int_add __g_atomic_int_add
 811 #  define g_atomic_int_compare_and_exchange __g_atomic_int_compare_and_exchange
 812 #  define g_atomic_pointer_compare_and_exchange __g_atomic_pointer_compare_and_exchange
 813
 814 # else /* !G_ATOMIC_* */
 815 #  define DEFINE_WITH_MUTEXES
 816 # endif /* G_ATOMIC_* */
 817 #else /* !__GNUC__ */
 818 # ifdef G_PLATFORM_WIN32
 819 #  define DEFINE_WITH_WIN32_INTERLOCKED
 820 # else
 821 #  define DEFINE_WITH_MUTEXES
 822 # endif
 823 #endif /* __GNUC__ */
 824
 825 #ifdef DEFINE_WITH_WIN32_INTERLOCKED
 826 # include <windows.h>
 827 /* Following indicates that InterlockedCompareExchangePointer is
 828  * declared in winbase.h (included by windows.h) and needs to be
 829  * commented out if not true. It is defined iff WINVER > 0x0400,
 830  * which is usually correct but can be wrong if WINVER is set before
 831  * windows.h is included.
 832  */
 833 # if WINVER > 0x0400
 834 #  define HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 835 # endif
 836
 837 gint32
 838 g_atomic_int_exchange_and_add (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 839                                gint32           val)
 840 {
 841   return InterlockedExchangeAdd (atomic, val);
 842 }
 843
 844 void
 845 g_atomic_int_add (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 846                   gint32           val)
 847 {
 848   InterlockedExchangeAdd (atomic, val);
 849 }
 850
 851 gboolean
 852 g_atomic_int_compare_and_exchange (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 853                                    gint32           oldval,
 854                                    gint32           newval)
 855 {
 856 #ifndef HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 857   return (guint32) InterlockedCompareExchange ((PVOID*)atomic,
 858                                                (PVOID)newval,
 859                                                (PVOID)oldval) == oldval;
 860 #else
 861   return InterlockedCompareExchange (atomic,
 862                                      newval,
 863                                      oldval) == oldval;
 864 #endif
 865 }
 866
 867 gboolean
 868 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 869                                        gpointer           oldval,
 870                                        gpointer           newval)
 871 {
 872 # ifdef HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 873   return InterlockedCompareExchangePointer (atomic, newval, oldval) == oldval;
 874 # else
 875 #  if GLIB_SIZEOF_VOID_P != 4 /* no 32-bit system */
 876 #   error "InterlockedCompareExchangePointer needed"
 877 #  else
 878    return InterlockedCompareExchange (atomic, newval, oldval) == oldval;
 879 #  endif
 880 # endif
 881 }
 882 #endif /* DEFINE_WITH_WIN32_INTERLOCKED */
 883
 884 #ifdef DEFINE_WITH_MUTEXES
 885 /* We have to use the slow, but safe locking method */
 886 static GMutex *g_atomic_mutex;
 887
 888 /**
 889  * g_atomic_int_exchange_and_add:
 890  * @atomic: a pointer to an integer
 891  * @val: the value to add to *@atomic
 892  *
 893  * Atomically adds @val to the integer pointed to by @atomic.
 894  * It returns the value of *@atomic just before the addition
 895  * took place. Also acts as a memory barrier.
 896  *
 897  * Returns: the value of *@atomic before the addition.
 898  *
 899  * Since: 2.4
 900  */
 901 gint
 902 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 903                                gint           val)
 904 {
 905   gint result;
 906
 907   g_mutex_lock (g_atomic_mutex);
 908   result = *atomic;
 909   *atomic += val;
 910   g_mutex_unlock (g_atomic_mutex);
 911
 912   return result;
 913 }
 914
 915 /**
 916  * g_atomic_int_add:
 917  * @atomic: a pointer to an integer
 918  * @val: the value to add to *@atomic
 919  *
 920  * Atomically adds @val to the integer pointed to by @atomic.
 921  * Also acts as a memory barrier.
 922  *
 923  * Since: 2.4
 924  */
 925 void
 926 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 927                   gint           val)
 928 {
 929   g_mutex_lock (g_atomic_mutex);
 930   *atomic += val;
 931   g_mutex_unlock (g_atomic_mutex);
 932 }
 933
 934 /**
 935  * g_atomic_int_compare_and_exchange:
 936  * @atomic: a pointer to an integer
 937  * @oldval: the assumed old value of *@atomic
 938  * @newval: the new value of *@atomic
 939  *
 940  * Compares @oldval with the integer pointed to by @atomic and
 941  * if they are equal, atomically exchanges *@atomic with @newval.
 942  * Also acts as a memory barrier.
 943  *
 944  * Returns: %TRUE, if *@atomic was equal @oldval. %FALSE otherwise.
 945  *
 946  * Since: 2.4
 947  */
 948 gboolean
 949 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 950                                    gint           oldval,
 951                                    gint           newval)
 952 {
 953   gboolean result;
 954
 955   g_mutex_lock (g_atomic_mutex);
 956   if (*atomic == oldval)
 957     {
 958       result = TRUE;
 959       *atomic = newval;
 960     }
 961   else
 962     result = FALSE;
 963   g_mutex_unlock (g_atomic_mutex);
 964
 965   return result;
 966 }
 967
 968 /**
 969  * g_atomic_pointer_compare_and_exchange:
 970  * @atomic: a pointer to a #gpointer
 971  * @oldval: the assumed old value of *@atomic
 972  * @newval: the new value of *@atomic
 973  *
 974  * Compares @oldval with the pointer pointed to by @atomic and
 975  * if they are equal, atomically exchanges *@atomic with @newval.
 976  * Also acts as a memory barrier.
 977  *
 978  * Returns: %TRUE, if *@atomic was equal @oldval. %FALSE otherwise.
 979  *
 980  * Since: 2.4
 981  */
 982 gboolean
 983 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 984                                        gpointer           oldval,
 985                                        gpointer           newval)
 986 {
 987   gboolean result;
 988
 989   g_mutex_lock (g_atomic_mutex);
 990   if (*atomic == oldval)
 991     {
 992       result = TRUE;
 993       *atomic = newval;
 994     }
 995   else
 996     result = FALSE;
 997   g_mutex_unlock (g_atomic_mutex);
 998
 999   return result;
1000 }
1001
1002 #ifdef G_ATOMIC_OP_MEMORY_BARRIER_NEEDED
1003
1004 /**
1005  * g_atomic_int_get:
1006  * @atomic: a pointer to an integer
1007  *
1008  * Reads the value of the integer pointed to by @atomic.
1009  * Also acts as a memory barrier.
1010  *
1011  * Returns: the value of *@atomic
1012  *
1013  * Since: 2.4
1014  */
1015 gint
1016 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1017 {
1018   gint result;
1019
1020   g_mutex_lock (g_atomic_mutex);
1021   result = *atomic;
1022   g_mutex_unlock (g_atomic_mutex);
1023
1024   return result;
1025 }
1026
1027 /**
1028  * g_atomic_int_set:
1029  * @atomic: a pointer to an integer
1030  * @newval: the new value
1031  *
1032  * Sets the value of the integer pointed to by @atomic.
1033  * Also acts as a memory barrier.
1034  *
1035  * Since: 2.10
1036  */
1037 void
1038 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1039                   gint           newval)
1040 {
1041   g_mutex_lock (g_atomic_mutex);
1042   *atomic = newval;
1043   g_mutex_unlock (g_atomic_mutex);
1044 }
1045
1046 /**
1047  * g_atomic_pointer_get:
1048  * @atomic: a pointer to a #gpointer.
1049  *
1050  * Reads the value of the pointer pointed to by @atomic.
1051  * Also acts as a memory barrier.
1052  *
1053  * Returns: the value to add to *@atomic.
1054  *
1055  * Since: 2.4
1056  */
1057 gpointer
1058 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1059 {
1060   gpointer result;
1061
1062   g_mutex_lock (g_atomic_mutex);
1063   result = *atomic;
1064   g_mutex_unlock (g_atomic_mutex);
1065
1066   return result;
1067 }
1068
1069 /**
1070  * g_atomic_pointer_set:
1071  * @atomic: a pointer to a #gpointer
1072  * @newval: the new value
1073  *
1074  * Sets the value of the pointer pointed to by @atomic.
1075  * Also acts as a memory barrier.
1076  *
1077  * Since: 2.10
1078  */
1079 void
1080 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1081                       gpointer           newval)
1082 {
1083   g_mutex_lock (g_atomic_mutex);
1084   *atomic = newval;
1085   g_mutex_unlock (g_atomic_mutex);
1086 }
1087 #endif /* G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */
1088 #elif defined (G_ATOMIC_OP_MEMORY_BARRIER_NEEDED)
1089 gint
1090 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1091 {
1092   G_ATOMIC_MEMORY_BARRIER;
1093   return *atomic;
1094 }
1095
1096 void
1097 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1098                   gint           newval)
1099 {
1100   *atomic = newval;
1101   G_ATOMIC_MEMORY_BARRIER;
1102 }
1103
1104 gpointer
1105 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1106 {
1107   G_ATOMIC_MEMORY_BARRIER;
1108   return *atomic;
1109 }
1110
1111 void
1112 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1113                       gpointer           newval)
1114 {
1115   *atomic = newval;
1116   G_ATOMIC_MEMORY_BARRIER;
1117 }
1118 #endif /* DEFINE_WITH_MUTEXES || G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */
1119
1120 #ifdef ATOMIC_INT_CMP_XCHG
1121 gboolean
1122 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
1123                                    gint           oldval,
1124                                    gint           newval)
1125 {
1126   return ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
1127 }
1128
1129 gint
1130 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
1131                                gint           val)
1132 {
1133   gint result;
1134   do
1135     result = *atomic;
1136   while (!ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
1137
1138   return result;
1139 }
1140
1141 void
1142 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
1143                   gint           val)
1144 {
1145   gint result;
1146   do
1147     result = *atomic;
1148   while (!ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
1149 }
1150 #endif /* ATOMIC_INT_CMP_XCHG */
1151
1152 void
1153 _g_atomic_thread_init (void)
1154 {
1155 #ifdef DEFINE_WITH_MUTEXES
1156   g_atomic_mutex = g_mutex_new ();
1157 #endif /* DEFINE_WITH_MUTEXES */
1158 }
1159
1160 #ifndef G_ATOMIC_OP_MEMORY_BARRIER_NEEDED
1161 gint
1162 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1163 {
1164   return g_atomic_int_get (atomic);
1165 }
1166
1167 void
1168 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1169                     gint           newval)
1170 {
1171   g_atomic_int_set (atomic, newval);
1172 }
1173
1174 gpointer
1175 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1176 {
1177   return g_atomic_pointer_get (atomic);
1178 }
1179
1180 void
1181 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1182                         gpointer           newval)
1183 {
1184   g_atomic_pointer_set (atomic, newval);
1185 }
1186 #endif /* G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */