glib/gatomic.c

   1 /* GLIB - Library of useful routines for C programming
   2  * Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald
   3  *
   4  * g_atomic_*: atomic operations.
   5  * Copyright (C) 2003 Sebastian Wilhelmi
   6  * Copyright (C) 2007 Nokia Corporation
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the
  20  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  21  * Boston, MA 02111-1307, USA.
  22  */
  23
  24 #include "config.h"
  25
  26 #if defined (G_ATOMIC_ARM)
  27 #include <sched.h>
  28 #endif
  29
  30 #include "glib.h"
  31 #include "gthreadprivate.h"
  32
  33 #if defined (__GNUC__)
  34 # if defined (G_ATOMIC_I486)
  35 /* Adapted from CVS version 1.10 of glibc's sysdeps/i386/i486/bits/atomic.h
  36  */
  37 gint
  38 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
  39                                gint           val)
  40 {
  41   gint result;
  42
  43   __asm__ __volatile__ ("lock; xaddl %0,%1"
  44                         : "=r" (result), "=m" (*atomic)
  45                         : "0" (val), "m" (*atomic));
  46   return result;
  47 }
  48
  49 void
  50 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
  51                   gint           val)
  52 {
  53   __asm__ __volatile__ ("lock; addl %1,%0"
  54                         : "=m" (*atomic)
  55                         : "ir" (val), "m" (*atomic));
  56 }
  57
  58 gboolean
  59 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
  60                                    gint           oldval,
  61                                    gint           newval)
  62 {
  63   gint result;
  64
  65   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
  66                         : "=a" (result), "=m" (*atomic)
  67                         : "r" (newval), "m" (*atomic), "0" (oldval));
  68
  69   return result == oldval;
  70 }
  71
  72 /* The same code as above, as on i386 gpointer is 32 bit as well.
  73  * Duplicating the code here seems more natural than casting the
  74  * arguments and calling the former function */
  75
  76 gboolean
  77 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
  78                                        gpointer           oldval,
  79                                        gpointer           newval)
  80 {
  81   gpointer result;
  82
  83   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
  84                         : "=a" (result), "=m" (*atomic)
  85                         : "r" (newval), "m" (*atomic), "0" (oldval));
  86
  87   return result == oldval;
  88 }
  89
  90 # elif defined (G_ATOMIC_SPARCV9)
  91 /* Adapted from CVS version 1.3 of glibc's sysdeps/sparc/sparc64/bits/atomic.h
  92  */
  93 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
  94   ({                                                                    \
  95      gint __result;                                                     \
  96      __asm__ __volatile__ ("cas [%4], %2, %0"                           \
  97                            : "=r" (__result), "=m" (*(atomic))          \
  98                            : "r" (oldval), "m" (*(atomic)), "r" (atomic),\
  99                            "0" (newval));                               \
 100      __result == oldval;                                                \
 101   })
 102
 103 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 104 gboolean
 105 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 106                                        gpointer           oldval,
 107                                        gpointer           newval)
 108 {
 109   gpointer result;
 110   __asm__ __volatile__ ("cas [%4], %2, %0"
 111                         : "=r" (result), "=m" (*atomic)
 112                         : "r" (oldval), "m" (*atomic), "r" (atomic),
 113                         "0" (newval));
 114   return result == oldval;
 115 }
 116 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 117 gboolean
 118 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 119                                        gpointer           oldval,
 120                                        gpointer           newval)
 121 {
 122   gpointer result;
 123   gpointer *a = atomic;
 124   __asm__ __volatile__ ("casx [%4], %2, %0"
 125                         : "=r" (result), "=m" (*a)
 126                         : "r" (oldval), "m" (*a), "r" (a),
 127                         "0" (newval));
 128   return result == oldval;
 129 }
 130 #  else /* What's that */
 131 #    error "Your system has an unsupported pointer size"
 132 #  endif /* GLIB_SIZEOF_VOID_P */
 133 #  define G_ATOMIC_MEMORY_BARRIER                                       \
 134   __asm__ __volatile__ ("membar #LoadLoad | #LoadStore"                 \
 135                         " | #StoreLoad | #StoreStore" : : : "memory")
 136
 137 # elif defined (G_ATOMIC_ALPHA)
 138 /* Adapted from CVS version 1.3 of glibc's sysdeps/alpha/bits/atomic.h
 139  */
 140 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 141   ({                                                                    \
 142      gint __result;                                                     \
 143      gint __prev;                                                       \
 144      __asm__ __volatile__ (                                             \
 145         "       mb\n"                                                   \
 146         "1:     ldl_l   %0,%2\n"                                        \
 147         "       cmpeq   %0,%3,%1\n"                                     \
 148         "       beq     %1,2f\n"                                        \
 149         "       mov     %4,%1\n"                                        \
 150         "       stl_c   %1,%2\n"                                        \
 151         "       beq     %1,1b\n"                                        \
 152         "       mb\n"                                                   \
 153         "2:"                                                            \
 154         : "=&r" (__prev),                                               \
 155           "=&r" (__result)                                              \
 156         : "m" (*(atomic)),                                              \
 157           "Ir" (oldval),                                                \
 158           "Ir" (newval)                                                 \
 159         : "memory");                                                    \
 160      __result != 0;                                                     \
 161   })
 162 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 163 gboolean
 164 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 165                                        gpointer           oldval,
 166                                        gpointer           newval)
 167 {
 168   gint result;
 169   gpointer prev;
 170   __asm__ __volatile__ (
 171         "       mb\n"
 172         "1:     ldl_l   %0,%2\n"
 173         "       cmpeq   %0,%3,%1\n"
 174         "       beq     %1,2f\n"
 175         "       mov     %4,%1\n"
 176         "       stl_c   %1,%2\n"
 177         "       beq     %1,1b\n"
 178         "       mb\n"
 179         "2:"
 180         : "=&r" (prev),
 181           "=&r" (result)
 182         : "m" (*atomic),
 183           "Ir" (oldval),
 184           "Ir" (newval)
 185         : "memory");
 186   return result != 0;
 187 }
 188 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 189 gboolean
 190 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 191                                        gpointer           oldval,
 192                                        gpointer           newval)
 193 {
 194   gint result;
 195   gpointer prev;
 196   __asm__ __volatile__ (
 197         "       mb\n"
 198         "1:     ldq_l   %0,%2\n"
 199         "       cmpeq   %0,%3,%1\n"
 200         "       beq     %1,2f\n"
 201         "       mov     %4,%1\n"
 202         "       stq_c   %1,%2\n"
 203         "       beq     %1,1b\n"
 204         "       mb\n"
 205         "2:"
 206         : "=&r" (prev),
 207           "=&r" (result)
 208         : "m" (*atomic),
 209           "Ir" (oldval),
 210           "Ir" (newval)
 211         : "memory");
 212   return result != 0;
 213 }
 214 #  else /* What's that */
 215 #   error "Your system has an unsupported pointer size"
 216 #  endif /* GLIB_SIZEOF_VOID_P */
 217 #  define G_ATOMIC_MEMORY_BARRIER  __asm__ ("mb" : : : "memory")
 218 # elif defined (G_ATOMIC_X86_64)
 219 /* Adapted from CVS version 1.9 of glibc's sysdeps/x86_64/bits/atomic.h
 220  */
 221 gint
 222 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 223                                gint           val)
 224 {
 225   gint result;
 226
 227   __asm__ __volatile__ ("lock; xaddl %0,%1"
 228                         : "=r" (result), "=m" (*atomic)
 229                         : "0" (val), "m" (*atomic));
 230   return result;
 231 }
 232
 233 void
 234 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 235                   gint           val)
 236 {
 237   __asm__ __volatile__ ("lock; addl %1,%0"
 238                         : "=m" (*atomic)
 239                         : "ir" (val), "m" (*atomic));
 240 }
 241
 242 gboolean
 243 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 244                                    gint           oldval,
 245                                    gint           newval)
 246 {
 247   gint result;
 248
 249   __asm__ __volatile__ ("lock; cmpxchgl %2, %1"
 250                         : "=a" (result), "=m" (*atomic)
 251                         : "r" (newval), "m" (*atomic), "0" (oldval));
 252
 253   return result == oldval;
 254 }
 255
 256 gboolean
 257 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 258                                        gpointer           oldval,
 259                                        gpointer           newval)
 260 {
 261   gpointer result;
 262
 263   __asm__ __volatile__ ("lock; cmpxchgq %q2, %1"
 264                         : "=a" (result), "=m" (*atomic)
 265                         : "r" (newval), "m" (*atomic), "0" (oldval));
 266
 267   return result == oldval;
 268 }
 269
 270 # elif defined (G_ATOMIC_POWERPC)
 271 /* Adapted from CVS version 1.16 of glibc's sysdeps/powerpc/bits/atomic.h
 272  * and CVS version 1.4 of glibc's sysdeps/powerpc/powerpc32/bits/atomic.h
 273  * and CVS version 1.7 of glibc's sysdeps/powerpc/powerpc64/bits/atomic.h
 274  */
 275 #   ifdef __OPTIMIZE__
 276 /* Non-optimizing compile bails on the following two asm statements
 277  * for reasons unknown to the author */
 278 gint
 279 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 280                                gint           val)
 281 {
 282   gint result, temp;
 283 #if ASM_NUMERIC_LABELS
 284   __asm__ __volatile__ ("1:       lwarx   %0,0,%3\n"
 285                         "         add     %1,%0,%4\n"
 286                         "         stwcx.  %1,0,%3\n"
 287                         "         bne-    1b"
 288                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 289                         : "b" (atomic), "r" (val), "m" (*atomic)
 290                         : "cr0", "memory");
 291 #else
 292   __asm__ __volatile__ (".Lieaa%=:       lwarx   %0,0,%3\n"
 293                         "         add     %1,%0,%4\n"
 294                         "         stwcx.  %1,0,%3\n"
 295                         "         bne-    .Lieaa%="
 296                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 297                         : "b" (atomic), "r" (val), "m" (*atomic)
 298                         : "cr0", "memory");
 299 #endif
 300   return result;
 301 }
 302
 303 /* The same as above, to save a function call repeated here */
 304 void
 305 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 306                   gint           val)
 307 {
 308   gint result, temp;
 309 #if ASM_NUMERIC_LABELS
 310   __asm__ __volatile__ ("1:       lwarx   %0,0,%3\n"
 311                         "         add     %1,%0,%4\n"
 312                         "         stwcx.  %1,0,%3\n"
 313                         "         bne-    1b"
 314                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 315                         : "b" (atomic), "r" (val), "m" (*atomic)
 316                         : "cr0", "memory");
 317 #else
 318   __asm__ __volatile__ (".Lia%=:       lwarx   %0,0,%3\n"
 319                         "         add     %1,%0,%4\n"
 320                         "         stwcx.  %1,0,%3\n"
 321                         "         bne-    .Lia%="
 322                         : "=&b" (result), "=&r" (temp), "=m" (*atomic)
 323                         : "b" (atomic), "r" (val), "m" (*atomic)
 324                         : "cr0", "memory");
 325 #endif
 326 }
 327 #   else /* !__OPTIMIZE__ */
 328 gint
 329 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 330                                gint           val)
 331 {
 332   gint result;
 333   do
 334     result = *atomic;
 335   while (!g_atomic_int_compare_and_exchange (atomic, result, result + val));
 336
 337   return result;
 338 }
 339
 340 void
 341 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 342                   gint           val)
 343 {
 344   gint result;
 345   do
 346     result = *atomic;
 347   while (!g_atomic_int_compare_and_exchange (atomic, result, result + val));
 348 }
 349 #   endif /* !__OPTIMIZE__ */
 350
 351 #   if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 352 gboolean
 353 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 354                                    gint           oldval,
 355                                    gint           newval)
 356 {
 357   gint result;
 358 #if ASM_NUMERIC_LABELS
 359   __asm__ __volatile__ ("sync\n"
 360                         "1: lwarx   %0,0,%1\n"
 361                         "   subf.   %0,%2,%0\n"
 362                         "   bne     2f\n"
 363                         "   stwcx.  %3,0,%1\n"
 364                         "   bne-    1b\n"
 365                         "2: isync"
 366                         : "=&r" (result)
 367                         : "b" (atomic), "r" (oldval), "r" (newval)
 368                         : "cr0", "memory");
 369 #else
 370   __asm__ __volatile__ ("sync\n"
 371                         ".L1icae%=: lwarx   %0,0,%1\n"
 372                         "   subf.   %0,%2,%0\n"
 373                         "   bne     .L2icae%=\n"
 374                         "   stwcx.  %3,0,%1\n"
 375                         "   bne-    .L1icae%=\n"
 376                         ".L2icae%=: isync"
 377                         : "=&r" (result)
 378                         : "b" (atomic), "r" (oldval), "r" (newval)
 379                         : "cr0", "memory");
 380 #endif
 381   return result == 0;
 382 }
 383
 384 gboolean
 385 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 386                                        gpointer           oldval,
 387                                        gpointer           newval)
 388 {
 389   gpointer result;
 390 #if ASM_NUMERIC_LABELS
 391   __asm__ __volatile__ ("sync\n"
 392                         "1: lwarx   %0,0,%1\n"
 393                         "   subf.   %0,%2,%0\n"
 394                         "   bne     2f\n"
 395                         "   stwcx.  %3,0,%1\n"
 396                         "   bne-    1b\n"
 397                         "2: isync"
 398                         : "=&r" (result)
 399                         : "b" (atomic), "r" (oldval), "r" (newval)
 400                         : "cr0", "memory");
 401 #else
 402   __asm__ __volatile__ ("sync\n"
 403                         ".L1pcae%=: lwarx   %0,0,%1\n"
 404                         "   subf.   %0,%2,%0\n"
 405                         "   bne     .L2pcae%=\n"
 406                         "   stwcx.  %3,0,%1\n"
 407                         "   bne-    .L1pcae%=\n"
 408                         ".L2pcae%=: isync"
 409                         : "=&r" (result)
 410                         : "b" (atomic), "r" (oldval), "r" (newval)
 411                         : "cr0", "memory");
 412 #endif
 413   return result == 0;
 414 }
 415 #   elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 416 gboolean
 417 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 418                                    gint           oldval,
 419                                    gint           newval)
 420 {
 421   gpointer result;
 422 #if ASM_NUMERIC_LABELS
 423   __asm__ __volatile__ ("sync\n"
 424                         "1: lwarx   %0,0,%1\n"
 425                         "   extsw   %0,%0\n"
 426                         "   subf.   %0,%2,%0\n"
 427                         "   bne     2f\n"
 428                         "   stwcx.  %3,0,%1\n"
 429                         "   bne-    1b\n"
 430                         "2: isync"
 431                         : "=&r" (result)
 432                         : "b" (atomic), "r" (oldval), "r" (newval)
 433                         : "cr0", "memory");
 434 #else
 435   __asm__ __volatile__ ("sync\n"
 436                         ".L1icae%=: lwarx   %0,0,%1\n"
 437                         "   extsw   %0,%0\n"
 438                         "   subf.   %0,%2,%0\n"
 439                         "   bne     .L2icae%=\n"
 440                         "   stwcx.  %3,0,%1\n"
 441                         "   bne-    .L1icae%=\n"
 442                         ".L2icae%=: isync"
 443                         : "=&r" (result)
 444                         : "b" (atomic), "r" (oldval), "r" (newval)
 445                         : "cr0", "memory");
 446 #endif
 447   return result == 0;
 448 }
 449
 450 gboolean
 451 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 452                                        gpointer           oldval,
 453                                        gpointer           newval)
 454 {
 455   gpointer result;
 456 #if ASM_NUMERIC_LABELS
 457   __asm__ __volatile__ ("sync\n"
 458                         "1: ldarx   %0,0,%1\n"
 459                         "   subf.   %0,%2,%0\n"
 460                         "   bne     2f\n"
 461                         "   stdcx.  %3,0,%1\n"
 462                         "   bne-    1b\n"
 463                         "2: isync"
 464                         : "=&r" (result)
 465                         : "b" (atomic), "r" (oldval), "r" (newval)
 466                         : "cr0", "memory");
 467 #else
 468   __asm__ __volatile__ ("sync\n"
 469                         ".L1pcae%=: ldarx   %0,0,%1\n"
 470                         "   subf.   %0,%2,%0\n"
 471                         "   bne     .L2pcae%=\n"
 472                         "   stdcx.  %3,0,%1\n"
 473                         "   bne-    .L1pcae%=\n"
 474                         ".L2pcae%=: isync"
 475                         : "=&r" (result)
 476                         : "b" (atomic), "r" (oldval), "r" (newval)
 477                         : "cr0", "memory");
 478 #endif
 479   return result == 0;
 480 }
 481 #  else /* What's that */
 482 #   error "Your system has an unsupported pointer size"
 483 #  endif /* GLIB_SIZEOF_VOID_P */
 484
 485 #  define G_ATOMIC_MEMORY_BARRIER __asm__ ("sync" : : : "memory")
 486
 487 # elif defined (G_ATOMIC_IA64)
 488 /* Adapted from CVS version 1.8 of glibc's sysdeps/ia64/bits/atomic.h
 489  */
 490 gint
 491 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 492                                gint           val)
 493 {
 494   return __sync_fetch_and_add (atomic, val);
 495 }
 496
 497 void
 498 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 499                   gint val)
 500 {
 501   __sync_fetch_and_add (atomic, val);
 502 }
 503
 504 gboolean
 505 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 506                                    gint           oldval,
 507                                    gint           newval)
 508 {
 509   return __sync_bool_compare_and_swap (atomic, oldval, newval);
 510 }
 511
 512 gboolean
 513 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 514                                        gpointer           oldval,
 515                                        gpointer           newval)
 516 {
 517   return __sync_bool_compare_and_swap ((long *)atomic,
 518                                        (long)oldval, (long)newval);
 519 }
 520
 521 #  define G_ATOMIC_MEMORY_BARRIER __sync_synchronize ()
 522 # elif defined (G_ATOMIC_S390)
 523 /* Adapted from glibc's sysdeps/s390/bits/atomic.h
 524  */
 525 #  define ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)                   \
 526   ({                                                                    \
 527      gint __result = oldval;                                    \
 528      __asm__ __volatile__ ("cs %0, %2, %1"                              \
 529                            : "+d" (__result), "=Q" (*(atomic))          \
 530                            : "d" (newval), "m" (*(atomic)) : "cc" );    \
 531      __result == oldval;                                                \
 532   })
 533
 534 #  if GLIB_SIZEOF_VOID_P == 4 /* 32-bit system */
 535 gboolean
 536 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 537                                        gpointer           oldval,
 538                                        gpointer           newval)
 539 {
 540   gpointer result = oldval;
 541   __asm__ __volatile__ ("cs %0, %2, %1"
 542                         : "+d" (result), "=Q" (*(atomic))
 543                         : "d" (newval), "m" (*(atomic)) : "cc" );
 544   return result == oldval;
 545 }
 546 #  elif GLIB_SIZEOF_VOID_P == 8 /* 64-bit system */
 547 gboolean
 548 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 549                                        gpointer           oldval,
 550                                        gpointer           newval)
 551 {
 552   gpointer result = oldval;
 553   gpointer *a = atomic;
 554   __asm__ __volatile__ ("csg %0, %2, %1"
 555                         : "+d" (result), "=Q" (*a)
 556                         : "d" ((long)(newval)), "m" (*a) : "cc" );
 557   return result == oldval;
 558 }
 559 #  else /* What's that */
 560 #    error "Your system has an unsupported pointer size"
 561 #  endif /* GLIB_SIZEOF_VOID_P */
 562 # elif defined (G_ATOMIC_ARM)
 563 static volatile int atomic_spin = 0;
 564
 565 static int atomic_spin_trylock (void)
 566 {
 567   int result;
 568
 569   asm volatile (
 570     "swp %0, %1, [%2]\n"
 571     : "=&r,&r" (result)
 572     : "r,0" (1), "r,r" (&atomic_spin)
 573     : "memory");
 574   if (result == 0)
 575     return 0;
 576   else
 577     return -1;
 578 }
 579
 580 static void atomic_spin_lock (void)
 581 {
 582   while (atomic_spin_trylock())
 583     sched_yield();
 584 }
 585
 586 static void atomic_spin_unlock (void)
 587 {
 588   atomic_spin = 0;
 589 }
 590
 591 gint
 592 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 593                                gint           val)
 594 {
 595   gint result;
 596
 597   atomic_spin_lock();
 598   result = *atomic;
 599   *atomic += val;
 600   atomic_spin_unlock();
 601
 602   return result;
 603 }
 604
 605 void
 606 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 607                   gint           val)
 608 {
 609   atomic_spin_lock();
 610   *atomic += val;
 611   atomic_spin_unlock();
 612 }
 613
 614 gboolean
 615 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 616                                    gint           oldval,
 617                                    gint           newval)
 618 {
 619   gboolean result;
 620
 621   atomic_spin_lock();
 622   if (*atomic == oldval)
 623     {
 624       result = TRUE;
 625       *atomic = newval;
 626     }
 627   else
 628     result = FALSE;
 629   atomic_spin_unlock();
 630
 631   return result;
 632 }
 633
 634 gboolean
 635 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 636                                        gpointer           oldval,
 637                                        gpointer           newval)
 638 {
 639   gboolean result;
 640
 641   atomic_spin_lock();
 642   if (*atomic == oldval)
 643     {
 644       result = TRUE;
 645       *atomic = newval;
 646     }
 647   else
 648     result = FALSE;
 649   atomic_spin_unlock();
 650
 651   return result;
 652 }
 653 # elif defined (G_ATOMIC_CRIS) || defined (G_ATOMIC_CRISV32)
 654 #  ifdef G_ATOMIC_CRIS
 655 #   define CRIS_ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)             \
 656   ({                                                                    \
 657      gboolean __result;                                                 \
 658      __asm__ __volatile__ ("\n"                                         \
 659                            "0:\tclearf\n\t"                             \
 660                            "cmp.d [%[Atomic]], %[OldVal]\n\t"           \
 661                            "bne 1f\n\t"                                 \
 662                            "ax\n\t"                                     \
 663                            "move.d %[NewVal], [%[Atomic]]\n\t"          \
 664                            "bwf 0b\n"                                   \
 665                            "1:\tseq %[Result]"                          \
 666                            : [Result] "=&r" (__result),                 \
 667                                       "=m" (*(atomic))                  \
 668                            : [Atomic] "r" (atomic),                     \
 669                              [OldVal] "r" (oldval),                     \
 670                              [NewVal] "r" (newval),                     \
 671                                       "g" (*(gpointer*) (atomic))       \
 672                            : "memory");                                 \
 673      __result;                                                          \
 674   })
 675 #  else
 676 #   define CRIS_ATOMIC_INT_CMP_XCHG(atomic, oldval, newval)             \
 677   ({                                                                    \
 678      gboolean __result;                                                 \
 679      __asm__ __volatile__ ("\n"                                         \
 680                            "0:\tclearf p\n\t"                           \
 681                            "cmp.d [%[Atomic]], %[OldVal]\n\t"           \
 682                            "bne 1f\n\t"                                 \
 683                            "ax\n\t"                                     \
 684                            "move.d %[NewVal], [%[Atomic]]\n\t"          \
 685                            "bcs 0b\n"                                   \
 686                            "1:\tseq %[Result]"                          \
 687                            : [Result] "=&r" (__result),                 \
 688                                       "=m" (*(atomic))                  \
 689                            : [Atomic] "r" (atomic),                     \
 690                              [OldVal] "r" (oldval),                     \
 691                              [NewVal] "r" (newval),                     \
 692                                       "g" (*(gpointer*) (atomic))       \
 693                            : "memory");                                 \
 694      __result;                                                          \
 695   })
 696 #  endif
 697
 698 #define CRIS_CACHELINE_SIZE 32
 699 #define CRIS_ATOMIC_BREAKS_CACHELINE(atomic) \
 700   (((gulong)(atomic) & (CRIS_CACHELINE_SIZE - 1)) > (CRIS_CACHELINE_SIZE - sizeof (atomic)))
 701
 702 gint     __g_atomic_int_exchange_and_add         (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 703                                                   gint             val);
 704 void     __g_atomic_int_add                      (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 705                                                   gint             val);
 706 gboolean __g_atomic_int_compare_and_exchange     (volatile gint   G_GNUC_MAY_ALIAS *atomic,
 707                                                   gint             oldval,
 708                                                   gint             newval);
 709 gboolean __g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 710                                                   gpointer         oldval,
 711                                                   gpointer         newval);
 712
 713 gboolean
 714 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 715                                        gpointer           oldval,
 716                                        gpointer           newval)
 717 {
 718   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 719     return __g_atomic_pointer_compare_and_exchange (atomic, oldval, newval);
 720
 721   return CRIS_ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 722 }
 723
 724 gboolean
 725 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 726                                    gint           oldval,
 727                                    gint           newval)
 728 {
 729   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 730     return __g_atomic_int_compare_and_exchange (atomic, oldval, newval);
 731
 732   return CRIS_ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 733 }
 734
 735 gint
 736 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 737                                gint           val)
 738 {
 739   gint result;
 740
 741   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 742     return __g_atomic_int_exchange_and_add (atomic, val);
 743
 744   do
 745     result = *atomic;
 746   while (!CRIS_ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
 747
 748   return result;
 749 }
 750
 751 void
 752 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 753                   gint           val)
 754 {
 755   gint result;
 756
 757   if (G_UNLIKELY (CRIS_ATOMIC_BREAKS_CACHELINE (atomic)))
 758     return __g_atomic_int_add (atomic, val);
 759
 760   do
 761     result = *atomic;
 762   while (!CRIS_ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
 763 }
 764
 765 /* We need the atomic mutex for atomic operations where the atomic variable
 766  * breaks the 32 byte cache line since the CRIS architecture does not support
 767  * atomic operations on such variables. Fortunately this should be rare.
 768  */
 769 #  define DEFINE_WITH_MUTEXES
 770 #  define g_atomic_int_exchange_and_add __g_atomic_int_exchange_and_add
 771 #  define g_atomic_int_add __g_atomic_int_add
 772 #  define g_atomic_int_compare_and_exchange __g_atomic_int_compare_and_exchange
 773 #  define g_atomic_pointer_compare_and_exchange __g_atomic_pointer_compare_and_exchange
 774
 775 # else /* !G_ATOMIC_* */
 776 #  define DEFINE_WITH_MUTEXES
 777 # endif /* G_ATOMIC_* */
 778 #else /* !__GNUC__ */
 779 # ifdef G_PLATFORM_WIN32
 780 #  define DEFINE_WITH_WIN32_INTERLOCKED
 781 # else
 782 #  define DEFINE_WITH_MUTEXES
 783 # endif
 784 #endif /* __GNUC__ */
 785
 786 #ifdef DEFINE_WITH_WIN32_INTERLOCKED
 787 # include <windows.h>
 788 /* Following indicates that InterlockedCompareExchangePointer is
 789  * declared in winbase.h (included by windows.h) and needs to be
 790  * commented out if not true. It is defined iff WINVER > 0x0400,
 791  * which is usually correct but can be wrong if WINVER is set before
 792  * windows.h is included.
 793  */
 794 # if WINVER > 0x0400
 795 #  define HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 796 # endif
 797
 798 gint32
 799 g_atomic_int_exchange_and_add (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 800                                gint32           val)
 801 {
 802   return InterlockedExchangeAdd (atomic, val);
 803 }
 804
 805 void
 806 g_atomic_int_add (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 807                   gint32           val)
 808 {
 809   InterlockedExchangeAdd (atomic, val);
 810 }
 811
 812 gboolean
 813 g_atomic_int_compare_and_exchange (volatile gint32 G_GNUC_MAY_ALIAS *atomic,
 814                                    gint32           oldval,
 815                                    gint32           newval)
 816 {
 817 #ifndef HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 818   return (guint32) InterlockedCompareExchange ((PVOID*)atomic,
 819                                                (PVOID)newval,
 820                                                (PVOID)oldval) == oldval;
 821 #else
 822   return InterlockedCompareExchange (atomic,
 823                                      newval,
 824                                      oldval) == oldval;
 825 #endif
 826 }
 827
 828 gboolean
 829 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 830                                        gpointer           oldval,
 831                                        gpointer           newval)
 832 {
 833 # ifdef HAVE_INTERLOCKED_COMPARE_EXCHANGE_POINTER
 834   return InterlockedCompareExchangePointer (atomic, newval, oldval) == oldval;
 835 # else
 836 #  if GLIB_SIZEOF_VOID_P != 4 /* no 32-bit system */
 837 #   error "InterlockedCompareExchangePointer needed"
 838 #  else
 839    return InterlockedCompareExchange (atomic, newval, oldval) == oldval;
 840 #  endif
 841 # endif
 842 }
 843 #endif /* DEFINE_WITH_WIN32_INTERLOCKED */
 844
 845 #ifdef DEFINE_WITH_MUTEXES
 846 /* We have to use the slow, but safe locking method */
 847 static GMutex *g_atomic_mutex;
 848
 849 gint
 850 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 851                                gint           val)
 852 {
 853   gint result;
 854
 855   g_mutex_lock (g_atomic_mutex);
 856   result = *atomic;
 857   *atomic += val;
 858   g_mutex_unlock (g_atomic_mutex);
 859
 860   return result;
 861 }
 862
 863
 864 void
 865 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 866                   gint           val)
 867 {
 868   g_mutex_lock (g_atomic_mutex);
 869   *atomic += val;
 870   g_mutex_unlock (g_atomic_mutex);
 871 }
 872
 873 gboolean
 874 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 875                                    gint           oldval,
 876                                    gint           newval)
 877 {
 878   gboolean result;
 879
 880   g_mutex_lock (g_atomic_mutex);
 881   if (*atomic == oldval)
 882     {
 883       result = TRUE;
 884       *atomic = newval;
 885     }
 886   else
 887     result = FALSE;
 888   g_mutex_unlock (g_atomic_mutex);
 889
 890   return result;
 891 }
 892
 893 gboolean
 894 g_atomic_pointer_compare_and_exchange (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 895                                        gpointer           oldval,
 896                                        gpointer           newval)
 897 {
 898   gboolean result;
 899
 900   g_mutex_lock (g_atomic_mutex);
 901   if (*atomic == oldval)
 902     {
 903       result = TRUE;
 904       *atomic = newval;
 905     }
 906   else
 907     result = FALSE;
 908   g_mutex_unlock (g_atomic_mutex);
 909
 910   return result;
 911 }
 912
 913 #ifdef G_ATOMIC_OP_MEMORY_BARRIER_NEEDED
 914 gint
 915 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
 916 {
 917   gint result;
 918
 919   g_mutex_lock (g_atomic_mutex);
 920   result = *atomic;
 921   g_mutex_unlock (g_atomic_mutex);
 922
 923   return result;
 924 }
 925
 926 void
 927 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
 928                   gint           newval)
 929 {
 930   g_mutex_lock (g_atomic_mutex);
 931   *atomic = newval;
 932   g_mutex_unlock (g_atomic_mutex);
 933 }
 934
 935 gpointer
 936 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
 937 {
 938   gpointer result;
 939
 940   g_mutex_lock (g_atomic_mutex);
 941   result = *atomic;
 942   g_mutex_unlock (g_atomic_mutex);
 943
 944   return result;
 945 }
 946
 947 void
 948 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 949                       gpointer           newval)
 950 {
 951   g_mutex_lock (g_atomic_mutex);
 952   *atomic = newval;
 953   g_mutex_unlock (g_atomic_mutex);
 954 }
 955 #endif /* G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */
 956 #elif defined (G_ATOMIC_OP_MEMORY_BARRIER_NEEDED)
 957 gint
 958 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
 959 {
 960   G_ATOMIC_MEMORY_BARRIER;
 961   return *atomic;
 962 }
 963
 964 void
 965 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
 966                   gint           newval)
 967 {
 968   *atomic = newval;
 969   G_ATOMIC_MEMORY_BARRIER;
 970 }
 971
 972 gpointer
 973 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
 974 {
 975   G_ATOMIC_MEMORY_BARRIER;
 976   return *atomic;
 977 }
 978
 979 void
 980 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
 981                       gpointer           newval)
 982 {
 983   *atomic = newval;
 984   G_ATOMIC_MEMORY_BARRIER;
 985 }
 986 #endif /* DEFINE_WITH_MUTEXES || G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */
 987
 988 #ifdef ATOMIC_INT_CMP_XCHG
 989 gboolean
 990 g_atomic_int_compare_and_exchange (volatile gint G_GNUC_MAY_ALIAS *atomic,
 991                                    gint           oldval,
 992                                    gint           newval)
 993 {
 994   return ATOMIC_INT_CMP_XCHG (atomic, oldval, newval);
 995 }
 996
 997 gint
 998 g_atomic_int_exchange_and_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
 999                                gint           val)
1000 {
1001   gint result;
1002   do
1003     result = *atomic;
1004   while (!ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
1005
1006   return result;
1007 }
1008
1009 void
1010 g_atomic_int_add (volatile gint G_GNUC_MAY_ALIAS *atomic,
1011                   gint           val)
1012 {
1013   gint result;
1014   do
1015     result = *atomic;
1016   while (!ATOMIC_INT_CMP_XCHG (atomic, result, result + val));
1017 }
1018 #endif /* ATOMIC_INT_CMP_XCHG */
1019
1020 void
1021 _g_atomic_thread_init (void)
1022 {
1023 #ifdef DEFINE_WITH_MUTEXES
1024   g_atomic_mutex = g_mutex_new ();
1025 #endif /* DEFINE_WITH_MUTEXES */
1026 }
1027
1028 #ifndef G_ATOMIC_OP_MEMORY_BARRIER_NEEDED
1029 gint
1030 (g_atomic_int_get) (volatile gint G_GNUC_MAY_ALIAS *atomic)
1031 {
1032   return g_atomic_int_get (atomic);
1033 }
1034
1035 void
1036 (g_atomic_int_set) (volatile gint G_GNUC_MAY_ALIAS *atomic,
1037                     gint           newval)
1038 {
1039   g_atomic_int_set (atomic, newval);
1040 }
1041
1042 gpointer
1043 (g_atomic_pointer_get) (volatile gpointer G_GNUC_MAY_ALIAS *atomic)
1044 {
1045   return g_atomic_pointer_get (atomic);
1046 }
1047
1048 void
1049 (g_atomic_pointer_set) (volatile gpointer G_GNUC_MAY_ALIAS *atomic,
1050                         gpointer           newval)
1051 {
1052   g_atomic_pointer_set (atomic, newval);
1053 }
1054 #endif /* G_ATOMIC_OP_MEMORY_BARRIER_NEEDED */