nptl/pthread_mutex_timedlock.c

   1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <time.h>
  22 #include <sys/param.h>
  23 #include "pthreadP.h"
  24 #include <lowlevellock.h>
  25 #include <not-cancel.h>
  26
  27 #include <stap-probe.h>
  28
  29 #ifndef lll_timedlock_elision
  30 #define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
  31 #endif
  32
  33 #ifndef lll_trylock_elision
  34 #define lll_trylock_elision(a,t) lll_trylock(a)
  35 #endif
  36
  37 #ifndef FORCE_ELISION
  38 #define FORCE_ELISION(m, s)
  39 #endif
  40
  41 int
  42 pthread_mutex_timedlock (mutex, abstime)
  43      pthread_mutex_t *mutex;
  44      const struct timespec *abstime;
  45 {
  46   int oldval;
  47   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  48   int result = 0;
  49
  50   LIBC_PROBE (mutex_timedlock_entry, 2, mutex, abstime);
  51
  52   /* We must not check ABSTIME here.  If the thread does not block
  53      abstime must not be checked for a valid value.  */
  54
  55   switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
  56                             PTHREAD_MUTEX_TIMED_NP))
  57     {
  58       /* Recursive mutex.  */
  59     case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
  60     case PTHREAD_MUTEX_RECURSIVE_NP:
  61       /* Check whether we already hold the mutex.  */
  62       if (mutex->__data.__owner == id)
  63         {
  64           /* Just bump the counter.  */
  65           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
  66             /* Overflow of the counter.  */
  67             return EAGAIN;
  68
  69           ++mutex->__data.__count;
  70
  71           goto out;
  72         }
  73
  74       /* We have to get the mutex.  */
  75       result = lll_timedlock (mutex->__data.__lock, abstime,
  76                               PTHREAD_MUTEX_PSHARED (mutex));
  77
  78       if (result != 0)
  79         goto out;
  80
  81       /* Only locked once so far.  */
  82       mutex->__data.__count = 1;
  83       break;
  84
  85       /* Error checking mutex.  */
  86     case PTHREAD_MUTEX_ERRORCHECK_NP:
  87       /* Check whether we already hold the mutex.  */
  88       if (__glibc_unlikely (mutex->__data.__owner == id))
  89         return EDEADLK;
  90
  91       /* FALLTHROUGH */
  92
  93     case PTHREAD_MUTEX_TIMED_NP:
  94       FORCE_ELISION (mutex, goto elision);
  95     simple:
  96       /* Normal mutex.  */
  97       result = lll_timedlock (mutex->__data.__lock, abstime,
  98                               PTHREAD_MUTEX_PSHARED (mutex));
  99       break;
 100
 101     case PTHREAD_MUTEX_TIMED_ELISION_NP:
 102     elision: __attribute__((unused))
 103       /* Don't record ownership */
 104       return lll_timedlock_elision (mutex->__data.__lock,
 105                                     mutex->__data.__spins,
 106                                     abstime,
 107                                     PTHREAD_MUTEX_PSHARED (mutex));
 108
 109
 110     case PTHREAD_MUTEX_ADAPTIVE_NP:
 111       if (! __is_smp)
 112         goto simple;
 113
 114       if (lll_trylock (mutex->__data.__lock) != 0)
 115         {
 116           int cnt = 0;
 117           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
 118                              mutex->__data.__spins * 2 + 10);
 119           do
 120             {
 121               if (cnt++ >= max_cnt)
 122                 {
 123                   result = lll_timedlock (mutex->__data.__lock, abstime,
 124                                           PTHREAD_MUTEX_PSHARED (mutex));
 125                   break;
 126                 }
 127
 128 #ifdef BUSY_WAIT_NOP
 129               BUSY_WAIT_NOP;
 130 #endif
 131             }
 132           while (lll_trylock (mutex->__data.__lock) != 0);
 133
 134           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 135         }
 136       break;
 137
 138     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 139     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 140     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 141     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 142       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 143                      &mutex->__data.__list.__next);
 144
 145       oldval = mutex->__data.__lock;
 146       do
 147         {
 148         again:
 149           if ((oldval & FUTEX_OWNER_DIED) != 0)
 150             {
 151               /* The previous owner died.  Try locking the mutex.  */
 152               int newval = id | (oldval & FUTEX_WAITERS);
 153
 154               newval
 155                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 156                                                        newval, oldval);
 157               if (newval != oldval)
 158                 {
 159                   oldval = newval;
 160                   goto again;
 161                 }
 162
 163               /* We got the mutex.  */
 164               mutex->__data.__count = 1;
 165               /* But it is inconsistent unless marked otherwise.  */
 166               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 167
 168               ENQUEUE_MUTEX (mutex);
 169               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 170
 171               /* Note that we deliberately exit here.  If we fall
 172                  through to the end of the function __nusers would be
 173                  incremented which is not correct because the old
 174                  owner has to be discounted.  */
 175               return EOWNERDEAD;
 176             }
 177
 178           /* Check whether we already hold the mutex.  */
 179           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 180             {
 181               int kind = PTHREAD_MUTEX_TYPE (mutex);
 182               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 183                 {
 184                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 185                                  NULL);
 186                   return EDEADLK;
 187                 }
 188
 189               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 190                 {
 191                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 192                                  NULL);
 193
 194                   /* Just bump the counter.  */
 195                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 196                     /* Overflow of the counter.  */
 197                     return EAGAIN;
 198
 199                   ++mutex->__data.__count;
 200
 201                   LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 202
 203                   return 0;
 204                 }
 205             }
 206
 207           result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
 208                                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 209
 210           if (__builtin_expect (mutex->__data.__owner
 211                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 212             {
 213               /* This mutex is now not recoverable.  */
 214               mutex->__data.__count = 0;
 215               lll_unlock (mutex->__data.__lock,
 216                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 217               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 218               return ENOTRECOVERABLE;
 219             }
 220
 221           if (result == ETIMEDOUT || result == EINVAL)
 222             goto out;
 223
 224           oldval = result;
 225         }
 226       while ((oldval & FUTEX_OWNER_DIED) != 0);
 227
 228       mutex->__data.__count = 1;
 229       ENQUEUE_MUTEX (mutex);
 230       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 231       break;
 232
 233     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 234     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 235     case PTHREAD_MUTEX_PI_NORMAL_NP:
 236     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 237     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 238     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 239     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 240     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 241       {
 242         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 243         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 244
 245         if (robust)
 246           /* Note: robust PI futexes are signaled by setting bit 0.  */
 247           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 248                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 249                                    | 1));
 250
 251         oldval = mutex->__data.__lock;
 252
 253         /* Check whether we already hold the mutex.  */
 254         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 255           {
 256             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 257               {
 258                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 259                 return EDEADLK;
 260               }
 261
 262             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 263               {
 264                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 265
 266                 /* Just bump the counter.  */
 267                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 268                   /* Overflow of the counter.  */
 269                   return EAGAIN;
 270
 271                 ++mutex->__data.__count;
 272
 273                 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 274
 275                 return 0;
 276               }
 277           }
 278
 279         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 280                                                       id, 0);
 281
 282         if (oldval != 0)
 283           {
 284             /* The mutex is locked.  The kernel will now take care of
 285                everything.  The timeout value must be a relative value.
 286                Convert it.  */
 287             int private = (robust
 288                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 289                            : PTHREAD_MUTEX_PSHARED (mutex));
 290             INTERNAL_SYSCALL_DECL (__err);
 291
 292             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 293                                       __lll_private_flag (FUTEX_LOCK_PI,
 294                                                           private), 1,
 295                                       abstime);
 296             if (INTERNAL_SYSCALL_ERROR_P (e, __err))
 297               {
 298                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
 299                   return ETIMEDOUT;
 300
 301                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 302                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
 303                   {
 304                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 305                             || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 306                                 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 307                     /* ESRCH can happen only for non-robust PI mutexes where
 308                        the owner of the lock died.  */
 309                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
 310                             || !robust);
 311
 312                     /* Delay the thread until the timeout is reached.
 313                        Then return ETIMEDOUT.  */
 314                     struct timespec reltime;
 315                     struct timespec now;
 316
 317                     INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
 318                                       &now);
 319                     reltime.tv_sec = abstime->tv_sec - now.tv_sec;
 320                     reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
 321                     if (reltime.tv_nsec < 0)
 322                       {
 323                         reltime.tv_nsec += 1000000000;
 324                         --reltime.tv_sec;
 325                       }
 326                     if (reltime.tv_sec >= 0)
 327                       while (nanosleep_not_cancel (&reltime, &reltime) != 0)
 328                         continue;
 329
 330                     return ETIMEDOUT;
 331                   }
 332
 333                 return INTERNAL_SYSCALL_ERRNO (e, __err);
 334               }
 335
 336             oldval = mutex->__data.__lock;
 337
 338             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 339           }
 340
 341         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 342           {
 343             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 344
 345             /* We got the mutex.  */
 346             mutex->__data.__count = 1;
 347             /* But it is inconsistent unless marked otherwise.  */
 348             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 349
 350             ENQUEUE_MUTEX_PI (mutex);
 351             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 352
 353             /* Note that we deliberately exit here.  If we fall
 354                through to the end of the function __nusers would be
 355                incremented which is not correct because the old owner
 356                has to be discounted.  */
 357             return EOWNERDEAD;
 358           }
 359
 360         if (robust
 361             && __builtin_expect (mutex->__data.__owner
 362                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 363           {
 364             /* This mutex is now not recoverable.  */
 365             mutex->__data.__count = 0;
 366
 367             INTERNAL_SYSCALL_DECL (__err);
 368             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 369                               __lll_private_flag (FUTEX_UNLOCK_PI,
 370                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 371                               0, 0);
 372
 373             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 374             return ENOTRECOVERABLE;
 375           }
 376
 377         mutex->__data.__count = 1;
 378         if (robust)
 379           {
 380             ENQUEUE_MUTEX_PI (mutex);
 381             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 382           }
 383         }
 384       break;
 385
 386     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 387     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 388     case PTHREAD_MUTEX_PP_NORMAL_NP:
 389     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 390       {
 391         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 392
 393         oldval = mutex->__data.__lock;
 394
 395         /* Check whether we already hold the mutex.  */
 396         if (mutex->__data.__owner == id)
 397           {
 398             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 399               return EDEADLK;
 400
 401             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 402               {
 403                 /* Just bump the counter.  */
 404                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 405                   /* Overflow of the counter.  */
 406                   return EAGAIN;
 407
 408                 ++mutex->__data.__count;
 409
 410                 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 411
 412                 return 0;
 413               }
 414           }
 415
 416         int oldprio = -1, ceilval;
 417         do
 418           {
 419             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 420                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 421
 422             if (__pthread_current_priority () > ceiling)
 423               {
 424                 result = EINVAL;
 425               failpp:
 426                 if (oldprio != -1)
 427                   __pthread_tpp_change_priority (oldprio, -1);
 428                 return result;
 429               }
 430
 431             result = __pthread_tpp_change_priority (oldprio, ceiling);
 432             if (result)
 433               return result;
 434
 435             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 436             oldprio = ceiling;
 437
 438             oldval
 439               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 440                                                      ceilval | 1, ceilval);
 441
 442             if (oldval == ceilval)
 443               break;
 444
 445             do
 446               {
 447                 oldval
 448                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 449                                                          ceilval | 2,
 450                                                          ceilval | 1);
 451
 452                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 453                   break;
 454
 455                 if (oldval != ceilval)
 456                   {
 457                     /* Reject invalid timeouts.  */
 458                     if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
 459                       {
 460                         result = EINVAL;
 461                         goto failpp;
 462                       }
 463
 464                     struct timeval tv;
 465                     struct timespec rt;
 466
 467                     /* Get the current time.  */
 468                     (void) __gettimeofday (&tv, NULL);
 469
 470                     /* Compute relative timeout.  */
 471                     rt.tv_sec = abstime->tv_sec - tv.tv_sec;
 472                     rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
 473                     if (rt.tv_nsec < 0)
 474                       {
 475                         rt.tv_nsec += 1000000000;
 476                         --rt.tv_sec;
 477                       }
 478
 479                     /* Already timed out?  */
 480                     if (rt.tv_sec < 0)
 481                       {
 482                         result = ETIMEDOUT;
 483                         goto failpp;
 484                       }
 485
 486                     lll_futex_timed_wait (&mutex->__data.__lock,
 487                                           ceilval | 2, &rt,
 488                                           PTHREAD_MUTEX_PSHARED (mutex));
 489                   }
 490               }
 491             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 492                                                         ceilval | 2, ceilval)
 493                    != ceilval);
 494           }
 495         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 496
 497         assert (mutex->__data.__owner == 0);
 498         mutex->__data.__count = 1;
 499       }
 500       break;
 501
 502     default:
 503       /* Correct code cannot set any other type.  */
 504       return EINVAL;
 505     }
 506
 507   if (result == 0)
 508     {
 509       /* Record the ownership.  */
 510       mutex->__data.__owner = id;
 511       ++mutex->__data.__nusers;
 512
 513       LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 514     }
 515
 516  out:
 517   return result;
 518 }