lib/lockref.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/lockref.h>
   4
   5 #if USE_CMPXCHG_LOCKREF
   6
   7 /*
   8  * Note that the "cmpxchg()" reloads the "old" value for the
   9  * failure case.
  10  */
  11 #define CMPXCHG_LOOP(CODE, SUCCESS) do {                                        \
  12         int retry = 100;                                                        \
  13         struct lockref old;                                                     \
  14         BUILD_BUG_ON(sizeof(old) != 8);                                         \
  15         old.lock_count = READ_ONCE(lockref->lock_count);                        \
  16         while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {     \
  17                 struct lockref new = old, prev = old;                           \
  18                 CODE                                                            \
  19                 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,        \
  20                                                    old.lock_count,              \
  21                                                    new.lock_count);             \
  22                 if (likely(old.lock_count == prev.lock_count)) {                \
  23                         SUCCESS;                                                \
  24                 }                                                               \
  25                 if (!--retry)                                                   \
  26                         break;                                                  \
  27         }                                                                       \
  28 } while (0)
  29
  30 #else
  31
  32 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
  33
  34 #endif
  35
  36 /**
  37  * lockref_get - Increments reference count unconditionally
  38  * @lockref: pointer to lockref structure
  39  *
  40  * This operation is only valid if you already hold a reference
  41  * to the object, so you know the count cannot be zero.
  42  */
  43 void lockref_get(struct lockref *lockref)
  44 {
  45         CMPXCHG_LOOP(
  46                 new.count++;
  47         ,
  48                 return;
  49         );
  50
  51         spin_lock(&lockref->lock);
  52         lockref->count++;
  53         spin_unlock(&lockref->lock);
  54 }
  55 EXPORT_SYMBOL(lockref_get);
  56
  57 /**
  58  * lockref_get_not_zero - Increments count unless the count is 0 or dead
  59  * @lockref: pointer to lockref structure
  60  * Return: 1 if count updated successfully or 0 if count was zero
  61  */
  62 int lockref_get_not_zero(struct lockref *lockref)
  63 {
  64         int retval;
  65
  66         CMPXCHG_LOOP(
  67                 new.count++;
  68                 if (old.count <= 0)
  69                         return 0;
  70         ,
  71                 return 1;
  72         );
  73
  74         spin_lock(&lockref->lock);
  75         retval = 0;
  76         if (lockref->count > 0) {
  77                 lockref->count++;
  78                 retval = 1;
  79         }
  80         spin_unlock(&lockref->lock);
  81         return retval;
  82 }
  83 EXPORT_SYMBOL(lockref_get_not_zero);
  84
  85 /**
  86  * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
  87  * @lockref: pointer to lockref structure
  88  * Return: 1 if count updated successfully or 0 if count would become zero
  89  */
  90 int lockref_put_not_zero(struct lockref *lockref)
  91 {
  92         int retval;
  93
  94         CMPXCHG_LOOP(
  95                 new.count--;
  96                 if (old.count <= 1)
  97                         return 0;
  98         ,
  99                 return 1;
 100         );
 101
 102         spin_lock(&lockref->lock);
 103         retval = 0;
 104         if (lockref->count > 1) {
 105                 lockref->count--;
 106                 retval = 1;
 107         }
 108         spin_unlock(&lockref->lock);
 109         return retval;
 110 }
 111 EXPORT_SYMBOL(lockref_put_not_zero);
 112
 113 /**
 114  * lockref_get_or_lock - Increments count unless the count is 0 or dead
 115  * @lockref: pointer to lockref structure
 116  * Return: 1 if count updated successfully or 0 if count was zero
 117  * and we got the lock instead.
 118  */
 119 int lockref_get_or_lock(struct lockref *lockref)
 120 {
 121         CMPXCHG_LOOP(
 122                 new.count++;
 123                 if (old.count <= 0)
 124                         break;
 125         ,
 126                 return 1;
 127         );
 128
 129         spin_lock(&lockref->lock);
 130         if (lockref->count <= 0)
 131                 return 0;
 132         lockref->count++;
 133         spin_unlock(&lockref->lock);
 134         return 1;
 135 }
 136 EXPORT_SYMBOL(lockref_get_or_lock);
 137
 138 /**
 139  * lockref_put_return - Decrement reference count if possible
 140  * @lockref: pointer to lockref structure
 141  *
 142  * Decrement the reference count and return the new value.
 143  * If the lockref was dead or locked, return an error.
 144  */
 145 int lockref_put_return(struct lockref *lockref)
 146 {
 147         CMPXCHG_LOOP(
 148                 new.count--;
 149                 if (old.count <= 0)
 150                         return -1;
 151         ,
 152                 return new.count;
 153         );
 154         return -1;
 155 }
 156 EXPORT_SYMBOL(lockref_put_return);
 157
 158 /**
 159  * lockref_put_or_lock - decrements count unless count <= 1 before decrement
 160  * @lockref: pointer to lockref structure
 161  * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 162  */
 163 int lockref_put_or_lock(struct lockref *lockref)
 164 {
 165         CMPXCHG_LOOP(
 166                 new.count--;
 167                 if (old.count <= 1)
 168                         break;
 169         ,
 170                 return 1;
 171         );
 172
 173         spin_lock(&lockref->lock);
 174         if (lockref->count <= 1)
 175                 return 0;
 176         lockref->count--;
 177         spin_unlock(&lockref->lock);
 178         return 1;
 179 }
 180 EXPORT_SYMBOL(lockref_put_or_lock);
 181
 182 /**
 183  * lockref_mark_dead - mark lockref dead
 184  * @lockref: pointer to lockref structure
 185  */
 186 void lockref_mark_dead(struct lockref *lockref)
 187 {
 188         assert_spin_locked(&lockref->lock);
 189         lockref->count = -128;
 190 }
 191 EXPORT_SYMBOL(lockref_mark_dead);
 192
 193 /**
 194  * lockref_get_not_dead - Increments count unless the ref is dead
 195  * @lockref: pointer to lockref structure
 196  * Return: 1 if count updated successfully or 0 if lockref was dead
 197  */
 198 int lockref_get_not_dead(struct lockref *lockref)
 199 {
 200         int retval;
 201
 202         CMPXCHG_LOOP(
 203                 new.count++;
 204                 if (old.count < 0)
 205                         return 0;
 206         ,
 207                 return 1;
 208         );
 209
 210         spin_lock(&lockref->lock);
 211         retval = 0;
 212         if (lockref->count >= 0) {
 213                 lockref->count++;
 214                 retval = 1;
 215         }
 216         spin_unlock(&lockref->lock);
 217         return retval;
 218 }
 219 EXPORT_SYMBOL(lockref_get_not_dead);