lib/lockref.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/lockref.h>
   4
   5 #if USE_CMPXCHG_LOCKREF
   6
   7 /*
   8  * Note that the "cmpxchg()" reloads the "old" value for the
   9  * failure case.
  10  */
  11 #define CMPXCHG_LOOP(CODE, SUCCESS) do {                                        \
  12         int retry = 100;                                                        \
  13         struct lockref old;                                                     \
  14         BUILD_BUG_ON(sizeof(old) != 8);                                         \
  15         old.lock_count = READ_ONCE(lockref->lock_count);                        \
  16         while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {     \
  17                 struct lockref new = old;                                       \
  18                 CODE                                                            \
  19                 if (likely(try_cmpxchg64_relaxed(&lockref->lock_count,          \
  20                                                  &old.lock_count,               \
  21                                                  new.lock_count))) {            \
  22                         SUCCESS;                                                \
  23                 }                                                               \
  24                 if (!--retry)                                                   \
  25                         break;                                                  \
  26                 cpu_relax();                                                    \
  27         }                                                                       \
  28 } while (0)
  29
  30 #else
  31
  32 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
  33
  34 #endif
  35
  36 /**
  37  * lockref_get - Increments reference count unconditionally
  38  * @lockref: pointer to lockref structure
  39  *
  40  * This operation is only valid if you already hold a reference
  41  * to the object, so you know the count cannot be zero.
  42  */
  43 void lockref_get(struct lockref *lockref)
  44 {
  45         CMPXCHG_LOOP(
  46                 new.count++;
  47         ,
  48                 return;
  49         );
  50
  51         spin_lock(&lockref->lock);
  52         lockref->count++;
  53         spin_unlock(&lockref->lock);
  54 }
  55 EXPORT_SYMBOL(lockref_get);
  56
  57 /**
  58  * lockref_get_not_zero - Increments count unless the count is 0 or dead
  59  * @lockref: pointer to lockref structure
  60  * Return: 1 if count updated successfully or 0 if count was zero
  61  */
  62 int lockref_get_not_zero(struct lockref *lockref)
  63 {
  64         int retval;
  65
  66         CMPXCHG_LOOP(
  67                 new.count++;
  68                 if (old.count <= 0)
  69                         return 0;
  70         ,
  71                 return 1;
  72         );
  73
  74         spin_lock(&lockref->lock);
  75         retval = 0;
  76         if (lockref->count > 0) {
  77                 lockref->count++;
  78                 retval = 1;
  79         }
  80         spin_unlock(&lockref->lock);
  81         return retval;
  82 }
  83 EXPORT_SYMBOL(lockref_get_not_zero);
  84
  85 /**
  86  * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
  87  * @lockref: pointer to lockref structure
  88  * Return: 1 if count updated successfully or 0 if count would become zero
  89  */
  90 int lockref_put_not_zero(struct lockref *lockref)
  91 {
  92         int retval;
  93
  94         CMPXCHG_LOOP(
  95                 new.count--;
  96                 if (old.count <= 1)
  97                         return 0;
  98         ,
  99                 return 1;
 100         );
 101
 102         spin_lock(&lockref->lock);
 103         retval = 0;
 104         if (lockref->count > 1) {
 105                 lockref->count--;
 106                 retval = 1;
 107         }
 108         spin_unlock(&lockref->lock);
 109         return retval;
 110 }
 111 EXPORT_SYMBOL(lockref_put_not_zero);
 112
 113 /**
 114  * lockref_put_return - Decrement reference count if possible
 115  * @lockref: pointer to lockref structure
 116  *
 117  * Decrement the reference count and return the new value.
 118  * If the lockref was dead or locked, return an error.
 119  */
 120 int lockref_put_return(struct lockref *lockref)
 121 {
 122         CMPXCHG_LOOP(
 123                 new.count--;
 124                 if (old.count <= 0)
 125                         return -1;
 126         ,
 127                 return new.count;
 128         );
 129         return -1;
 130 }
 131 EXPORT_SYMBOL(lockref_put_return);
 132
 133 /**
 134  * lockref_put_or_lock - decrements count unless count <= 1 before decrement
 135  * @lockref: pointer to lockref structure
 136  * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 137  */
 138 int lockref_put_or_lock(struct lockref *lockref)
 139 {
 140         CMPXCHG_LOOP(
 141                 new.count--;
 142                 if (old.count <= 1)
 143                         break;
 144         ,
 145                 return 1;
 146         );
 147
 148         spin_lock(&lockref->lock);
 149         if (lockref->count <= 1)
 150                 return 0;
 151         lockref->count--;
 152         spin_unlock(&lockref->lock);
 153         return 1;
 154 }
 155 EXPORT_SYMBOL(lockref_put_or_lock);
 156
 157 /**
 158  * lockref_mark_dead - mark lockref dead
 159  * @lockref: pointer to lockref structure
 160  */
 161 void lockref_mark_dead(struct lockref *lockref)
 162 {
 163         assert_spin_locked(&lockref->lock);
 164         lockref->count = -128;
 165 }
 166 EXPORT_SYMBOL(lockref_mark_dead);
 167
 168 /**
 169  * lockref_get_not_dead - Increments count unless the ref is dead
 170  * @lockref: pointer to lockref structure
 171  * Return: 1 if count updated successfully or 0 if lockref was dead
 172  */
 173 int lockref_get_not_dead(struct lockref *lockref)
 174 {
 175         int retval;
 176
 177         CMPXCHG_LOOP(
 178                 new.count++;
 179                 if (old.count < 0)
 180                         return 0;
 181         ,
 182                 return 1;
 183         );
 184
 185         spin_lock(&lockref->lock);
 186         retval = 0;
 187         if (lockref->count >= 0) {
 188                 lockref->count++;
 189                 retval = 1;
 190         }
 191         spin_unlock(&lockref->lock);
 192         return retval;
 193 }
 194 EXPORT_SYMBOL(lockref_get_not_dead);