lib/lockref.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/lockref.h>
   4
   5 #if USE_CMPXCHG_LOCKREF
   6
   7 /*
   8  * Note that the "cmpxchg()" reloads the "old" value for the
   9  * failure case.
  10  */
  11 #define CMPXCHG_LOOP(CODE, SUCCESS) do {                                        \
  12         struct lockref old;                                                     \
  13         BUILD_BUG_ON(sizeof(old) != 8);                                         \
  14         old.lock_count = READ_ONCE(lockref->lock_count);                        \
  15         while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {     \
  16                 struct lockref new = old, prev = old;                           \
  17                 CODE                                                            \
  18                 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,        \
  19                                                    old.lock_count,              \
  20                                                    new.lock_count);             \
  21                 if (likely(old.lock_count == prev.lock_count)) {                \
  22                         SUCCESS;                                                \
  23                 }                                                               \
  24                 cpu_relax();                                                    \
  25         }                                                                       \
  26 } while (0)
  27
  28 #else
  29
  30 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
  31
  32 #endif
  33
  34 /**
  35  * lockref_get - Increments reference count unconditionally
  36  * @lockref: pointer to lockref structure
  37  *
  38  * This operation is only valid if you already hold a reference
  39  * to the object, so you know the count cannot be zero.
  40  */
  41 void lockref_get(struct lockref *lockref)
  42 {
  43         CMPXCHG_LOOP(
  44                 new.count++;
  45         ,
  46                 return;
  47         );
  48
  49         spin_lock(&lockref->lock);
  50         lockref->count++;
  51         spin_unlock(&lockref->lock);
  52 }
  53 EXPORT_SYMBOL(lockref_get);
  54
  55 /**
  56  * lockref_get_not_zero - Increments count unless the count is 0 or dead
  57  * @lockref: pointer to lockref structure
  58  * Return: 1 if count updated successfully or 0 if count was zero
  59  */
  60 int lockref_get_not_zero(struct lockref *lockref)
  61 {
  62         int retval;
  63
  64         CMPXCHG_LOOP(
  65                 new.count++;
  66                 if (old.count <= 0)
  67                         return 0;
  68         ,
  69                 return 1;
  70         );
  71
  72         spin_lock(&lockref->lock);
  73         retval = 0;
  74         if (lockref->count > 0) {
  75                 lockref->count++;
  76                 retval = 1;
  77         }
  78         spin_unlock(&lockref->lock);
  79         return retval;
  80 }
  81 EXPORT_SYMBOL(lockref_get_not_zero);
  82
  83 /**
  84  * lockref_get_or_lock - Increments count unless the count is 0 or dead
  85  * @lockref: pointer to lockref structure
  86  * Return: 1 if count updated successfully or 0 if count was zero
  87  * and we got the lock instead.
  88  */
  89 int lockref_get_or_lock(struct lockref *lockref)
  90 {
  91         CMPXCHG_LOOP(
  92                 new.count++;
  93                 if (old.count <= 0)
  94                         break;
  95         ,
  96                 return 1;
  97         );
  98
  99         spin_lock(&lockref->lock);
 100         if (lockref->count <= 0)
 101                 return 0;
 102         lockref->count++;
 103         spin_unlock(&lockref->lock);
 104         return 1;
 105 }
 106 EXPORT_SYMBOL(lockref_get_or_lock);
 107
 108 /**
 109  * lockref_put_return - Decrement reference count if possible
 110  * @lockref: pointer to lockref structure
 111  *
 112  * Decrement the reference count and return the new value.
 113  * If the lockref was dead or locked, return an error.
 114  */
 115 int lockref_put_return(struct lockref *lockref)
 116 {
 117         CMPXCHG_LOOP(
 118                 new.count--;
 119                 if (old.count <= 0)
 120                         return -1;
 121         ,
 122                 return new.count;
 123         );
 124         return -1;
 125 }
 126 EXPORT_SYMBOL(lockref_put_return);
 127
 128 /**
 129  * lockref_put_or_lock - decrements count unless count <= 1 before decrement
 130  * @lockref: pointer to lockref structure
 131  * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 132  */
 133 int lockref_put_or_lock(struct lockref *lockref)
 134 {
 135         CMPXCHG_LOOP(
 136                 new.count--;
 137                 if (old.count <= 1)
 138                         break;
 139         ,
 140                 return 1;
 141         );
 142
 143         spin_lock(&lockref->lock);
 144         if (lockref->count <= 1)
 145                 return 0;
 146         lockref->count--;
 147         spin_unlock(&lockref->lock);
 148         return 1;
 149 }
 150 EXPORT_SYMBOL(lockref_put_or_lock);
 151
 152 /**
 153  * lockref_mark_dead - mark lockref dead
 154  * @lockref: pointer to lockref structure
 155  */
 156 void lockref_mark_dead(struct lockref *lockref)
 157 {
 158         assert_spin_locked(&lockref->lock);
 159         lockref->count = -128;
 160 }
 161 EXPORT_SYMBOL(lockref_mark_dead);
 162
 163 /**
 164  * lockref_get_not_dead - Increments count unless the ref is dead
 165  * @lockref: pointer to lockref structure
 166  * Return: 1 if count updated successfully or 0 if lockref was dead
 167  */
 168 int lockref_get_not_dead(struct lockref *lockref)
 169 {
 170         int retval;
 171
 172         CMPXCHG_LOOP(
 173                 new.count++;
 174                 if (old.count < 0)
 175                         return 0;
 176         ,
 177                 return 1;
 178         );
 179
 180         spin_lock(&lockref->lock);
 181         retval = 0;
 182         if (lockref->count >= 0) {
 183                 lockref->count++;
 184                 retval = 1;
 185         }
 186         spin_unlock(&lockref->lock);
 187         return retval;
 188 }
 189 EXPORT_SYMBOL(lockref_get_not_dead);