include/linux/seqlock.h

   1 #ifndef __LINUX_SEQLOCK_H
   2 #define __LINUX_SEQLOCK_H
   3 /*
   4  * Reader/writer consistent mechanism without starving writers. This type of
   5  * lock for data where the reader wants a consistent set of information
   6  * and is willing to retry if the information changes.  Readers never
   7  * block but they may have to retry if a writer is in
   8  * progress. Writers do not wait for readers.
   9  *
  10  * This is not as cache friendly as brlock. Also, this will not work
  11  * for data that contains pointers, because any writer could
  12  * invalidate a pointer that a reader was following.
  13  *
  14  * Expected reader usage:
  15  *      do {
  16  *          seq = read_seqbegin(&foo);
  17  *      ...
  18  *      } while (read_seqretry(&foo, seq));
  19  *
  20  *
  21  * On non-SMP the spin locks disappear but the writer still needs
  22  * to increment the sequence variables because an interrupt routine could
  23  * change the state of the data.
  24  *
  25  * Based on x86_64 vsyscall gettimeofday
  26  * by Keith Owens and Andrea Arcangeli
  27  */
  28
  29 #include <linux/spinlock.h>
  30 #include <linux/preempt.h>
  31 #include <asm/processor.h>
  32
  33 typedef struct {
  34         unsigned sequence;
  35         spinlock_t lock;
  36 } seqlock_t;
  37
  38 /*
  39  * These macros triggered gcc-3.x compile-time problems.  We think these are
  40  * OK now.  Be cautious.
  41  */
  42 #define __SEQLOCK_UNLOCKED(lockname) \
  43                  { 0, __SPIN_LOCK_UNLOCKED(lockname) }
  44
  45 #define seqlock_init(x)                                 \
  46         do {                                            \
  47                 (x)->sequence = 0;                      \
  48                 spin_lock_init(&(x)->lock);             \
  49         } while (0)
  50
  51 #define DEFINE_SEQLOCK(x) \
  52                 seqlock_t x = __SEQLOCK_UNLOCKED(x)
  53
  54 /* Lock out other writers and update the count.
  55  * Acts like a normal spin_lock/unlock.
  56  * Don't need preempt_disable() because that is in the spin_lock already.
  57  */
  58 static inline void write_seqlock(seqlock_t *sl)
  59 {
  60         spin_lock(&sl->lock);
  61         ++sl->sequence;
  62         smp_wmb();
  63 }
  64
  65 static inline void write_sequnlock(seqlock_t *sl)
  66 {
  67         smp_wmb();
  68         sl->sequence++;
  69         spin_unlock(&sl->lock);
  70 }
  71
  72 /* Start of read calculation -- fetch last complete writer token */
  73 static __always_inline unsigned read_seqbegin(const seqlock_t *sl)
  74 {
  75         unsigned ret;
  76
  77 repeat:
  78         ret = ACCESS_ONCE(sl->sequence);
  79         if (unlikely(ret & 1)) {
  80                 cpu_relax();
  81                 goto repeat;
  82         }
  83         smp_rmb();
  84
  85         return ret;
  86 }
  87
  88 /*
  89  * Test if reader processed invalid data.
  90  *
  91  * If sequence value changed then writer changed data while in section.
  92  */
  93 static __always_inline int read_seqretry(const seqlock_t *sl, unsigned start)
  94 {
  95         smp_rmb();
  96
  97         return unlikely(sl->sequence != start);
  98 }
  99
 100
 101 /*
 102  * Version using sequence counter only.
 103  * This can be used when code has its own mutex protecting the
 104  * updating starting before the write_seqcountbeqin() and ending
 105  * after the write_seqcount_end().
 106  */
 107
 108 typedef struct seqcount {
 109         unsigned sequence;
 110 } seqcount_t;
 111
 112 #define SEQCNT_ZERO { 0 }
 113 #define seqcount_init(x)        do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0)
 114
 115 /**
 116  * __read_seqcount_begin - begin a seq-read critical section (without barrier)
 117  * @s: pointer to seqcount_t
 118  * Returns: count to be passed to read_seqcount_retry
 119  *
 120  * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb()
 121  * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
 122  * provided before actually loading any of the variables that are to be
 123  * protected in this critical section.
 124  *
 125  * Use carefully, only in critical code, and comment how the barrier is
 126  * provided.
 127  */
 128 static inline unsigned __read_seqcount_begin(const seqcount_t *s)
 129 {
 130         unsigned ret;
 131
 132 repeat:
 133         ret = ACCESS_ONCE(s->sequence);
 134         if (unlikely(ret & 1)) {
 135                 cpu_relax();
 136                 goto repeat;
 137         }
 138         return ret;
 139 }
 140
 141 /**
 142  * read_seqcount_begin - begin a seq-read critical section
 143  * @s: pointer to seqcount_t
 144  * Returns: count to be passed to read_seqcount_retry
 145  *
 146  * read_seqcount_begin opens a read critical section of the given seqcount.
 147  * Validity of the critical section is tested by checking read_seqcount_retry
 148  * function.
 149  */
 150 static inline unsigned read_seqcount_begin(const seqcount_t *s)
 151 {
 152         unsigned ret = __read_seqcount_begin(s);
 153         smp_rmb();
 154         return ret;
 155 }
 156
 157 /**
 158  * raw_seqcount_begin - begin a seq-read critical section
 159  * @s: pointer to seqcount_t
 160  * Returns: count to be passed to read_seqcount_retry
 161  *
 162  * raw_seqcount_begin opens a read critical section of the given seqcount.
 163  * Validity of the critical section is tested by checking read_seqcount_retry
 164  * function.
 165  *
 166  * Unlike read_seqcount_begin(), this function will not wait for the count
 167  * to stabilize. If a writer is active when we begin, we will fail the
 168  * read_seqcount_retry() instead of stabilizing at the beginning of the
 169  * critical section.
 170  */
 171 static inline unsigned raw_seqcount_begin(const seqcount_t *s)
 172 {
 173         unsigned ret = ACCESS_ONCE(s->sequence);
 174         smp_rmb();
 175         return ret & ~1;
 176 }
 177
 178 /**
 179  * __read_seqcount_retry - end a seq-read critical section (without barrier)
 180  * @s: pointer to seqcount_t
 181  * @start: count, from read_seqcount_begin
 182  * Returns: 1 if retry is required, else 0
 183  *
 184  * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb()
 185  * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
 186  * provided before actually loading any of the variables that are to be
 187  * protected in this critical section.
 188  *
 189  * Use carefully, only in critical code, and comment how the barrier is
 190  * provided.
 191  */
 192 static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start)
 193 {
 194         return unlikely(s->sequence != start);
 195 }
 196
 197 /**
 198  * read_seqcount_retry - end a seq-read critical section
 199  * @s: pointer to seqcount_t
 200  * @start: count, from read_seqcount_begin
 201  * Returns: 1 if retry is required, else 0
 202  *
 203  * read_seqcount_retry closes a read critical section of the given seqcount.
 204  * If the critical section was invalid, it must be ignored (and typically
 205  * retried).
 206  */
 207 static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
 208 {
 209         smp_rmb();
 210
 211         return __read_seqcount_retry(s, start);
 212 }
 213
 214
 215 /*
 216  * Sequence counter only version assumes that callers are using their
 217  * own mutexing.
 218  */
 219 static inline void write_seqcount_begin(seqcount_t *s)
 220 {
 221         s->sequence++;
 222         smp_wmb();
 223 }
 224
 225 static inline void write_seqcount_end(seqcount_t *s)
 226 {
 227         smp_wmb();
 228         s->sequence++;
 229 }
 230
 231 /**
 232  * write_seqcount_barrier - invalidate in-progress read-side seq operations
 233  * @s: pointer to seqcount_t
 234  *
 235  * After write_seqcount_barrier, no read-side seq operations will complete
 236  * successfully and see data older than this.
 237  */
 238 static inline void write_seqcount_barrier(seqcount_t *s)
 239 {
 240         smp_wmb();
 241         s->sequence+=2;
 242 }
 243
 244 /*
 245  * Possible sw/hw IRQ protected versions of the interfaces.
 246  */
 247 #define write_seqlock_irqsave(lock, flags)                              \
 248         do { local_irq_save(flags); write_seqlock(lock); } while (0)
 249 #define write_seqlock_irq(lock)                                         \
 250         do { local_irq_disable();   write_seqlock(lock); } while (0)
 251 #define write_seqlock_bh(lock)                                          \
 252         do { local_bh_disable();    write_seqlock(lock); } while (0)
 253
 254 #define write_sequnlock_irqrestore(lock, flags)                         \
 255         do { write_sequnlock(lock); local_irq_restore(flags); } while(0)
 256 #define write_sequnlock_irq(lock)                                       \
 257         do { write_sequnlock(lock); local_irq_enable(); } while(0)
 258 #define write_sequnlock_bh(lock)                                        \
 259         do { write_sequnlock(lock); local_bh_enable(); } while(0)
 260
 261 #endif /* __LINUX_SEQLOCK_H */