fs/fscache/cookie.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* netfs cookie management
   3  *
   4  * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * See Documentation/filesystems/caching/netfs-api.rst for more information on
   8  * the netfs API.
   9  */
  10
  11 #define FSCACHE_DEBUG_LEVEL COOKIE
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include "internal.h"
  15
  16 struct kmem_cache *fscache_cookie_jar;
  17
  18 static void fscache_cookie_lru_timed_out(struct timer_list *timer);
  19 static void fscache_cookie_lru_worker(struct work_struct *work);
  20 static void fscache_cookie_worker(struct work_struct *work);
  21 static void fscache_unhash_cookie(struct fscache_cookie *cookie);
  22 static void fscache_perform_invalidation(struct fscache_cookie *cookie);
  23
  24 #define fscache_cookie_hash_shift 15
  25 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  26 static LIST_HEAD(fscache_cookies);
  27 static DEFINE_RWLOCK(fscache_cookies_lock);
  28 static LIST_HEAD(fscache_cookie_lru);
  29 static DEFINE_SPINLOCK(fscache_cookie_lru_lock);
  30 DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out);
  31 static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker);
  32 static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD";
  33 static unsigned int fscache_lru_cookie_timeout = 10 * HZ;
  34
  35 void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  36 {
  37         const u8 *k;
  38
  39         pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n",
  40                prefix,
  41                cookie->debug_id,
  42                cookie->flags,
  43                atomic_read(&cookie->n_active),
  44                atomic_read(&cookie->n_accesses),
  45                fscache_cookie_states[cookie->state]);
  46         pr_err("%c-cookie V=%08x [%s]\n",
  47                prefix,
  48                cookie->volume->debug_id,
  49                cookie->volume->key);
  50
  51         k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  52                 cookie->inline_key : cookie->key;
  53         pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k);
  54 }
  55
  56 static void fscache_free_cookie(struct fscache_cookie *cookie)
  57 {
  58         if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) {
  59                 spin_lock(&fscache_cookie_lru_lock);
  60                 list_del_init(&cookie->commit_link);
  61                 spin_unlock(&fscache_cookie_lru_lock);
  62                 fscache_stat_d(&fscache_n_cookies_lru);
  63                 fscache_stat(&fscache_n_cookies_lru_removed);
  64         }
  65
  66         if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) {
  67                 fscache_print_cookie(cookie, 'F');
  68                 return;
  69         }
  70
  71         write_lock(&fscache_cookies_lock);
  72         list_del(&cookie->proc_link);
  73         write_unlock(&fscache_cookies_lock);
  74         if (cookie->aux_len > sizeof(cookie->inline_aux))
  75                 kfree(cookie->aux);
  76         if (cookie->key_len > sizeof(cookie->inline_key))
  77                 kfree(cookie->key);
  78         fscache_stat_d(&fscache_n_cookies);
  79         kmem_cache_free(fscache_cookie_jar, cookie);
  80 }
  81
  82 static void __fscache_queue_cookie(struct fscache_cookie *cookie)
  83 {
  84         if (!queue_work(fscache_wq, &cookie->work))
  85                 fscache_put_cookie(cookie, fscache_cookie_put_over_queued);
  86 }
  87
  88 static void fscache_queue_cookie(struct fscache_cookie *cookie,
  89                                  enum fscache_cookie_trace where)
  90 {
  91         fscache_get_cookie(cookie, where);
  92         __fscache_queue_cookie(cookie);
  93 }
  94
  95 /*
  96  * Initialise the access gate on a cookie by setting a flag to prevent the
  97  * state machine from being queued when the access counter transitions to 0.
  98  * We're only interested in this when we withdraw caching services from the
  99  * cookie.
 100  */
 101 static void fscache_init_access_gate(struct fscache_cookie *cookie)
 102 {
 103         int n_accesses;
 104
 105         n_accesses = atomic_read(&cookie->n_accesses);
 106         trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
 107                              n_accesses, fscache_access_cache_pin);
 108         set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
 109 }
 110
 111 /**
 112  * fscache_end_cookie_access - Unpin a cache at the end of an access.
 113  * @cookie: A data file cookie
 114  * @why: An indication of the circumstances of the access for tracing
 115  *
 116  * Unpin a cache cookie after we've accessed it and bring a deferred
 117  * relinquishment or withdrawal state into effect.
 118  *
 119  * The @why indicator is provided for tracing purposes.
 120  */
 121 void fscache_end_cookie_access(struct fscache_cookie *cookie,
 122                                enum fscache_access_trace why)
 123 {
 124         int n_accesses;
 125
 126         smp_mb__before_atomic();
 127         n_accesses = atomic_dec_return(&cookie->n_accesses);
 128         trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
 129                              n_accesses, why);
 130         if (n_accesses == 0 &&
 131             !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags))
 132                 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
 133 }
 134 EXPORT_SYMBOL(fscache_end_cookie_access);
 135
 136 /*
 137  * Pin the cache behind a cookie so that we can access it.
 138  */
 139 static void __fscache_begin_cookie_access(struct fscache_cookie *cookie,
 140                                           enum fscache_access_trace why)
 141 {
 142         int n_accesses;
 143
 144         n_accesses = atomic_inc_return(&cookie->n_accesses);
 145         smp_mb__after_atomic(); /* (Future) read state after is-caching.
 146                                  * Reread n_accesses after is-caching
 147                                  */
 148         trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
 149                              n_accesses, why);
 150 }
 151
 152 /**
 153  * fscache_begin_cookie_access - Pin a cache so data can be accessed
 154  * @cookie: A data file cookie
 155  * @why: An indication of the circumstances of the access for tracing
 156  *
 157  * Attempt to pin the cache to prevent it from going away whilst we're
 158  * accessing data and returns true if successful.  This works as follows:
 159  *
 160  *  (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not
 161  *      set), we return false to indicate access was not permitted.
 162  *
 163  *  (2) If the cookie is being cached, we increment its n_accesses count and
 164  *      then recheck the IS_CACHING flag, ending the access if it got cleared.
 165  *
 166  *  (3) When we end the access, we decrement the cookie's n_accesses and wake
 167  *      up the any waiters if it reaches 0.
 168  *
 169  *  (4) Whilst the cookie is actively being cached, its n_accesses is kept
 170  *      artificially incremented to prevent wakeups from happening.
 171  *
 172  *  (5) When the cache is taken offline or if the cookie is culled, the flag is
 173  *      cleared to prevent new accesses, the cookie's n_accesses is decremented
 174  *      and we wait for it to become 0.
 175  *
 176  * The @why indicator are merely provided for tracing purposes.
 177  */
 178 bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
 179                                  enum fscache_access_trace why)
 180 {
 181         if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
 182                 return false;
 183         __fscache_begin_cookie_access(cookie, why);
 184         if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) ||
 185             !fscache_cache_is_live(cookie->volume->cache)) {
 186                 fscache_end_cookie_access(cookie, fscache_access_unlive);
 187                 return false;
 188         }
 189         return true;
 190 }
 191
 192 static inline void wake_up_cookie_state(struct fscache_cookie *cookie)
 193 {
 194         /* Use a barrier to ensure that waiters see the state variable
 195          * change, as spin_unlock doesn't guarantee a barrier.
 196          *
 197          * See comments over wake_up_bit() and waitqueue_active().
 198          */
 199         smp_mb();
 200         wake_up_var(&cookie->state);
 201 }
 202
 203 /*
 204  * Change the state a cookie is at and wake up anyone waiting for that.  Impose
 205  * an ordering between the stuff stored in the cookie and the state member.
 206  * Paired with fscache_cookie_state().
 207  */
 208 static void __fscache_set_cookie_state(struct fscache_cookie *cookie,
 209                                        enum fscache_cookie_state state)
 210 {
 211         smp_store_release(&cookie->state, state);
 212 }
 213
 214 static void fscache_set_cookie_state(struct fscache_cookie *cookie,
 215                                      enum fscache_cookie_state state)
 216 {
 217         spin_lock(&cookie->lock);
 218         __fscache_set_cookie_state(cookie, state);
 219         spin_unlock(&cookie->lock);
 220         wake_up_cookie_state(cookie);
 221 }
 222
 223 /**
 224  * fscache_cookie_lookup_negative - Note negative lookup
 225  * @cookie: The cookie that was being looked up
 226  *
 227  * Note that some part of the metadata path in the cache doesn't exist and so
 228  * we can release any waiting readers in the certain knowledge that there's
 229  * nothing for them to actually read.
 230  *
 231  * This function uses no locking and must only be called from the state machine.
 232  */
 233 void fscache_cookie_lookup_negative(struct fscache_cookie *cookie)
 234 {
 235         set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
 236         fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING);
 237 }
 238 EXPORT_SYMBOL(fscache_cookie_lookup_negative);
 239
 240 /**
 241  * fscache_resume_after_invalidation - Allow I/O to resume after invalidation
 242  * @cookie: The cookie that was invalidated
 243  *
 244  * Tell fscache that invalidation is sufficiently complete that I/O can be
 245  * allowed again.
 246  */
 247 void fscache_resume_after_invalidation(struct fscache_cookie *cookie)
 248 {
 249         fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
 250 }
 251 EXPORT_SYMBOL(fscache_resume_after_invalidation);
 252
 253 /**
 254  * fscache_caching_failed - Report that a failure stopped caching on a cookie
 255  * @cookie: The cookie that was affected
 256  *
 257  * Tell fscache that caching on a cookie needs to be stopped due to some sort
 258  * of failure.
 259  *
 260  * This function uses no locking and must only be called from the state machine.
 261  */
 262 void fscache_caching_failed(struct fscache_cookie *cookie)
 263 {
 264         clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
 265         fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED);
 266         trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
 267                                 fscache_cookie_failed);
 268 }
 269 EXPORT_SYMBOL(fscache_caching_failed);
 270
 271 /*
 272  * Set the index key in a cookie.  The cookie struct has space for a 16-byte
 273  * key plus length and hash, but if that's not big enough, it's instead a
 274  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
 275  * the key data.
 276  */
 277 static int fscache_set_key(struct fscache_cookie *cookie,
 278                            const void *index_key, size_t index_key_len)
 279 {
 280         void *buf;
 281         size_t buf_size;
 282
 283         buf_size = round_up(index_key_len, sizeof(__le32));
 284
 285         if (index_key_len > sizeof(cookie->inline_key)) {
 286                 buf = kzalloc(buf_size, GFP_KERNEL);
 287                 if (!buf)
 288                         return -ENOMEM;
 289                 cookie->key = buf;
 290         } else {
 291                 buf = cookie->inline_key;
 292         }
 293
 294         memcpy(buf, index_key, index_key_len);
 295         cookie->key_hash = fscache_hash(cookie->volume->key_hash,
 296                                         buf, buf_size);
 297         return 0;
 298 }
 299
 300 static bool fscache_cookie_same(const struct fscache_cookie *a,
 301                                 const struct fscache_cookie *b)
 302 {
 303         const void *ka, *kb;
 304
 305         if (a->key_hash != b->key_hash ||
 306             a->volume   != b->volume ||
 307             a->key_len  != b->key_len)
 308                 return false;
 309
 310         if (a->key_len <= sizeof(a->inline_key)) {
 311                 ka = &a->inline_key;
 312                 kb = &b->inline_key;
 313         } else {
 314                 ka = a->key;
 315                 kb = b->key;
 316         }
 317         return memcmp(ka, kb, a->key_len) == 0;
 318 }
 319
 320 static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
 321
 322 /*
 323  * Allocate a cookie.
 324  */
 325 static struct fscache_cookie *fscache_alloc_cookie(
 326         struct fscache_volume *volume,
 327         u8 advice,
 328         const void *index_key, size_t index_key_len,
 329         const void *aux_data, size_t aux_data_len,
 330         loff_t object_size)
 331 {
 332         struct fscache_cookie *cookie;
 333
 334         /* allocate and initialise a cookie */
 335         cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
 336         if (!cookie)
 337                 return NULL;
 338         fscache_stat(&fscache_n_cookies);
 339
 340         cookie->volume          = volume;
 341         cookie->advice          = advice;
 342         cookie->key_len         = index_key_len;
 343         cookie->aux_len         = aux_data_len;
 344         cookie->object_size     = object_size;
 345         if (object_size == 0)
 346                 __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
 347
 348         if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 349                 goto nomem;
 350
 351         if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 352                 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 353         } else {
 354                 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 355                 if (!cookie->aux)
 356                         goto nomem;
 357         }
 358
 359         refcount_set(&cookie->ref, 1);
 360         cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
 361         spin_lock_init(&cookie->lock);
 362         INIT_LIST_HEAD(&cookie->commit_link);
 363         INIT_WORK(&cookie->work, fscache_cookie_worker);
 364         __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
 365
 366         write_lock(&fscache_cookies_lock);
 367         list_add_tail(&cookie->proc_link, &fscache_cookies);
 368         write_unlock(&fscache_cookies_lock);
 369         fscache_see_cookie(cookie, fscache_cookie_new_acquire);
 370         return cookie;
 371
 372 nomem:
 373         fscache_free_cookie(cookie);
 374         return NULL;
 375 }
 376
 377 static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie)
 378 {
 379         return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED;
 380 }
 381
 382 static void fscache_wait_on_collision(struct fscache_cookie *candidate,
 383                                       struct fscache_cookie *wait_for)
 384 {
 385         enum fscache_cookie_state *statep = &wait_for->state;
 386
 387         wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for),
 388                                20 * HZ);
 389         if (!fscache_cookie_is_dropped(wait_for)) {
 390                 pr_notice("Potential collision c=%08x old: c=%08x",
 391                           candidate->debug_id, wait_for->debug_id);
 392                 wait_var_event(statep, fscache_cookie_is_dropped(wait_for));
 393         }
 394 }
 395
 396 /*
 397  * Attempt to insert the new cookie into the hash.  If there's a collision, we
 398  * wait for the old cookie to complete if it's being relinquished and an error
 399  * otherwise.
 400  */
 401 static bool fscache_hash_cookie(struct fscache_cookie *candidate)
 402 {
 403         struct fscache_cookie *cursor, *wait_for = NULL;
 404         struct hlist_bl_head *h;
 405         struct hlist_bl_node *p;
 406         unsigned int bucket;
 407
 408         bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 409         h = &fscache_cookie_hash[bucket];
 410
 411         hlist_bl_lock(h);
 412         hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 413                 if (fscache_cookie_same(candidate, cursor)) {
 414                         if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags))
 415                                 goto collision;
 416                         wait_for = fscache_get_cookie(cursor,
 417                                                       fscache_cookie_get_hash_collision);
 418                         break;
 419                 }
 420         }
 421
 422         fscache_get_volume(candidate->volume, fscache_volume_get_cookie);
 423         atomic_inc(&candidate->volume->n_cookies);
 424         hlist_bl_add_head(&candidate->hash_link, h);
 425         set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags);
 426         hlist_bl_unlock(h);
 427
 428         if (wait_for) {
 429                 fscache_wait_on_collision(candidate, wait_for);
 430                 fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision);
 431         }
 432         return true;
 433
 434 collision:
 435         trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref),
 436                              fscache_cookie_collision);
 437         pr_err("Duplicate cookie detected\n");
 438         fscache_print_cookie(cursor, 'O');
 439         fscache_print_cookie(candidate, 'N');
 440         hlist_bl_unlock(h);
 441         return false;
 442 }
 443
 444 /*
 445  * Request a cookie to represent a data storage object within a volume.
 446  *
 447  * We never let on to the netfs about errors.  We may set a negative cookie
 448  * pointer, but that's okay
 449  */
 450 struct fscache_cookie *__fscache_acquire_cookie(
 451         struct fscache_volume *volume,
 452         u8 advice,
 453         const void *index_key, size_t index_key_len,
 454         const void *aux_data, size_t aux_data_len,
 455         loff_t object_size)
 456 {
 457         struct fscache_cookie *cookie;
 458
 459         _enter("V=%x", volume->debug_id);
 460
 461         if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 462                 return NULL;
 463         if (!aux_data || !aux_data_len) {
 464                 aux_data = NULL;
 465                 aux_data_len = 0;
 466         }
 467
 468         fscache_stat(&fscache_n_acquires);
 469
 470         cookie = fscache_alloc_cookie(volume, advice,
 471                                       index_key, index_key_len,
 472                                       aux_data, aux_data_len,
 473                                       object_size);
 474         if (!cookie) {
 475                 fscache_stat(&fscache_n_acquires_oom);
 476                 return NULL;
 477         }
 478
 479         if (!fscache_hash_cookie(cookie)) {
 480                 fscache_see_cookie(cookie, fscache_cookie_discard);
 481                 fscache_free_cookie(cookie);
 482                 return NULL;
 483         }
 484
 485         trace_fscache_acquire(cookie);
 486         fscache_stat(&fscache_n_acquires_ok);
 487         _leave(" = c=%08x", cookie->debug_id);
 488         return cookie;
 489 }
 490 EXPORT_SYMBOL(__fscache_acquire_cookie);
 491
 492 /*
 493  * Prepare a cache object to be written to.
 494  */
 495 static void fscache_prepare_to_write(struct fscache_cookie *cookie)
 496 {
 497         cookie->volume->cache->ops->prepare_to_write(cookie);
 498 }
 499
 500 /*
 501  * Look up a cookie in the cache.
 502  */
 503 static void fscache_perform_lookup(struct fscache_cookie *cookie)
 504 {
 505         enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed;
 506         bool need_withdraw = false;
 507
 508         _enter("");
 509
 510         if (!cookie->volume->cache_priv) {
 511                 fscache_create_volume(cookie->volume, true);
 512                 if (!cookie->volume->cache_priv) {
 513                         fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
 514                         goto out;
 515                 }
 516         }
 517
 518         if (!cookie->volume->cache->ops->lookup_cookie(cookie)) {
 519                 if (cookie->state != FSCACHE_COOKIE_STATE_FAILED)
 520                         fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
 521                 need_withdraw = true;
 522                 _leave(" [fail]");
 523                 goto out;
 524         }
 525
 526         fscache_see_cookie(cookie, fscache_cookie_see_active);
 527         spin_lock(&cookie->lock);
 528         if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
 529                 __fscache_set_cookie_state(cookie,
 530                                            FSCACHE_COOKIE_STATE_INVALIDATING);
 531         else
 532                 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
 533         spin_unlock(&cookie->lock);
 534         wake_up_cookie_state(cookie);
 535         trace = fscache_access_lookup_cookie_end;
 536
 537 out:
 538         fscache_end_cookie_access(cookie, trace);
 539         if (need_withdraw)
 540                 fscache_withdraw_cookie(cookie);
 541         fscache_end_volume_access(cookie->volume, cookie, trace);
 542 }
 543
 544 /*
 545  * Begin the process of looking up a cookie.  We offload the actual process to
 546  * a worker thread.
 547  */
 548 static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify)
 549 {
 550         if (will_modify) {
 551                 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
 552                 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
 553         }
 554         if (!fscache_begin_volume_access(cookie->volume, cookie,
 555                                          fscache_access_lookup_cookie))
 556                 return false;
 557
 558         __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie);
 559         __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP);
 560         set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
 561         set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags);
 562         return true;
 563 }
 564
 565 /*
 566  * Start using the cookie for I/O.  This prevents the backing object from being
 567  * reaped by VM pressure.
 568  */
 569 void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify)
 570 {
 571         enum fscache_cookie_state state;
 572         bool queue = false;
 573         int n_active;
 574
 575         _enter("c=%08x", cookie->debug_id);
 576
 577         if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
 578                  "Trying to use relinquished cookie\n"))
 579                 return;
 580
 581         spin_lock(&cookie->lock);
 582
 583         n_active = atomic_inc_return(&cookie->n_active);
 584         trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref),
 585                              n_active, atomic_read(&cookie->n_accesses),
 586                              will_modify ?
 587                              fscache_active_use_modify : fscache_active_use);
 588
 589 again:
 590         state = fscache_cookie_state(cookie);
 591         switch (state) {
 592         case FSCACHE_COOKIE_STATE_QUIESCENT:
 593                 queue = fscache_begin_lookup(cookie, will_modify);
 594                 break;
 595
 596         case FSCACHE_COOKIE_STATE_LOOKING_UP:
 597         case FSCACHE_COOKIE_STATE_CREATING:
 598                 if (will_modify)
 599                         set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
 600                 break;
 601         case FSCACHE_COOKIE_STATE_ACTIVE:
 602         case FSCACHE_COOKIE_STATE_INVALIDATING:
 603                 if (will_modify &&
 604                     !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) {
 605                         set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
 606                         queue = true;
 607                 }
 608                 /*
 609                  * We could race with cookie_lru which may set LRU_DISCARD bit
 610                  * but has yet to run the cookie state machine.  If this happens
 611                  * and another thread tries to use the cookie, clear LRU_DISCARD
 612                  * so we don't end up withdrawing the cookie while in use.
 613                  */
 614                 if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags))
 615                         fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear);
 616                 break;
 617
 618         case FSCACHE_COOKIE_STATE_FAILED:
 619         case FSCACHE_COOKIE_STATE_WITHDRAWING:
 620                 break;
 621
 622         case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
 623                 spin_unlock(&cookie->lock);
 624                 wait_var_event(&cookie->state,
 625                                fscache_cookie_state(cookie) !=
 626                                FSCACHE_COOKIE_STATE_LRU_DISCARDING);
 627                 spin_lock(&cookie->lock);
 628                 goto again;
 629
 630         case FSCACHE_COOKIE_STATE_DROPPED:
 631         case FSCACHE_COOKIE_STATE_RELINQUISHING:
 632                 WARN(1, "Can't use cookie in state %u\n", state);
 633                 break;
 634         }
 635
 636         spin_unlock(&cookie->lock);
 637         if (queue)
 638                 fscache_queue_cookie(cookie, fscache_cookie_get_use_work);
 639         _leave("");
 640 }
 641 EXPORT_SYMBOL(__fscache_use_cookie);
 642
 643 static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
 644 {
 645         clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags);
 646         if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
 647                 return;
 648
 649         cookie->unused_at = jiffies;
 650         spin_lock(&fscache_cookie_lru_lock);
 651         if (list_empty(&cookie->commit_link)) {
 652                 fscache_get_cookie(cookie, fscache_cookie_get_lru);
 653                 fscache_stat(&fscache_n_cookies_lru);
 654         }
 655         list_move_tail(&cookie->commit_link, &fscache_cookie_lru);
 656
 657         spin_unlock(&fscache_cookie_lru_lock);
 658         timer_reduce(&fscache_cookie_lru_timer,
 659                      jiffies + fscache_lru_cookie_timeout);
 660 }
 661
 662 /*
 663  * Stop using the cookie for I/O.
 664  */
 665 void __fscache_unuse_cookie(struct fscache_cookie *cookie,
 666                             const void *aux_data, const loff_t *object_size)
 667 {
 668         unsigned int debug_id = cookie->debug_id;
 669         unsigned int r = refcount_read(&cookie->ref);
 670         unsigned int a = atomic_read(&cookie->n_accesses);
 671         unsigned int c;
 672
 673         if (aux_data || object_size)
 674                 __fscache_update_cookie(cookie, aux_data, object_size);
 675
 676         /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
 677         c = atomic_fetch_add_unless(&cookie->n_active, -1, 1);
 678         if (c != 1) {
 679                 trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse);
 680                 return;
 681         }
 682
 683         spin_lock(&cookie->lock);
 684         r = refcount_read(&cookie->ref);
 685         a = atomic_read(&cookie->n_accesses);
 686         c = atomic_dec_return(&cookie->n_active);
 687         trace_fscache_active(debug_id, r, c, a, fscache_active_unuse);
 688         if (c == 0)
 689                 fscache_unuse_cookie_locked(cookie);
 690         spin_unlock(&cookie->lock);
 691 }
 692 EXPORT_SYMBOL(__fscache_unuse_cookie);
 693
 694 /*
 695  * Perform work upon the cookie, such as committing its cache state,
 696  * relinquishing it or withdrawing the backing cache.  We're protected from the
 697  * cache going away under us as object withdrawal must come through this
 698  * non-reentrant work item.
 699  */
 700 static void fscache_cookie_state_machine(struct fscache_cookie *cookie)
 701 {
 702         enum fscache_cookie_state state;
 703         bool wake = false;
 704
 705         _enter("c=%x", cookie->debug_id);
 706
 707 again:
 708         spin_lock(&cookie->lock);
 709 again_locked:
 710         state = cookie->state;
 711         switch (state) {
 712         case FSCACHE_COOKIE_STATE_QUIESCENT:
 713                 /* The QUIESCENT state is jumped to the LOOKING_UP state by
 714                  * fscache_use_cookie().
 715                  */
 716
 717                 if (atomic_read(&cookie->n_accesses) == 0 &&
 718                     test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
 719                         __fscache_set_cookie_state(cookie,
 720                                                    FSCACHE_COOKIE_STATE_RELINQUISHING);
 721                         wake = true;
 722                         goto again_locked;
 723                 }
 724                 break;
 725
 726         case FSCACHE_COOKIE_STATE_LOOKING_UP:
 727                 spin_unlock(&cookie->lock);
 728                 fscache_init_access_gate(cookie);
 729                 fscache_perform_lookup(cookie);
 730                 goto again;
 731
 732         case FSCACHE_COOKIE_STATE_INVALIDATING:
 733                 spin_unlock(&cookie->lock);
 734                 fscache_perform_invalidation(cookie);
 735                 goto again;
 736
 737         case FSCACHE_COOKIE_STATE_ACTIVE:
 738                 if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) {
 739                         spin_unlock(&cookie->lock);
 740                         fscache_prepare_to_write(cookie);
 741                         spin_lock(&cookie->lock);
 742                 }
 743                 if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) {
 744                         __fscache_set_cookie_state(cookie,
 745                                                    FSCACHE_COOKIE_STATE_LRU_DISCARDING);
 746                         wake = true;
 747                         goto again_locked;
 748                 }
 749                 fallthrough;
 750
 751         case FSCACHE_COOKIE_STATE_FAILED:
 752                 if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
 753                         fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
 754
 755                 if (atomic_read(&cookie->n_accesses) != 0)
 756                         break;
 757                 if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
 758                         __fscache_set_cookie_state(cookie,
 759                                                    FSCACHE_COOKIE_STATE_RELINQUISHING);
 760                         wake = true;
 761                         goto again_locked;
 762                 }
 763                 if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) {
 764                         __fscache_set_cookie_state(cookie,
 765                                                    FSCACHE_COOKIE_STATE_WITHDRAWING);
 766                         wake = true;
 767                         goto again_locked;
 768                 }
 769                 break;
 770
 771         case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
 772         case FSCACHE_COOKIE_STATE_RELINQUISHING:
 773         case FSCACHE_COOKIE_STATE_WITHDRAWING:
 774                 if (cookie->cache_priv) {
 775                         spin_unlock(&cookie->lock);
 776                         cookie->volume->cache->ops->withdraw_cookie(cookie);
 777                         spin_lock(&cookie->lock);
 778                 }
 779
 780                 if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
 781                         fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
 782
 783                 switch (state) {
 784                 case FSCACHE_COOKIE_STATE_RELINQUISHING:
 785                         fscache_see_cookie(cookie, fscache_cookie_see_relinquish);
 786                         fscache_unhash_cookie(cookie);
 787                         __fscache_set_cookie_state(cookie,
 788                                                    FSCACHE_COOKIE_STATE_DROPPED);
 789                         wake = true;
 790                         goto out;
 791                 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
 792                         fscache_see_cookie(cookie, fscache_cookie_see_lru_discard);
 793                         break;
 794                 case FSCACHE_COOKIE_STATE_WITHDRAWING:
 795                         fscache_see_cookie(cookie, fscache_cookie_see_withdraw);
 796                         break;
 797                 default:
 798                         BUG();
 799                 }
 800
 801                 clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags);
 802                 clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
 803                 clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
 804                 clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
 805                 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
 806                 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
 807                 wake = true;
 808                 goto again_locked;
 809
 810         case FSCACHE_COOKIE_STATE_DROPPED:
 811                 break;
 812
 813         default:
 814                 WARN_ONCE(1, "Cookie %x in unexpected state %u\n",
 815                           cookie->debug_id, state);
 816                 break;
 817         }
 818
 819 out:
 820         spin_unlock(&cookie->lock);
 821         if (wake)
 822                 wake_up_cookie_state(cookie);
 823         _leave("");
 824 }
 825
 826 static void fscache_cookie_worker(struct work_struct *work)
 827 {
 828         struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work);
 829
 830         fscache_see_cookie(cookie, fscache_cookie_see_work);
 831         fscache_cookie_state_machine(cookie);
 832         fscache_put_cookie(cookie, fscache_cookie_put_work);
 833 }
 834
 835 /*
 836  * Wait for the object to become inactive.  The cookie's work item will be
 837  * scheduled when someone transitions n_accesses to 0 - but if someone's
 838  * already done that, schedule it anyway.
 839  */
 840 static void __fscache_withdraw_cookie(struct fscache_cookie *cookie)
 841 {
 842         int n_accesses;
 843         bool unpinned;
 844
 845         unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
 846
 847         /* Need to read the access count after unpinning */
 848         n_accesses = atomic_read(&cookie->n_accesses);
 849         if (unpinned)
 850                 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
 851                                      n_accesses, fscache_access_cache_unpin);
 852         if (n_accesses == 0)
 853                 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
 854 }
 855
 856 static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie)
 857 {
 858         fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one);
 859
 860         spin_lock(&cookie->lock);
 861         if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE ||
 862             time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) ||
 863             atomic_read(&cookie->n_active) > 0) {
 864                 spin_unlock(&cookie->lock);
 865                 fscache_stat(&fscache_n_cookies_lru_removed);
 866         } else {
 867                 set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
 868                 spin_unlock(&cookie->lock);
 869                 fscache_stat(&fscache_n_cookies_lru_expired);
 870                 _debug("lru c=%x", cookie->debug_id);
 871                 __fscache_withdraw_cookie(cookie);
 872         }
 873
 874         fscache_put_cookie(cookie, fscache_cookie_put_lru);
 875 }
 876
 877 static void fscache_cookie_lru_worker(struct work_struct *work)
 878 {
 879         struct fscache_cookie *cookie;
 880         unsigned long unused_at;
 881
 882         spin_lock(&fscache_cookie_lru_lock);
 883
 884         while (!list_empty(&fscache_cookie_lru)) {
 885                 cookie = list_first_entry(&fscache_cookie_lru,
 886                                           struct fscache_cookie, commit_link);
 887                 unused_at = cookie->unused_at + fscache_lru_cookie_timeout;
 888                 if (time_before(jiffies, unused_at)) {
 889                         timer_reduce(&fscache_cookie_lru_timer, unused_at);
 890                         break;
 891                 }
 892
 893                 list_del_init(&cookie->commit_link);
 894                 fscache_stat_d(&fscache_n_cookies_lru);
 895                 spin_unlock(&fscache_cookie_lru_lock);
 896                 fscache_cookie_lru_do_one(cookie);
 897                 spin_lock(&fscache_cookie_lru_lock);
 898         }
 899
 900         spin_unlock(&fscache_cookie_lru_lock);
 901 }
 902
 903 static void fscache_cookie_lru_timed_out(struct timer_list *timer)
 904 {
 905         queue_work(fscache_wq, &fscache_cookie_lru_work);
 906 }
 907
 908 static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie)
 909 {
 910         bool need_put = false;
 911
 912         if (!list_empty(&cookie->commit_link)) {
 913                 spin_lock(&fscache_cookie_lru_lock);
 914                 if (!list_empty(&cookie->commit_link)) {
 915                         list_del_init(&cookie->commit_link);
 916                         fscache_stat_d(&fscache_n_cookies_lru);
 917                         fscache_stat(&fscache_n_cookies_lru_dropped);
 918                         need_put = true;
 919                 }
 920                 spin_unlock(&fscache_cookie_lru_lock);
 921                 if (need_put)
 922                         fscache_put_cookie(cookie, fscache_cookie_put_lru);
 923         }
 924 }
 925
 926 /*
 927  * Remove a cookie from the hash table.
 928  */
 929 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 930 {
 931         struct hlist_bl_head *h;
 932         unsigned int bucket;
 933
 934         bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 935         h = &fscache_cookie_hash[bucket];
 936
 937         hlist_bl_lock(h);
 938         hlist_bl_del(&cookie->hash_link);
 939         clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags);
 940         hlist_bl_unlock(h);
 941         fscache_stat(&fscache_n_relinquishes_dropped);
 942 }
 943
 944 static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie)
 945 {
 946         fscache_cookie_drop_from_lru(cookie);
 947         __fscache_withdraw_cookie(cookie);
 948 }
 949
 950 /**
 951  * fscache_withdraw_cookie - Mark a cookie for withdrawal
 952  * @cookie: The cookie to be withdrawn.
 953  *
 954  * Allow the cache backend to withdraw the backing for a cookie for its own
 955  * reasons, even if that cookie is in active use.
 956  */
 957 void fscache_withdraw_cookie(struct fscache_cookie *cookie)
 958 {
 959         set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
 960         fscache_drop_withdraw_cookie(cookie);
 961 }
 962 EXPORT_SYMBOL(fscache_withdraw_cookie);
 963
 964 /*
 965  * Allow the netfs to release a cookie back to the cache.
 966  * - the object will be marked as recyclable on disk if retire is true
 967  */
 968 void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
 969 {
 970         fscache_stat(&fscache_n_relinquishes);
 971         if (retire)
 972                 fscache_stat(&fscache_n_relinquishes_retire);
 973
 974         _enter("c=%08x{%d},%d",
 975                cookie->debug_id, atomic_read(&cookie->n_active), retire);
 976
 977         if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
 978                  "Cookie c=%x already relinquished\n", cookie->debug_id))
 979                 return;
 980
 981         if (retire)
 982                 set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags);
 983         trace_fscache_relinquish(cookie, retire);
 984
 985         ASSERTCMP(atomic_read(&cookie->n_active), ==, 0);
 986         ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0);
 987         atomic_dec(&cookie->volume->n_cookies);
 988
 989         if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) {
 990                 set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags);
 991                 fscache_drop_withdraw_cookie(cookie);
 992         } else {
 993                 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED);
 994                 fscache_unhash_cookie(cookie);
 995         }
 996         fscache_put_cookie(cookie, fscache_cookie_put_relinquish);
 997 }
 998 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 999
1000 /*
1001  * Drop a reference to a cookie.
1002  */
1003 void fscache_put_cookie(struct fscache_cookie *cookie,
1004                         enum fscache_cookie_trace where)
1005 {
1006         struct fscache_volume *volume = cookie->volume;
1007         unsigned int cookie_debug_id = cookie->debug_id;
1008         bool zero;
1009         int ref;
1010
1011         zero = __refcount_dec_and_test(&cookie->ref, &ref);
1012         trace_fscache_cookie(cookie_debug_id, ref - 1, where);
1013         if (zero) {
1014                 fscache_free_cookie(cookie);
1015                 fscache_put_volume(volume, fscache_volume_put_cookie);
1016         }
1017 }
1018 EXPORT_SYMBOL(fscache_put_cookie);
1019
1020 /*
1021  * Get a reference to a cookie.
1022  */
1023 struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie,
1024                                           enum fscache_cookie_trace where)
1025 {
1026         int ref;
1027
1028         __refcount_inc(&cookie->ref, &ref);
1029         trace_fscache_cookie(cookie->debug_id, ref + 1, where);
1030         return cookie;
1031 }
1032 EXPORT_SYMBOL(fscache_get_cookie);
1033
1034 /*
1035  * Ask the cache to effect invalidation of a cookie.
1036  */
1037 static void fscache_perform_invalidation(struct fscache_cookie *cookie)
1038 {
1039         if (!cookie->volume->cache->ops->invalidate_cookie(cookie))
1040                 fscache_caching_failed(cookie);
1041         fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
1042 }
1043
1044 /*
1045  * Invalidate an object.
1046  */
1047 void __fscache_invalidate(struct fscache_cookie *cookie,
1048                           const void *aux_data, loff_t new_size,
1049                           unsigned int flags)
1050 {
1051         bool is_caching;
1052
1053         _enter("c=%x", cookie->debug_id);
1054
1055         fscache_stat(&fscache_n_invalidates);
1056
1057         if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
1058                  "Trying to invalidate relinquished cookie\n"))
1059                 return;
1060
1061         if ((flags & FSCACHE_INVAL_DIO_WRITE) &&
1062             test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags))
1063                 return;
1064
1065         spin_lock(&cookie->lock);
1066         set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
1067         fscache_update_aux(cookie, aux_data, &new_size);
1068         cookie->inval_counter++;
1069         trace_fscache_invalidate(cookie, new_size);
1070
1071         switch (cookie->state) {
1072         case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */
1073         default:
1074                 spin_unlock(&cookie->lock);
1075                 _leave(" [no %u]", cookie->state);
1076                 return;
1077
1078         case FSCACHE_COOKIE_STATE_LOOKING_UP:
1079                 if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
1080                         __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie);
1081                 fallthrough;
1082         case FSCACHE_COOKIE_STATE_CREATING:
1083                 spin_unlock(&cookie->lock);
1084                 _leave(" [look %x]", cookie->inval_counter);
1085                 return;
1086
1087         case FSCACHE_COOKIE_STATE_ACTIVE:
1088                 is_caching = fscache_begin_cookie_access(
1089                         cookie, fscache_access_invalidate_cookie);
1090                 if (is_caching)
1091                         __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING);
1092                 spin_unlock(&cookie->lock);
1093                 wake_up_cookie_state(cookie);
1094
1095                 if (is_caching)
1096                         fscache_queue_cookie(cookie, fscache_cookie_get_inval_work);
1097                 _leave(" [inv]");
1098                 return;
1099         }
1100 }
1101 EXPORT_SYMBOL(__fscache_invalidate);
1102
1103 #ifdef CONFIG_PROC_FS
1104 /*
1105  * Generate a list of extant cookies in /proc/fs/fscache/cookies
1106  */
1107 static int fscache_cookies_seq_show(struct seq_file *m, void *v)
1108 {
1109         struct fscache_cookie *cookie;
1110         unsigned int keylen = 0, auxlen = 0;
1111         u8 *p;
1112
1113         if (v == &fscache_cookies) {
1114                 seq_puts(m,
1115                          "COOKIE   VOLUME   REF ACT ACC S FL DEF             \n"
1116                          "======== ======== === === === = == ================\n"
1117                          );
1118                 return 0;
1119         }
1120
1121         cookie = list_entry(v, struct fscache_cookie, proc_link);
1122
1123         seq_printf(m,
1124                    "%08x %08x %3d %3d %3d %c %02lx",
1125                    cookie->debug_id,
1126                    cookie->volume->debug_id,
1127                    refcount_read(&cookie->ref),
1128                    atomic_read(&cookie->n_active),
1129                    atomic_read(&cookie->n_accesses),
1130                    fscache_cookie_states[cookie->state],
1131                    cookie->flags);
1132
1133         keylen = cookie->key_len;
1134         auxlen = cookie->aux_len;
1135
1136         if (keylen > 0 || auxlen > 0) {
1137                 seq_puts(m, " ");
1138                 p = keylen <= sizeof(cookie->inline_key) ?
1139                         cookie->inline_key : cookie->key;
1140                 for (; keylen > 0; keylen--)
1141                         seq_printf(m, "%02x", *p++);
1142                 if (auxlen > 0) {
1143                         seq_puts(m, ", ");
1144                         p = auxlen <= sizeof(cookie->inline_aux) ?
1145                                 cookie->inline_aux : cookie->aux;
1146                         for (; auxlen > 0; auxlen--)
1147                                 seq_printf(m, "%02x", *p++);
1148                 }
1149         }
1150
1151         seq_puts(m, "\n");
1152         return 0;
1153 }
1154
1155 static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
1156         __acquires(fscache_cookies_lock)
1157 {
1158         read_lock(&fscache_cookies_lock);
1159         return seq_list_start_head(&fscache_cookies, *_pos);
1160 }
1161
1162 static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
1163 {
1164         return seq_list_next(v, &fscache_cookies, _pos);
1165 }
1166
1167 static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
1168         __releases(rcu)
1169 {
1170         read_unlock(&fscache_cookies_lock);
1171 }
1172
1173
1174 const struct seq_operations fscache_cookies_seq_ops = {
1175         .start  = fscache_cookies_seq_start,
1176         .next   = fscache_cookies_seq_next,
1177         .stop   = fscache_cookies_seq_stop,
1178         .show   = fscache_cookies_seq_show,
1179 };
1180 #endif