fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 /*
  21  * check to see if a page is being written to the cache
  22  */
  23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  24 {
  25         void *val;
  26
  27         rcu_read_lock();
  28         val = radix_tree_lookup(&cookie->stores, page->index);
  29         rcu_read_unlock();
  30
  31         return val != NULL;
  32 }
  33 EXPORT_SYMBOL(__fscache_check_page_write);
  34
  35 /*
  36  * wait for a page to finish being written to the cache
  37  */
  38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  39 {
  40         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  41
  42         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  43 }
  44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  45
  46 /*
  47  * decide whether a page can be released, possibly by cancelling a store to it
  48  * - we're allowed to sleep if __GFP_WAIT is flagged
  49  */
  50 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  51                                   struct page *page,
  52                                   gfp_t gfp)
  53 {
  54         struct page *xpage;
  55         void *val;
  56
  57         _enter("%p,%p,%x", cookie, page, gfp);
  58
  59 try_again:
  60         rcu_read_lock();
  61         val = radix_tree_lookup(&cookie->stores, page->index);
  62         if (!val) {
  63                 rcu_read_unlock();
  64                 fscache_stat(&fscache_n_store_vmscan_not_storing);
  65                 __fscache_uncache_page(cookie, page);
  66                 return true;
  67         }
  68
  69         /* see if the page is actually undergoing storage - if so we can't get
  70          * rid of it till the cache has finished with it */
  71         if (radix_tree_tag_get(&cookie->stores, page->index,
  72                                FSCACHE_COOKIE_STORING_TAG)) {
  73                 rcu_read_unlock();
  74                 goto page_busy;
  75         }
  76
  77         /* the page is pending storage, so we attempt to cancel the store and
  78          * discard the store request so that the page can be reclaimed */
  79         spin_lock(&cookie->stores_lock);
  80         rcu_read_unlock();
  81
  82         if (radix_tree_tag_get(&cookie->stores, page->index,
  83                                FSCACHE_COOKIE_STORING_TAG)) {
  84                 /* the page started to undergo storage whilst we were looking,
  85                  * so now we can only wait or return */
  86                 spin_unlock(&cookie->stores_lock);
  87                 goto page_busy;
  88         }
  89
  90         xpage = radix_tree_delete(&cookie->stores, page->index);
  91         spin_unlock(&cookie->stores_lock);
  92
  93         if (xpage) {
  94                 fscache_stat(&fscache_n_store_vmscan_cancelled);
  95                 fscache_stat(&fscache_n_store_radix_deletes);
  96                 ASSERTCMP(xpage, ==, page);
  97         } else {
  98                 fscache_stat(&fscache_n_store_vmscan_gone);
  99         }
 100
 101         wake_up_bit(&cookie->flags, 0);
 102         if (xpage)
 103                 page_cache_release(xpage);
 104         __fscache_uncache_page(cookie, page);
 105         return true;
 106
 107 page_busy:
 108         /* We will wait here if we're allowed to, but that could deadlock the
 109          * allocator as the work threads writing to the cache may all end up
 110          * sleeping on memory allocation, so we may need to impose a timeout
 111          * too. */
 112         if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
 113                 fscache_stat(&fscache_n_store_vmscan_busy);
 114                 return false;
 115         }
 116
 117         fscache_stat(&fscache_n_store_vmscan_wait);
 118         __fscache_wait_on_page_write(cookie, page);
 119         gfp &= ~__GFP_WAIT;
 120         goto try_again;
 121 }
 122 EXPORT_SYMBOL(__fscache_maybe_release_page);
 123
 124 /*
 125  * note that a page has finished being written to the cache
 126  */
 127 static void fscache_end_page_write(struct fscache_object *object,
 128                                    struct page *page)
 129 {
 130         struct fscache_cookie *cookie;
 131         struct page *xpage = NULL;
 132
 133         spin_lock(&object->lock);
 134         cookie = object->cookie;
 135         if (cookie) {
 136                 /* delete the page from the tree if it is now no longer
 137                  * pending */
 138                 spin_lock(&cookie->stores_lock);
 139                 radix_tree_tag_clear(&cookie->stores, page->index,
 140                                      FSCACHE_COOKIE_STORING_TAG);
 141                 if (!radix_tree_tag_get(&cookie->stores, page->index,
 142                                         FSCACHE_COOKIE_PENDING_TAG)) {
 143                         fscache_stat(&fscache_n_store_radix_deletes);
 144                         xpage = radix_tree_delete(&cookie->stores, page->index);
 145                 }
 146                 spin_unlock(&cookie->stores_lock);
 147                 wake_up_bit(&cookie->flags, 0);
 148         }
 149         spin_unlock(&object->lock);
 150         if (xpage)
 151                 page_cache_release(xpage);
 152 }
 153
 154 /*
 155  * actually apply the changed attributes to a cache object
 156  */
 157 static void fscache_attr_changed_op(struct fscache_operation *op)
 158 {
 159         struct fscache_object *object = op->object;
 160         int ret;
 161
 162         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 163
 164         fscache_stat(&fscache_n_attr_changed_calls);
 165
 166         if (fscache_object_is_active(object) &&
 167             fscache_use_cookie(object)) {
 168                 fscache_stat(&fscache_n_cop_attr_changed);
 169                 ret = object->cache->ops->attr_changed(object);
 170                 fscache_stat_d(&fscache_n_cop_attr_changed);
 171                 fscache_unuse_cookie(object);
 172                 if (ret < 0)
 173                         fscache_abort_object(object);
 174         }
 175
 176         fscache_op_complete(op, true);
 177         _leave("");
 178 }
 179
 180 /*
 181  * notification that the attributes on an object have changed
 182  */
 183 int __fscache_attr_changed(struct fscache_cookie *cookie)
 184 {
 185         struct fscache_operation *op;
 186         struct fscache_object *object;
 187
 188         _enter("%p", cookie);
 189
 190         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 191
 192         fscache_stat(&fscache_n_attr_changed);
 193
 194         op = kzalloc(sizeof(*op), GFP_KERNEL);
 195         if (!op) {
 196                 fscache_stat(&fscache_n_attr_changed_nomem);
 197                 _leave(" = -ENOMEM");
 198                 return -ENOMEM;
 199         }
 200
 201         fscache_operation_init(op, fscache_attr_changed_op, NULL);
 202         op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE);
 203
 204         spin_lock(&cookie->lock);
 205
 206         if (hlist_empty(&cookie->backing_objects))
 207                 goto nobufs;
 208         object = hlist_entry(cookie->backing_objects.first,
 209                              struct fscache_object, cookie_link);
 210
 211         if (fscache_submit_exclusive_op(object, op) < 0)
 212                 goto nobufs;
 213         spin_unlock(&cookie->lock);
 214         fscache_stat(&fscache_n_attr_changed_ok);
 215         fscache_put_operation(op);
 216         _leave(" = 0");
 217         return 0;
 218
 219 nobufs:
 220         spin_unlock(&cookie->lock);
 221         kfree(op);
 222         fscache_stat(&fscache_n_attr_changed_nobufs);
 223         _leave(" = %d", -ENOBUFS);
 224         return -ENOBUFS;
 225 }
 226 EXPORT_SYMBOL(__fscache_attr_changed);
 227
 228 /*
 229  * release a retrieval op reference
 230  */
 231 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 232 {
 233         struct fscache_retrieval *op =
 234                 container_of(_op, struct fscache_retrieval, op);
 235
 236         _enter("{OP%x}", op->op.debug_id);
 237
 238         ASSERTCMP(atomic_read(&op->n_pages), ==, 0);
 239
 240         fscache_hist(fscache_retrieval_histogram, op->start_time);
 241         if (op->context)
 242                 fscache_put_context(op->op.object->cookie, op->context);
 243
 244         _leave("");
 245 }
 246
 247 /*
 248  * allocate a retrieval op
 249  */
 250 static struct fscache_retrieval *fscache_alloc_retrieval(
 251         struct fscache_cookie *cookie,
 252         struct address_space *mapping,
 253         fscache_rw_complete_t end_io_func,
 254         void *context)
 255 {
 256         struct fscache_retrieval *op;
 257
 258         /* allocate a retrieval operation and attempt to submit it */
 259         op = kzalloc(sizeof(*op), GFP_NOIO);
 260         if (!op) {
 261                 fscache_stat(&fscache_n_retrievals_nomem);
 262                 return NULL;
 263         }
 264
 265         fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
 266         atomic_inc(&cookie->n_active);
 267         op->op.flags    = FSCACHE_OP_MYTHREAD |
 268                 (1UL << FSCACHE_OP_WAITING) |
 269                 (1UL << FSCACHE_OP_UNUSE_COOKIE);
 270         op->mapping     = mapping;
 271         op->end_io_func = end_io_func;
 272         op->context     = context;
 273         op->start_time  = jiffies;
 274         INIT_LIST_HEAD(&op->to_do);
 275         return op;
 276 }
 277
 278 /*
 279  * wait for a deferred lookup to complete
 280  */
 281 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 282 {
 283         unsigned long jif;
 284
 285         _enter("");
 286
 287         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 288                 _leave(" = 0 [imm]");
 289                 return 0;
 290         }
 291
 292         fscache_stat(&fscache_n_retrievals_wait);
 293
 294         jif = jiffies;
 295         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 296                         fscache_wait_bit_interruptible,
 297                         TASK_INTERRUPTIBLE) != 0) {
 298                 fscache_stat(&fscache_n_retrievals_intr);
 299                 _leave(" = -ERESTARTSYS");
 300                 return -ERESTARTSYS;
 301         }
 302
 303         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 304
 305         smp_rmb();
 306         fscache_hist(fscache_retrieval_delay_histogram, jif);
 307         _leave(" = 0 [dly]");
 308         return 0;
 309 }
 310
 311 /*
 312  * Handle cancellation of a pending retrieval op
 313  */
 314 static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
 315 {
 316         struct fscache_retrieval *op =
 317                 container_of(_op, struct fscache_retrieval, op);
 318
 319         atomic_set(&op->n_pages, 0);
 320 }
 321
 322 /*
 323  * wait for an object to become active (or dead)
 324  */
 325 int fscache_wait_for_operation_activation(struct fscache_object *object,
 326                                           struct fscache_operation *op,
 327                                           atomic_t *stat_op_waits,
 328                                           atomic_t *stat_object_dead,
 329                                           void (*do_cancel)(struct fscache_operation *))
 330 {
 331         int ret;
 332
 333         if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
 334                 goto check_if_dead;
 335
 336         _debug(">>> WT");
 337         if (stat_op_waits)
 338                 fscache_stat(stat_op_waits);
 339         if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 340                         fscache_wait_bit_interruptible,
 341                         TASK_INTERRUPTIBLE) != 0) {
 342                 ret = fscache_cancel_op(op, do_cancel);
 343                 if (ret == 0)
 344                         return -ERESTARTSYS;
 345
 346                 /* it's been removed from the pending queue by another party,
 347                  * so we should get to run shortly */
 348                 wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 349                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 350         }
 351         _debug("<<< GO");
 352
 353 check_if_dead:
 354         if (op->state == FSCACHE_OP_ST_CANCELLED) {
 355                 if (stat_object_dead)
 356                         fscache_stat(stat_object_dead);
 357                 _leave(" = -ENOBUFS [cancelled]");
 358                 return -ENOBUFS;
 359         }
 360         if (unlikely(fscache_object_is_dead(object))) {
 361                 pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
 362                 fscache_cancel_op(op, do_cancel);
 363                 if (stat_object_dead)
 364                         fscache_stat(stat_object_dead);
 365                 return -ENOBUFS;
 366         }
 367         return 0;
 368 }
 369
 370 /*
 371  * read a page from the cache or allocate a block in which to store it
 372  * - we return:
 373  *   -ENOMEM    - out of memory, nothing done
 374  *   -ERESTARTSYS - interrupted
 375  *   -ENOBUFS   - no backing object available in which to cache the block
 376  *   -ENODATA   - no data available in the backing object for this block
 377  *   0          - dispatched a read - it'll call end_io_func() when finished
 378  */
 379 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 380                                  struct page *page,
 381                                  fscache_rw_complete_t end_io_func,
 382                                  void *context,
 383                                  gfp_t gfp)
 384 {
 385         struct fscache_retrieval *op;
 386         struct fscache_object *object;
 387         int ret;
 388
 389         _enter("%p,%p,,,", cookie, page);
 390
 391         fscache_stat(&fscache_n_retrievals);
 392
 393         if (hlist_empty(&cookie->backing_objects))
 394                 goto nobufs;
 395
 396         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 397                 _leave(" = -ENOBUFS [invalidating]");
 398                 return -ENOBUFS;
 399         }
 400
 401         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 402         ASSERTCMP(page, !=, NULL);
 403
 404         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 405                 return -ERESTARTSYS;
 406
 407         op = fscache_alloc_retrieval(cookie, page->mapping,
 408                                      end_io_func,context);
 409         if (!op) {
 410                 _leave(" = -ENOMEM");
 411                 return -ENOMEM;
 412         }
 413         atomic_set(&op->n_pages, 1);
 414
 415         spin_lock(&cookie->lock);
 416
 417         if (hlist_empty(&cookie->backing_objects))
 418                 goto nobufs_unlock;
 419         object = hlist_entry(cookie->backing_objects.first,
 420                              struct fscache_object, cookie_link);
 421
 422         ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
 423
 424         atomic_inc(&object->n_reads);
 425         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 426
 427         if (fscache_submit_op(object, &op->op) < 0)
 428                 goto nobufs_unlock_dec;
 429         spin_unlock(&cookie->lock);
 430
 431         fscache_stat(&fscache_n_retrieval_ops);
 432
 433         /* pin the netfs read context in case we need to do the actual netfs
 434          * read because we've encountered a cache read failure */
 435         fscache_get_context(object->cookie, op->context);
 436
 437         /* we wait for the operation to become active, and then process it
 438          * *here*, in this thread, and not in the thread pool */
 439         ret = fscache_wait_for_operation_activation(
 440                 object, &op->op,
 441                 __fscache_stat(&fscache_n_retrieval_op_waits),
 442                 __fscache_stat(&fscache_n_retrievals_object_dead),
 443                 fscache_do_cancel_retrieval);
 444         if (ret < 0)
 445                 goto error;
 446
 447         /* ask the cache to honour the operation */
 448         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 449                 fscache_stat(&fscache_n_cop_allocate_page);
 450                 ret = object->cache->ops->allocate_page(op, page, gfp);
 451                 fscache_stat_d(&fscache_n_cop_allocate_page);
 452                 if (ret == 0)
 453                         ret = -ENODATA;
 454         } else {
 455                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 456                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 457                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 458         }
 459
 460 error:
 461         if (ret == -ENOMEM)
 462                 fscache_stat(&fscache_n_retrievals_nomem);
 463         else if (ret == -ERESTARTSYS)
 464                 fscache_stat(&fscache_n_retrievals_intr);
 465         else if (ret == -ENODATA)
 466                 fscache_stat(&fscache_n_retrievals_nodata);
 467         else if (ret < 0)
 468                 fscache_stat(&fscache_n_retrievals_nobufs);
 469         else
 470                 fscache_stat(&fscache_n_retrievals_ok);
 471
 472         fscache_put_retrieval(op);
 473         _leave(" = %d", ret);
 474         return ret;
 475
 476 nobufs_unlock_dec:
 477         atomic_dec(&object->n_reads);
 478 nobufs_unlock:
 479         spin_unlock(&cookie->lock);
 480         atomic_dec(&cookie->n_active);
 481         kfree(op);
 482 nobufs:
 483         fscache_stat(&fscache_n_retrievals_nobufs);
 484         _leave(" = -ENOBUFS");
 485         return -ENOBUFS;
 486 }
 487 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 488
 489 /*
 490  * read a list of page from the cache or allocate a block in which to store
 491  * them
 492  * - we return:
 493  *   -ENOMEM    - out of memory, some pages may be being read
 494  *   -ERESTARTSYS - interrupted, some pages may be being read
 495  *   -ENOBUFS   - no backing object or space available in which to cache any
 496  *                pages not being read
 497  *   -ENODATA   - no data available in the backing object for some or all of
 498  *                the pages
 499  *   0          - dispatched a read on all pages
 500  *
 501  * end_io_func() will be called for each page read from the cache as it is
 502  * finishes being read
 503  *
 504  * any pages for which a read is dispatched will be removed from pages and
 505  * nr_pages
 506  */
 507 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 508                                   struct address_space *mapping,
 509                                   struct list_head *pages,
 510                                   unsigned *nr_pages,
 511                                   fscache_rw_complete_t end_io_func,
 512                                   void *context,
 513                                   gfp_t gfp)
 514 {
 515         struct fscache_retrieval *op;
 516         struct fscache_object *object;
 517         int ret;
 518
 519         _enter("%p,,%d,,,", cookie, *nr_pages);
 520
 521         fscache_stat(&fscache_n_retrievals);
 522
 523         if (hlist_empty(&cookie->backing_objects))
 524                 goto nobufs;
 525
 526         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 527                 _leave(" = -ENOBUFS [invalidating]");
 528                 return -ENOBUFS;
 529         }
 530
 531         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 532         ASSERTCMP(*nr_pages, >, 0);
 533         ASSERT(!list_empty(pages));
 534
 535         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 536                 return -ERESTARTSYS;
 537
 538         op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
 539         if (!op)
 540                 return -ENOMEM;
 541         atomic_set(&op->n_pages, *nr_pages);
 542
 543         spin_lock(&cookie->lock);
 544
 545         if (hlist_empty(&cookie->backing_objects))
 546                 goto nobufs_unlock;
 547         object = hlist_entry(cookie->backing_objects.first,
 548                              struct fscache_object, cookie_link);
 549
 550         atomic_inc(&object->n_reads);
 551         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 552
 553         if (fscache_submit_op(object, &op->op) < 0)
 554                 goto nobufs_unlock_dec;
 555         spin_unlock(&cookie->lock);
 556
 557         fscache_stat(&fscache_n_retrieval_ops);
 558
 559         /* pin the netfs read context in case we need to do the actual netfs
 560          * read because we've encountered a cache read failure */
 561         fscache_get_context(object->cookie, op->context);
 562
 563         /* we wait for the operation to become active, and then process it
 564          * *here*, in this thread, and not in the thread pool */
 565         ret = fscache_wait_for_operation_activation(
 566                 object, &op->op,
 567                 __fscache_stat(&fscache_n_retrieval_op_waits),
 568                 __fscache_stat(&fscache_n_retrievals_object_dead),
 569                 fscache_do_cancel_retrieval);
 570         if (ret < 0)
 571                 goto error;
 572
 573         /* ask the cache to honour the operation */
 574         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 575                 fscache_stat(&fscache_n_cop_allocate_pages);
 576                 ret = object->cache->ops->allocate_pages(
 577                         op, pages, nr_pages, gfp);
 578                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 579         } else {
 580                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 581                 ret = object->cache->ops->read_or_alloc_pages(
 582                         op, pages, nr_pages, gfp);
 583                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 584         }
 585
 586 error:
 587         if (ret == -ENOMEM)
 588                 fscache_stat(&fscache_n_retrievals_nomem);
 589         else if (ret == -ERESTARTSYS)
 590                 fscache_stat(&fscache_n_retrievals_intr);
 591         else if (ret == -ENODATA)
 592                 fscache_stat(&fscache_n_retrievals_nodata);
 593         else if (ret < 0)
 594                 fscache_stat(&fscache_n_retrievals_nobufs);
 595         else
 596                 fscache_stat(&fscache_n_retrievals_ok);
 597
 598         fscache_put_retrieval(op);
 599         _leave(" = %d", ret);
 600         return ret;
 601
 602 nobufs_unlock_dec:
 603         atomic_dec(&object->n_reads);
 604 nobufs_unlock:
 605         spin_unlock(&cookie->lock);
 606         atomic_dec(&cookie->n_active);
 607         kfree(op);
 608 nobufs:
 609         fscache_stat(&fscache_n_retrievals_nobufs);
 610         _leave(" = -ENOBUFS");
 611         return -ENOBUFS;
 612 }
 613 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 614
 615 /*
 616  * allocate a block in the cache on which to store a page
 617  * - we return:
 618  *   -ENOMEM    - out of memory, nothing done
 619  *   -ERESTARTSYS - interrupted
 620  *   -ENOBUFS   - no backing object available in which to cache the block
 621  *   0          - block allocated
 622  */
 623 int __fscache_alloc_page(struct fscache_cookie *cookie,
 624                          struct page *page,
 625                          gfp_t gfp)
 626 {
 627         struct fscache_retrieval *op;
 628         struct fscache_object *object;
 629         int ret;
 630
 631         _enter("%p,%p,,,", cookie, page);
 632
 633         fscache_stat(&fscache_n_allocs);
 634
 635         if (hlist_empty(&cookie->backing_objects))
 636                 goto nobufs;
 637
 638         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 639         ASSERTCMP(page, !=, NULL);
 640
 641         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 642                 _leave(" = -ENOBUFS [invalidating]");
 643                 return -ENOBUFS;
 644         }
 645
 646         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 647                 return -ERESTARTSYS;
 648
 649         op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
 650         if (!op)
 651                 return -ENOMEM;
 652         atomic_set(&op->n_pages, 1);
 653
 654         spin_lock(&cookie->lock);
 655
 656         if (hlist_empty(&cookie->backing_objects))
 657                 goto nobufs_unlock;
 658         object = hlist_entry(cookie->backing_objects.first,
 659                              struct fscache_object, cookie_link);
 660
 661         if (fscache_submit_op(object, &op->op) < 0)
 662                 goto nobufs_unlock;
 663         spin_unlock(&cookie->lock);
 664
 665         fscache_stat(&fscache_n_alloc_ops);
 666
 667         ret = fscache_wait_for_operation_activation(
 668                 object, &op->op,
 669                 __fscache_stat(&fscache_n_alloc_op_waits),
 670                 __fscache_stat(&fscache_n_allocs_object_dead),
 671                 fscache_do_cancel_retrieval);
 672         if (ret < 0)
 673                 goto error;
 674
 675         /* ask the cache to honour the operation */
 676         fscache_stat(&fscache_n_cop_allocate_page);
 677         ret = object->cache->ops->allocate_page(op, page, gfp);
 678         fscache_stat_d(&fscache_n_cop_allocate_page);
 679
 680 error:
 681         if (ret == -ERESTARTSYS)
 682                 fscache_stat(&fscache_n_allocs_intr);
 683         else if (ret < 0)
 684                 fscache_stat(&fscache_n_allocs_nobufs);
 685         else
 686                 fscache_stat(&fscache_n_allocs_ok);
 687
 688         fscache_put_retrieval(op);
 689         _leave(" = %d", ret);
 690         return ret;
 691
 692 nobufs_unlock:
 693         spin_unlock(&cookie->lock);
 694         atomic_dec(&cookie->n_active);
 695         kfree(op);
 696 nobufs:
 697         fscache_stat(&fscache_n_allocs_nobufs);
 698         _leave(" = -ENOBUFS");
 699         return -ENOBUFS;
 700 }
 701 EXPORT_SYMBOL(__fscache_alloc_page);
 702
 703 /*
 704  * Unmark pages allocate in the readahead code path (via:
 705  * fscache_readpages_or_alloc) after delegating to the base filesystem
 706  */
 707 void __fscache_readpages_cancel(struct fscache_cookie *cookie,
 708                                 struct list_head *pages)
 709 {
 710         struct page *page;
 711
 712         list_for_each_entry(page, pages, lru) {
 713                 if (PageFsCache(page))
 714                         __fscache_uncache_page(cookie, page);
 715         }
 716 }
 717 EXPORT_SYMBOL(__fscache_readpages_cancel);
 718
 719 /*
 720  * release a write op reference
 721  */
 722 static void fscache_release_write_op(struct fscache_operation *_op)
 723 {
 724         _enter("{OP%x}", _op->debug_id);
 725 }
 726
 727 /*
 728  * perform the background storage of a page into the cache
 729  */
 730 static void fscache_write_op(struct fscache_operation *_op)
 731 {
 732         struct fscache_storage *op =
 733                 container_of(_op, struct fscache_storage, op);
 734         struct fscache_object *object = op->op.object;
 735         struct fscache_cookie *cookie;
 736         struct page *page;
 737         unsigned n;
 738         void *results[1];
 739         int ret;
 740
 741         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 742
 743         spin_lock(&object->lock);
 744         cookie = object->cookie;
 745
 746         if (!fscache_object_is_active(object)) {
 747                 /* If we get here, then the on-disk cache object likely longer
 748                  * exists, so we should just cancel this write operation.
 749                  */
 750                 spin_unlock(&object->lock);
 751                 fscache_op_complete(&op->op, false);
 752                 _leave(" [inactive]");
 753                 return;
 754         }
 755
 756         if (!cookie) {
 757                 /* If we get here, then the cookie belonging to the object was
 758                  * detached, probably by the cookie being withdrawn due to
 759                  * memory pressure, which means that the pages we might write
 760                  * to the cache from no longer exist - therefore, we can just
 761                  * cancel this write operation.
 762                  */
 763                 spin_unlock(&object->lock);
 764                 fscache_op_complete(&op->op, false);
 765                 _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
 766                        _op->flags, _op->state, object->state->short_name,
 767                        object->flags);
 768                 return;
 769         }
 770
 771         spin_lock(&cookie->stores_lock);
 772
 773         fscache_stat(&fscache_n_store_calls);
 774
 775         /* find a page to store */
 776         page = NULL;
 777         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 778                                        FSCACHE_COOKIE_PENDING_TAG);
 779         if (n != 1)
 780                 goto superseded;
 781         page = results[0];
 782         _debug("gang %d [%lx]", n, page->index);
 783         if (page->index > op->store_limit) {
 784                 fscache_stat(&fscache_n_store_pages_over_limit);
 785                 goto superseded;
 786         }
 787
 788         radix_tree_tag_set(&cookie->stores, page->index,
 789                            FSCACHE_COOKIE_STORING_TAG);
 790         radix_tree_tag_clear(&cookie->stores, page->index,
 791                              FSCACHE_COOKIE_PENDING_TAG);
 792
 793         spin_unlock(&cookie->stores_lock);
 794         spin_unlock(&object->lock);
 795
 796         fscache_stat(&fscache_n_store_pages);
 797         fscache_stat(&fscache_n_cop_write_page);
 798         ret = object->cache->ops->write_page(op, page);
 799         fscache_stat_d(&fscache_n_cop_write_page);
 800         fscache_end_page_write(object, page);
 801         if (ret < 0) {
 802                 fscache_abort_object(object);
 803                 fscache_op_complete(&op->op, true);
 804         } else {
 805                 fscache_enqueue_operation(&op->op);
 806         }
 807
 808         _leave("");
 809         return;
 810
 811 superseded:
 812         /* this writer is going away and there aren't any more things to
 813          * write */
 814         _debug("cease");
 815         spin_unlock(&cookie->stores_lock);
 816         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 817         spin_unlock(&object->lock);
 818         fscache_op_complete(&op->op, true);
 819         _leave("");
 820 }
 821
 822 /*
 823  * Clear the pages pending writing for invalidation
 824  */
 825 void fscache_invalidate_writes(struct fscache_cookie *cookie)
 826 {
 827         struct page *page;
 828         void *results[16];
 829         int n, i;
 830
 831         _enter("");
 832
 833         for (;;) {
 834                 spin_lock(&cookie->stores_lock);
 835                 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
 836                                                ARRAY_SIZE(results),
 837                                                FSCACHE_COOKIE_PENDING_TAG);
 838                 if (n == 0) {
 839                         spin_unlock(&cookie->stores_lock);
 840                         break;
 841                 }
 842
 843                 for (i = n - 1; i >= 0; i--) {
 844                         page = results[i];
 845                         radix_tree_delete(&cookie->stores, page->index);
 846                 }
 847
 848                 spin_unlock(&cookie->stores_lock);
 849
 850                 for (i = n - 1; i >= 0; i--)
 851                         page_cache_release(results[i]);
 852         }
 853
 854         _leave("");
 855 }
 856
 857 /*
 858  * request a page be stored in the cache
 859  * - returns:
 860  *   -ENOMEM    - out of memory, nothing done
 861  *   -ENOBUFS   - no backing object available in which to cache the page
 862  *   0          - dispatched a write - it'll call end_io_func() when finished
 863  *
 864  * if the cookie still has a backing object at this point, that object can be
 865  * in one of a few states with respect to storage processing:
 866  *
 867  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 868  *      set)
 869  *
 870  *      (a) no writes yet
 871  *
 872  *      (b) writes deferred till post-creation (mark page for writing and
 873  *          return immediately)
 874  *
 875  *  (2) negative lookup, object created, initial fill being made from netfs
 876  *
 877  *      (a) fill point not yet reached this page (mark page for writing and
 878  *          return)
 879  *
 880  *      (b) fill point passed this page (queue op to store this page)
 881  *
 882  *  (3) object extant (queue op to store this page)
 883  *
 884  * any other state is invalid
 885  */
 886 int __fscache_write_page(struct fscache_cookie *cookie,
 887                          struct page *page,
 888                          gfp_t gfp)
 889 {
 890         struct fscache_storage *op;
 891         struct fscache_object *object;
 892         int ret;
 893
 894         _enter("%p,%x,", cookie, (u32) page->flags);
 895
 896         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 897         ASSERT(PageFsCache(page));
 898
 899         fscache_stat(&fscache_n_stores);
 900
 901         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 902                 _leave(" = -ENOBUFS [invalidating]");
 903                 return -ENOBUFS;
 904         }
 905
 906         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 907         if (!op)
 908                 goto nomem;
 909
 910         fscache_operation_init(&op->op, fscache_write_op,
 911                                fscache_release_write_op);
 912         op->op.flags = FSCACHE_OP_ASYNC |
 913                 (1 << FSCACHE_OP_WAITING) |
 914                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 915
 916         ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
 917         if (ret < 0)
 918                 goto nomem_free;
 919
 920         ret = -ENOBUFS;
 921         spin_lock(&cookie->lock);
 922
 923         if (hlist_empty(&cookie->backing_objects))
 924                 goto nobufs;
 925         object = hlist_entry(cookie->backing_objects.first,
 926                              struct fscache_object, cookie_link);
 927         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 928                 goto nobufs;
 929
 930         /* add the page to the pending-storage radix tree on the backing
 931          * object */
 932         spin_lock(&object->lock);
 933         spin_lock(&cookie->stores_lock);
 934
 935         _debug("store limit %llx", (unsigned long long) object->store_limit);
 936
 937         ret = radix_tree_insert(&cookie->stores, page->index, page);
 938         if (ret < 0) {
 939                 if (ret == -EEXIST)
 940                         goto already_queued;
 941                 _debug("insert failed %d", ret);
 942                 goto nobufs_unlock_obj;
 943         }
 944
 945         radix_tree_tag_set(&cookie->stores, page->index,
 946                            FSCACHE_COOKIE_PENDING_TAG);
 947         page_cache_get(page);
 948
 949         /* we only want one writer at a time, but we do need to queue new
 950          * writers after exclusive ops */
 951         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 952                 goto already_pending;
 953
 954         spin_unlock(&cookie->stores_lock);
 955         spin_unlock(&object->lock);
 956
 957         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 958         op->store_limit = object->store_limit;
 959
 960         atomic_inc(&cookie->n_active);
 961         if (fscache_submit_op(object, &op->op) < 0)
 962                 goto submit_failed;
 963
 964         spin_unlock(&cookie->lock);
 965         radix_tree_preload_end();
 966         fscache_stat(&fscache_n_store_ops);
 967         fscache_stat(&fscache_n_stores_ok);
 968
 969         /* the work queue now carries its own ref on the object */
 970         fscache_put_operation(&op->op);
 971         _leave(" = 0");
 972         return 0;
 973
 974 already_queued:
 975         fscache_stat(&fscache_n_stores_again);
 976 already_pending:
 977         spin_unlock(&cookie->stores_lock);
 978         spin_unlock(&object->lock);
 979         spin_unlock(&cookie->lock);
 980         radix_tree_preload_end();
 981         kfree(op);
 982         fscache_stat(&fscache_n_stores_ok);
 983         _leave(" = 0");
 984         return 0;
 985
 986 submit_failed:
 987         atomic_dec(&cookie->n_active);
 988         spin_lock(&cookie->stores_lock);
 989         radix_tree_delete(&cookie->stores, page->index);
 990         spin_unlock(&cookie->stores_lock);
 991         page_cache_release(page);
 992         ret = -ENOBUFS;
 993         goto nobufs;
 994
 995 nobufs_unlock_obj:
 996         spin_unlock(&cookie->stores_lock);
 997         spin_unlock(&object->lock);
 998 nobufs:
 999         spin_unlock(&cookie->lock);
1000         radix_tree_preload_end();
1001         kfree(op);
1002         fscache_stat(&fscache_n_stores_nobufs);
1003         _leave(" = -ENOBUFS");
1004         return -ENOBUFS;
1005
1006 nomem_free:
1007         kfree(op);
1008 nomem:
1009         fscache_stat(&fscache_n_stores_oom);
1010         _leave(" = -ENOMEM");
1011         return -ENOMEM;
1012 }
1013 EXPORT_SYMBOL(__fscache_write_page);
1014
1015 /*
1016  * remove a page from the cache
1017  */
1018 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1019 {
1020         struct fscache_object *object;
1021
1022         _enter(",%p", page);
1023
1024         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1025         ASSERTCMP(page, !=, NULL);
1026
1027         fscache_stat(&fscache_n_uncaches);
1028
1029         /* cache withdrawal may beat us to it */
1030         if (!PageFsCache(page))
1031                 goto done;
1032
1033         /* get the object */
1034         spin_lock(&cookie->lock);
1035
1036         if (hlist_empty(&cookie->backing_objects)) {
1037                 ClearPageFsCache(page);
1038                 goto done_unlock;
1039         }
1040
1041         object = hlist_entry(cookie->backing_objects.first,
1042                              struct fscache_object, cookie_link);
1043
1044         /* there might now be stuff on disk we could read */
1045         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1046
1047         /* only invoke the cache backend if we managed to mark the page
1048          * uncached here; this deals with synchronisation vs withdrawal */
1049         if (TestClearPageFsCache(page) &&
1050             object->cache->ops->uncache_page) {
1051                 /* the cache backend releases the cookie lock */
1052                 fscache_stat(&fscache_n_cop_uncache_page);
1053                 object->cache->ops->uncache_page(object, page);
1054                 fscache_stat_d(&fscache_n_cop_uncache_page);
1055                 goto done;
1056         }
1057
1058 done_unlock:
1059         spin_unlock(&cookie->lock);
1060 done:
1061         _leave("");
1062 }
1063 EXPORT_SYMBOL(__fscache_uncache_page);
1064
1065 /**
1066  * fscache_mark_page_cached - Mark a page as being cached
1067  * @op: The retrieval op pages are being marked for
1068  * @page: The page to be marked
1069  *
1070  * Mark a netfs page as being cached.  After this is called, the netfs
1071  * must call fscache_uncache_page() to remove the mark.
1072  */
1073 void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1074 {
1075         struct fscache_cookie *cookie = op->op.object->cookie;
1076
1077 #ifdef CONFIG_FSCACHE_STATS
1078         atomic_inc(&fscache_n_marks);
1079 #endif
1080
1081         _debug("- mark %p{%lx}", page, page->index);
1082         if (TestSetPageFsCache(page)) {
1083                 static bool once_only;
1084                 if (!once_only) {
1085                         once_only = true;
1086                         printk(KERN_WARNING "FS-Cache:"
1087                                " Cookie type %s marked page %lx"
1088                                " multiple times\n",
1089                                cookie->def->name, page->index);
1090                 }
1091         }
1092
1093         if (cookie->def->mark_page_cached)
1094                 cookie->def->mark_page_cached(cookie->netfs_data,
1095                                               op->mapping, page);
1096 }
1097 EXPORT_SYMBOL(fscache_mark_page_cached);
1098
1099 /**
1100  * fscache_mark_pages_cached - Mark pages as being cached
1101  * @op: The retrieval op pages are being marked for
1102  * @pagevec: The pages to be marked
1103  *
1104  * Mark a bunch of netfs pages as being cached.  After this is called,
1105  * the netfs must call fscache_uncache_page() to remove the mark.
1106  */
1107 void fscache_mark_pages_cached(struct fscache_retrieval *op,
1108                                struct pagevec *pagevec)
1109 {
1110         unsigned long loop;
1111
1112         for (loop = 0; loop < pagevec->nr; loop++)
1113                 fscache_mark_page_cached(op, pagevec->pages[loop]);
1114
1115         pagevec_reinit(pagevec);
1116 }
1117 EXPORT_SYMBOL(fscache_mark_pages_cached);
1118
1119 /*
1120  * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1121  * to be associated with the given cookie.
1122  */
1123 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1124                                        struct inode *inode)
1125 {
1126         struct address_space *mapping = inode->i_mapping;
1127         struct pagevec pvec;
1128         pgoff_t next;
1129         int i;
1130
1131         _enter("%p,%p", cookie, inode);
1132
1133         if (!mapping || mapping->nrpages == 0) {
1134                 _leave(" [no pages]");
1135                 return;
1136         }
1137
1138         pagevec_init(&pvec, 0);
1139         next = 0;
1140         do {
1141                 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
1142                         break;
1143                 for (i = 0; i < pagevec_count(&pvec); i++) {
1144                         struct page *page = pvec.pages[i];
1145                         next = page->index;
1146                         if (PageFsCache(page)) {
1147                                 __fscache_wait_on_page_write(cookie, page);
1148                                 __fscache_uncache_page(cookie, page);
1149                         }
1150                 }
1151                 pagevec_release(&pvec);
1152                 cond_resched();
1153         } while (++next);
1154
1155         _leave("");
1156 }
1157 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);