fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 /*
  21  * check to see if a page is being written to the cache
  22  */
  23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  24 {
  25         void *val;
  26
  27         rcu_read_lock();
  28         val = radix_tree_lookup(&cookie->stores, page->index);
  29         rcu_read_unlock();
  30
  31         return val != NULL;
  32 }
  33 EXPORT_SYMBOL(__fscache_check_page_write);
  34
  35 /*
  36  * wait for a page to finish being written to the cache
  37  */
  38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  39 {
  40         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  41
  42         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  43 }
  44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  45
  46 /*
  47  * decide whether a page can be released, possibly by cancelling a store to it
  48  * - we're allowed to sleep if __GFP_WAIT is flagged
  49  */
  50 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  51                                   struct page *page,
  52                                   gfp_t gfp)
  53 {
  54         struct page *xpage;
  55         void *val;
  56
  57         _enter("%p,%p,%x", cookie, page, gfp);
  58
  59 try_again:
  60         rcu_read_lock();
  61         val = radix_tree_lookup(&cookie->stores, page->index);
  62         if (!val) {
  63                 rcu_read_unlock();
  64                 fscache_stat(&fscache_n_store_vmscan_not_storing);
  65                 __fscache_uncache_page(cookie, page);
  66                 return true;
  67         }
  68
  69         /* see if the page is actually undergoing storage - if so we can't get
  70          * rid of it till the cache has finished with it */
  71         if (radix_tree_tag_get(&cookie->stores, page->index,
  72                                FSCACHE_COOKIE_STORING_TAG)) {
  73                 rcu_read_unlock();
  74                 goto page_busy;
  75         }
  76
  77         /* the page is pending storage, so we attempt to cancel the store and
  78          * discard the store request so that the page can be reclaimed */
  79         spin_lock(&cookie->stores_lock);
  80         rcu_read_unlock();
  81
  82         if (radix_tree_tag_get(&cookie->stores, page->index,
  83                                FSCACHE_COOKIE_STORING_TAG)) {
  84                 /* the page started to undergo storage whilst we were looking,
  85                  * so now we can only wait or return */
  86                 spin_unlock(&cookie->stores_lock);
  87                 goto page_busy;
  88         }
  89
  90         xpage = radix_tree_delete(&cookie->stores, page->index);
  91         spin_unlock(&cookie->stores_lock);
  92
  93         if (xpage) {
  94                 fscache_stat(&fscache_n_store_vmscan_cancelled);
  95                 fscache_stat(&fscache_n_store_radix_deletes);
  96                 ASSERTCMP(xpage, ==, page);
  97         } else {
  98                 fscache_stat(&fscache_n_store_vmscan_gone);
  99         }
 100
 101         wake_up_bit(&cookie->flags, 0);
 102         if (xpage)
 103                 page_cache_release(xpage);
 104         __fscache_uncache_page(cookie, page);
 105         return true;
 106
 107 page_busy:
 108         /* We will wait here if we're allowed to, but that could deadlock the
 109          * allocator as the work threads writing to the cache may all end up
 110          * sleeping on memory allocation, so we may need to impose a timeout
 111          * too. */
 112         if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
 113                 fscache_stat(&fscache_n_store_vmscan_busy);
 114                 return false;
 115         }
 116
 117         fscache_stat(&fscache_n_store_vmscan_wait);
 118         __fscache_wait_on_page_write(cookie, page);
 119         gfp &= ~__GFP_WAIT;
 120         goto try_again;
 121 }
 122 EXPORT_SYMBOL(__fscache_maybe_release_page);
 123
 124 /*
 125  * note that a page has finished being written to the cache
 126  */
 127 static void fscache_end_page_write(struct fscache_object *object,
 128                                    struct page *page)
 129 {
 130         struct fscache_cookie *cookie;
 131         struct page *xpage = NULL;
 132
 133         spin_lock(&object->lock);
 134         cookie = object->cookie;
 135         if (cookie) {
 136                 /* delete the page from the tree if it is now no longer
 137                  * pending */
 138                 spin_lock(&cookie->stores_lock);
 139                 radix_tree_tag_clear(&cookie->stores, page->index,
 140                                      FSCACHE_COOKIE_STORING_TAG);
 141                 if (!radix_tree_tag_get(&cookie->stores, page->index,
 142                                         FSCACHE_COOKIE_PENDING_TAG)) {
 143                         fscache_stat(&fscache_n_store_radix_deletes);
 144                         xpage = radix_tree_delete(&cookie->stores, page->index);
 145                 }
 146                 spin_unlock(&cookie->stores_lock);
 147                 wake_up_bit(&cookie->flags, 0);
 148         }
 149         spin_unlock(&object->lock);
 150         if (xpage)
 151                 page_cache_release(xpage);
 152 }
 153
 154 /*
 155  * actually apply the changed attributes to a cache object
 156  */
 157 static void fscache_attr_changed_op(struct fscache_operation *op)
 158 {
 159         struct fscache_object *object = op->object;
 160         int ret;
 161
 162         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 163
 164         fscache_stat(&fscache_n_attr_changed_calls);
 165
 166         if (fscache_object_is_active(object)) {
 167                 fscache_stat(&fscache_n_cop_attr_changed);
 168                 ret = object->cache->ops->attr_changed(object);
 169                 fscache_stat_d(&fscache_n_cop_attr_changed);
 170                 if (ret < 0)
 171                         fscache_abort_object(object);
 172         }
 173
 174         fscache_op_complete(op, true);
 175         _leave("");
 176 }
 177
 178 /*
 179  * notification that the attributes on an object have changed
 180  */
 181 int __fscache_attr_changed(struct fscache_cookie *cookie)
 182 {
 183         struct fscache_operation *op;
 184         struct fscache_object *object;
 185         bool wake_cookie;
 186
 187         _enter("%p", cookie);
 188
 189         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 190
 191         fscache_stat(&fscache_n_attr_changed);
 192
 193         op = kzalloc(sizeof(*op), GFP_KERNEL);
 194         if (!op) {
 195                 fscache_stat(&fscache_n_attr_changed_nomem);
 196                 _leave(" = -ENOMEM");
 197                 return -ENOMEM;
 198         }
 199
 200         fscache_operation_init(op, fscache_attr_changed_op, NULL);
 201         op->flags = FSCACHE_OP_ASYNC |
 202                 (1 << FSCACHE_OP_EXCLUSIVE) |
 203                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 204
 205         spin_lock(&cookie->lock);
 206
 207         if (hlist_empty(&cookie->backing_objects))
 208                 goto nobufs;
 209         object = hlist_entry(cookie->backing_objects.first,
 210                              struct fscache_object, cookie_link);
 211
 212         __fscache_use_cookie(cookie);
 213         if (fscache_submit_exclusive_op(object, op) < 0)
 214                 goto nobufs;
 215         spin_unlock(&cookie->lock);
 216         fscache_stat(&fscache_n_attr_changed_ok);
 217         fscache_put_operation(op);
 218         _leave(" = 0");
 219         return 0;
 220
 221 nobufs:
 222         wake_cookie = __fscache_unuse_cookie(cookie);
 223         spin_unlock(&cookie->lock);
 224         kfree(op);
 225         if (wake_cookie)
 226                 __fscache_wake_unused_cookie(cookie);
 227         fscache_stat(&fscache_n_attr_changed_nobufs);
 228         _leave(" = %d", -ENOBUFS);
 229         return -ENOBUFS;
 230 }
 231 EXPORT_SYMBOL(__fscache_attr_changed);
 232
 233 /*
 234  * release a retrieval op reference
 235  */
 236 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 237 {
 238         struct fscache_retrieval *op =
 239                 container_of(_op, struct fscache_retrieval, op);
 240
 241         _enter("{OP%x}", op->op.debug_id);
 242
 243         ASSERTCMP(atomic_read(&op->n_pages), ==, 0);
 244
 245         fscache_hist(fscache_retrieval_histogram, op->start_time);
 246         if (op->context)
 247                 fscache_put_context(op->op.object->cookie, op->context);
 248
 249         _leave("");
 250 }
 251
 252 /*
 253  * allocate a retrieval op
 254  */
 255 static struct fscache_retrieval *fscache_alloc_retrieval(
 256         struct fscache_cookie *cookie,
 257         struct address_space *mapping,
 258         fscache_rw_complete_t end_io_func,
 259         void *context)
 260 {
 261         struct fscache_retrieval *op;
 262
 263         /* allocate a retrieval operation and attempt to submit it */
 264         op = kzalloc(sizeof(*op), GFP_NOIO);
 265         if (!op) {
 266                 fscache_stat(&fscache_n_retrievals_nomem);
 267                 return NULL;
 268         }
 269
 270         fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
 271         op->op.flags    = FSCACHE_OP_MYTHREAD |
 272                 (1UL << FSCACHE_OP_WAITING) |
 273                 (1UL << FSCACHE_OP_UNUSE_COOKIE);
 274         op->mapping     = mapping;
 275         op->end_io_func = end_io_func;
 276         op->context     = context;
 277         op->start_time  = jiffies;
 278         INIT_LIST_HEAD(&op->to_do);
 279         return op;
 280 }
 281
 282 /*
 283  * wait for a deferred lookup to complete
 284  */
 285 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 286 {
 287         unsigned long jif;
 288
 289         _enter("");
 290
 291         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 292                 _leave(" = 0 [imm]");
 293                 return 0;
 294         }
 295
 296         fscache_stat(&fscache_n_retrievals_wait);
 297
 298         jif = jiffies;
 299         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 300                         fscache_wait_bit_interruptible,
 301                         TASK_INTERRUPTIBLE) != 0) {
 302                 fscache_stat(&fscache_n_retrievals_intr);
 303                 _leave(" = -ERESTARTSYS");
 304                 return -ERESTARTSYS;
 305         }
 306
 307         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 308
 309         smp_rmb();
 310         fscache_hist(fscache_retrieval_delay_histogram, jif);
 311         _leave(" = 0 [dly]");
 312         return 0;
 313 }
 314
 315 /*
 316  * Handle cancellation of a pending retrieval op
 317  */
 318 static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
 319 {
 320         struct fscache_retrieval *op =
 321                 container_of(_op, struct fscache_retrieval, op);
 322
 323         atomic_set(&op->n_pages, 0);
 324 }
 325
 326 /*
 327  * wait for an object to become active (or dead)
 328  */
 329 int fscache_wait_for_operation_activation(struct fscache_object *object,
 330                                           struct fscache_operation *op,
 331                                           atomic_t *stat_op_waits,
 332                                           atomic_t *stat_object_dead,
 333                                           void (*do_cancel)(struct fscache_operation *))
 334 {
 335         int ret;
 336
 337         if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
 338                 goto check_if_dead;
 339
 340         _debug(">>> WT");
 341         if (stat_op_waits)
 342                 fscache_stat(stat_op_waits);
 343         if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 344                         fscache_wait_bit_interruptible,
 345                         TASK_INTERRUPTIBLE) != 0) {
 346                 ret = fscache_cancel_op(op, do_cancel);
 347                 if (ret == 0)
 348                         return -ERESTARTSYS;
 349
 350                 /* it's been removed from the pending queue by another party,
 351                  * so we should get to run shortly */
 352                 wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 353                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 354         }
 355         _debug("<<< GO");
 356
 357 check_if_dead:
 358         if (op->state == FSCACHE_OP_ST_CANCELLED) {
 359                 if (stat_object_dead)
 360                         fscache_stat(stat_object_dead);
 361                 _leave(" = -ENOBUFS [cancelled]");
 362                 return -ENOBUFS;
 363         }
 364         if (unlikely(fscache_object_is_dead(object))) {
 365                 pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
 366                 fscache_cancel_op(op, do_cancel);
 367                 if (stat_object_dead)
 368                         fscache_stat(stat_object_dead);
 369                 return -ENOBUFS;
 370         }
 371         return 0;
 372 }
 373
 374 /*
 375  * read a page from the cache or allocate a block in which to store it
 376  * - we return:
 377  *   -ENOMEM    - out of memory, nothing done
 378  *   -ERESTARTSYS - interrupted
 379  *   -ENOBUFS   - no backing object available in which to cache the block
 380  *   -ENODATA   - no data available in the backing object for this block
 381  *   0          - dispatched a read - it'll call end_io_func() when finished
 382  */
 383 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 384                                  struct page *page,
 385                                  fscache_rw_complete_t end_io_func,
 386                                  void *context,
 387                                  gfp_t gfp)
 388 {
 389         struct fscache_retrieval *op;
 390         struct fscache_object *object;
 391         bool wake_cookie = false;
 392         int ret;
 393
 394         _enter("%p,%p,,,", cookie, page);
 395
 396         fscache_stat(&fscache_n_retrievals);
 397
 398         if (hlist_empty(&cookie->backing_objects))
 399                 goto nobufs;
 400
 401         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 402                 _leave(" = -ENOBUFS [invalidating]");
 403                 return -ENOBUFS;
 404         }
 405
 406         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 407         ASSERTCMP(page, !=, NULL);
 408
 409         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 410                 return -ERESTARTSYS;
 411
 412         op = fscache_alloc_retrieval(cookie, page->mapping,
 413                                      end_io_func,context);
 414         if (!op) {
 415                 _leave(" = -ENOMEM");
 416                 return -ENOMEM;
 417         }
 418         atomic_set(&op->n_pages, 1);
 419
 420         spin_lock(&cookie->lock);
 421
 422         if (hlist_empty(&cookie->backing_objects))
 423                 goto nobufs_unlock;
 424         object = hlist_entry(cookie->backing_objects.first,
 425                              struct fscache_object, cookie_link);
 426
 427         ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
 428
 429         __fscache_use_cookie(cookie);
 430         atomic_inc(&object->n_reads);
 431         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 432
 433         if (fscache_submit_op(object, &op->op) < 0)
 434                 goto nobufs_unlock_dec;
 435         spin_unlock(&cookie->lock);
 436
 437         fscache_stat(&fscache_n_retrieval_ops);
 438
 439         /* pin the netfs read context in case we need to do the actual netfs
 440          * read because we've encountered a cache read failure */
 441         fscache_get_context(object->cookie, op->context);
 442
 443         /* we wait for the operation to become active, and then process it
 444          * *here*, in this thread, and not in the thread pool */
 445         ret = fscache_wait_for_operation_activation(
 446                 object, &op->op,
 447                 __fscache_stat(&fscache_n_retrieval_op_waits),
 448                 __fscache_stat(&fscache_n_retrievals_object_dead),
 449                 fscache_do_cancel_retrieval);
 450         if (ret < 0)
 451                 goto error;
 452
 453         /* ask the cache to honour the operation */
 454         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 455                 fscache_stat(&fscache_n_cop_allocate_page);
 456                 ret = object->cache->ops->allocate_page(op, page, gfp);
 457                 fscache_stat_d(&fscache_n_cop_allocate_page);
 458                 if (ret == 0)
 459                         ret = -ENODATA;
 460         } else {
 461                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 462                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 463                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 464         }
 465
 466 error:
 467         if (ret == -ENOMEM)
 468                 fscache_stat(&fscache_n_retrievals_nomem);
 469         else if (ret == -ERESTARTSYS)
 470                 fscache_stat(&fscache_n_retrievals_intr);
 471         else if (ret == -ENODATA)
 472                 fscache_stat(&fscache_n_retrievals_nodata);
 473         else if (ret < 0)
 474                 fscache_stat(&fscache_n_retrievals_nobufs);
 475         else
 476                 fscache_stat(&fscache_n_retrievals_ok);
 477
 478         fscache_put_retrieval(op);
 479         _leave(" = %d", ret);
 480         return ret;
 481
 482 nobufs_unlock_dec:
 483         atomic_dec(&object->n_reads);
 484         wake_cookie = __fscache_unuse_cookie(cookie);
 485 nobufs_unlock:
 486         spin_unlock(&cookie->lock);
 487         if (wake_cookie)
 488                 __fscache_wake_unused_cookie(cookie);
 489         kfree(op);
 490 nobufs:
 491         fscache_stat(&fscache_n_retrievals_nobufs);
 492         _leave(" = -ENOBUFS");
 493         return -ENOBUFS;
 494 }
 495 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 496
 497 /*
 498  * read a list of page from the cache or allocate a block in which to store
 499  * them
 500  * - we return:
 501  *   -ENOMEM    - out of memory, some pages may be being read
 502  *   -ERESTARTSYS - interrupted, some pages may be being read
 503  *   -ENOBUFS   - no backing object or space available in which to cache any
 504  *                pages not being read
 505  *   -ENODATA   - no data available in the backing object for some or all of
 506  *                the pages
 507  *   0          - dispatched a read on all pages
 508  *
 509  * end_io_func() will be called for each page read from the cache as it is
 510  * finishes being read
 511  *
 512  * any pages for which a read is dispatched will be removed from pages and
 513  * nr_pages
 514  */
 515 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 516                                   struct address_space *mapping,
 517                                   struct list_head *pages,
 518                                   unsigned *nr_pages,
 519                                   fscache_rw_complete_t end_io_func,
 520                                   void *context,
 521                                   gfp_t gfp)
 522 {
 523         struct fscache_retrieval *op;
 524         struct fscache_object *object;
 525         bool wake_cookie = false;
 526         int ret;
 527
 528         _enter("%p,,%d,,,", cookie, *nr_pages);
 529
 530         fscache_stat(&fscache_n_retrievals);
 531
 532         if (hlist_empty(&cookie->backing_objects))
 533                 goto nobufs;
 534
 535         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 536                 _leave(" = -ENOBUFS [invalidating]");
 537                 return -ENOBUFS;
 538         }
 539
 540         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 541         ASSERTCMP(*nr_pages, >, 0);
 542         ASSERT(!list_empty(pages));
 543
 544         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 545                 return -ERESTARTSYS;
 546
 547         op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
 548         if (!op)
 549                 return -ENOMEM;
 550         atomic_set(&op->n_pages, *nr_pages);
 551
 552         spin_lock(&cookie->lock);
 553
 554         if (hlist_empty(&cookie->backing_objects))
 555                 goto nobufs_unlock;
 556         object = hlist_entry(cookie->backing_objects.first,
 557                              struct fscache_object, cookie_link);
 558
 559         __fscache_use_cookie(cookie);
 560         atomic_inc(&object->n_reads);
 561         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 562
 563         if (fscache_submit_op(object, &op->op) < 0)
 564                 goto nobufs_unlock_dec;
 565         spin_unlock(&cookie->lock);
 566
 567         fscache_stat(&fscache_n_retrieval_ops);
 568
 569         /* pin the netfs read context in case we need to do the actual netfs
 570          * read because we've encountered a cache read failure */
 571         fscache_get_context(object->cookie, op->context);
 572
 573         /* we wait for the operation to become active, and then process it
 574          * *here*, in this thread, and not in the thread pool */
 575         ret = fscache_wait_for_operation_activation(
 576                 object, &op->op,
 577                 __fscache_stat(&fscache_n_retrieval_op_waits),
 578                 __fscache_stat(&fscache_n_retrievals_object_dead),
 579                 fscache_do_cancel_retrieval);
 580         if (ret < 0)
 581                 goto error;
 582
 583         /* ask the cache to honour the operation */
 584         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 585                 fscache_stat(&fscache_n_cop_allocate_pages);
 586                 ret = object->cache->ops->allocate_pages(
 587                         op, pages, nr_pages, gfp);
 588                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 589         } else {
 590                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 591                 ret = object->cache->ops->read_or_alloc_pages(
 592                         op, pages, nr_pages, gfp);
 593                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 594         }
 595
 596 error:
 597         if (ret == -ENOMEM)
 598                 fscache_stat(&fscache_n_retrievals_nomem);
 599         else if (ret == -ERESTARTSYS)
 600                 fscache_stat(&fscache_n_retrievals_intr);
 601         else if (ret == -ENODATA)
 602                 fscache_stat(&fscache_n_retrievals_nodata);
 603         else if (ret < 0)
 604                 fscache_stat(&fscache_n_retrievals_nobufs);
 605         else
 606                 fscache_stat(&fscache_n_retrievals_ok);
 607
 608         fscache_put_retrieval(op);
 609         _leave(" = %d", ret);
 610         return ret;
 611
 612 nobufs_unlock_dec:
 613         atomic_dec(&object->n_reads);
 614         wake_cookie = __fscache_unuse_cookie(cookie);
 615 nobufs_unlock:
 616         spin_unlock(&cookie->lock);
 617         kfree(op);
 618         if (wake_cookie)
 619                 __fscache_wake_unused_cookie(cookie);
 620 nobufs:
 621         fscache_stat(&fscache_n_retrievals_nobufs);
 622         _leave(" = -ENOBUFS");
 623         return -ENOBUFS;
 624 }
 625 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 626
 627 /*
 628  * allocate a block in the cache on which to store a page
 629  * - we return:
 630  *   -ENOMEM    - out of memory, nothing done
 631  *   -ERESTARTSYS - interrupted
 632  *   -ENOBUFS   - no backing object available in which to cache the block
 633  *   0          - block allocated
 634  */
 635 int __fscache_alloc_page(struct fscache_cookie *cookie,
 636                          struct page *page,
 637                          gfp_t gfp)
 638 {
 639         struct fscache_retrieval *op;
 640         struct fscache_object *object;
 641         bool wake_cookie = false;
 642         int ret;
 643
 644         _enter("%p,%p,,,", cookie, page);
 645
 646         fscache_stat(&fscache_n_allocs);
 647
 648         if (hlist_empty(&cookie->backing_objects))
 649                 goto nobufs;
 650
 651         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 652         ASSERTCMP(page, !=, NULL);
 653
 654         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 655                 _leave(" = -ENOBUFS [invalidating]");
 656                 return -ENOBUFS;
 657         }
 658
 659         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 660                 return -ERESTARTSYS;
 661
 662         op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
 663         if (!op)
 664                 return -ENOMEM;
 665         atomic_set(&op->n_pages, 1);
 666
 667         spin_lock(&cookie->lock);
 668
 669         if (hlist_empty(&cookie->backing_objects))
 670                 goto nobufs_unlock;
 671         object = hlist_entry(cookie->backing_objects.first,
 672                              struct fscache_object, cookie_link);
 673
 674         __fscache_use_cookie(cookie);
 675         if (fscache_submit_op(object, &op->op) < 0)
 676                 goto nobufs_unlock_dec;
 677         spin_unlock(&cookie->lock);
 678
 679         fscache_stat(&fscache_n_alloc_ops);
 680
 681         ret = fscache_wait_for_operation_activation(
 682                 object, &op->op,
 683                 __fscache_stat(&fscache_n_alloc_op_waits),
 684                 __fscache_stat(&fscache_n_allocs_object_dead),
 685                 fscache_do_cancel_retrieval);
 686         if (ret < 0)
 687                 goto error;
 688
 689         /* ask the cache to honour the operation */
 690         fscache_stat(&fscache_n_cop_allocate_page);
 691         ret = object->cache->ops->allocate_page(op, page, gfp);
 692         fscache_stat_d(&fscache_n_cop_allocate_page);
 693
 694 error:
 695         if (ret == -ERESTARTSYS)
 696                 fscache_stat(&fscache_n_allocs_intr);
 697         else if (ret < 0)
 698                 fscache_stat(&fscache_n_allocs_nobufs);
 699         else
 700                 fscache_stat(&fscache_n_allocs_ok);
 701
 702         fscache_put_retrieval(op);
 703         _leave(" = %d", ret);
 704         return ret;
 705
 706 nobufs_unlock_dec:
 707         wake_cookie = __fscache_unuse_cookie(cookie);
 708 nobufs_unlock:
 709         spin_unlock(&cookie->lock);
 710         kfree(op);
 711         if (wake_cookie)
 712                 __fscache_wake_unused_cookie(cookie);
 713 nobufs:
 714         fscache_stat(&fscache_n_allocs_nobufs);
 715         _leave(" = -ENOBUFS");
 716         return -ENOBUFS;
 717 }
 718 EXPORT_SYMBOL(__fscache_alloc_page);
 719
 720 /*
 721  * Unmark pages allocate in the readahead code path (via:
 722  * fscache_readpages_or_alloc) after delegating to the base filesystem
 723  */
 724 void __fscache_readpages_cancel(struct fscache_cookie *cookie,
 725                                 struct list_head *pages)
 726 {
 727         struct page *page;
 728
 729         list_for_each_entry(page, pages, lru) {
 730                 if (PageFsCache(page))
 731                         __fscache_uncache_page(cookie, page);
 732         }
 733 }
 734 EXPORT_SYMBOL(__fscache_readpages_cancel);
 735
 736 /*
 737  * release a write op reference
 738  */
 739 static void fscache_release_write_op(struct fscache_operation *_op)
 740 {
 741         _enter("{OP%x}", _op->debug_id);
 742 }
 743
 744 /*
 745  * perform the background storage of a page into the cache
 746  */
 747 static void fscache_write_op(struct fscache_operation *_op)
 748 {
 749         struct fscache_storage *op =
 750                 container_of(_op, struct fscache_storage, op);
 751         struct fscache_object *object = op->op.object;
 752         struct fscache_cookie *cookie;
 753         struct page *page;
 754         unsigned n;
 755         void *results[1];
 756         int ret;
 757
 758         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 759
 760         spin_lock(&object->lock);
 761         cookie = object->cookie;
 762
 763         if (!fscache_object_is_active(object)) {
 764                 /* If we get here, then the on-disk cache object likely longer
 765                  * exists, so we should just cancel this write operation.
 766                  */
 767                 spin_unlock(&object->lock);
 768                 fscache_op_complete(&op->op, false);
 769                 _leave(" [inactive]");
 770                 return;
 771         }
 772
 773         if (!cookie) {
 774                 /* If we get here, then the cookie belonging to the object was
 775                  * detached, probably by the cookie being withdrawn due to
 776                  * memory pressure, which means that the pages we might write
 777                  * to the cache from no longer exist - therefore, we can just
 778                  * cancel this write operation.
 779                  */
 780                 spin_unlock(&object->lock);
 781                 fscache_op_complete(&op->op, false);
 782                 _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
 783                        _op->flags, _op->state, object->state->short_name,
 784                        object->flags);
 785                 return;
 786         }
 787
 788         spin_lock(&cookie->stores_lock);
 789
 790         fscache_stat(&fscache_n_store_calls);
 791
 792         /* find a page to store */
 793         page = NULL;
 794         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 795                                        FSCACHE_COOKIE_PENDING_TAG);
 796         if (n != 1)
 797                 goto superseded;
 798         page = results[0];
 799         _debug("gang %d [%lx]", n, page->index);
 800         if (page->index > op->store_limit) {
 801                 fscache_stat(&fscache_n_store_pages_over_limit);
 802                 goto superseded;
 803         }
 804
 805         radix_tree_tag_set(&cookie->stores, page->index,
 806                            FSCACHE_COOKIE_STORING_TAG);
 807         radix_tree_tag_clear(&cookie->stores, page->index,
 808                              FSCACHE_COOKIE_PENDING_TAG);
 809
 810         spin_unlock(&cookie->stores_lock);
 811         spin_unlock(&object->lock);
 812
 813         fscache_stat(&fscache_n_store_pages);
 814         fscache_stat(&fscache_n_cop_write_page);
 815         ret = object->cache->ops->write_page(op, page);
 816         fscache_stat_d(&fscache_n_cop_write_page);
 817         fscache_end_page_write(object, page);
 818         if (ret < 0) {
 819                 fscache_abort_object(object);
 820                 fscache_op_complete(&op->op, true);
 821         } else {
 822                 fscache_enqueue_operation(&op->op);
 823         }
 824
 825         _leave("");
 826         return;
 827
 828 superseded:
 829         /* this writer is going away and there aren't any more things to
 830          * write */
 831         _debug("cease");
 832         spin_unlock(&cookie->stores_lock);
 833         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 834         spin_unlock(&object->lock);
 835         fscache_op_complete(&op->op, true);
 836         _leave("");
 837 }
 838
 839 /*
 840  * Clear the pages pending writing for invalidation
 841  */
 842 void fscache_invalidate_writes(struct fscache_cookie *cookie)
 843 {
 844         struct page *page;
 845         void *results[16];
 846         int n, i;
 847
 848         _enter("");
 849
 850         for (;;) {
 851                 spin_lock(&cookie->stores_lock);
 852                 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
 853                                                ARRAY_SIZE(results),
 854                                                FSCACHE_COOKIE_PENDING_TAG);
 855                 if (n == 0) {
 856                         spin_unlock(&cookie->stores_lock);
 857                         break;
 858                 }
 859
 860                 for (i = n - 1; i >= 0; i--) {
 861                         page = results[i];
 862                         radix_tree_delete(&cookie->stores, page->index);
 863                 }
 864
 865                 spin_unlock(&cookie->stores_lock);
 866
 867                 for (i = n - 1; i >= 0; i--)
 868                         page_cache_release(results[i]);
 869         }
 870
 871         _leave("");
 872 }
 873
 874 /*
 875  * request a page be stored in the cache
 876  * - returns:
 877  *   -ENOMEM    - out of memory, nothing done
 878  *   -ENOBUFS   - no backing object available in which to cache the page
 879  *   0          - dispatched a write - it'll call end_io_func() when finished
 880  *
 881  * if the cookie still has a backing object at this point, that object can be
 882  * in one of a few states with respect to storage processing:
 883  *
 884  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 885  *      set)
 886  *
 887  *      (a) no writes yet
 888  *
 889  *      (b) writes deferred till post-creation (mark page for writing and
 890  *          return immediately)
 891  *
 892  *  (2) negative lookup, object created, initial fill being made from netfs
 893  *
 894  *      (a) fill point not yet reached this page (mark page for writing and
 895  *          return)
 896  *
 897  *      (b) fill point passed this page (queue op to store this page)
 898  *
 899  *  (3) object extant (queue op to store this page)
 900  *
 901  * any other state is invalid
 902  */
 903 int __fscache_write_page(struct fscache_cookie *cookie,
 904                          struct page *page,
 905                          gfp_t gfp)
 906 {
 907         struct fscache_storage *op;
 908         struct fscache_object *object;
 909         bool wake_cookie = false;
 910         int ret;
 911
 912         _enter("%p,%x,", cookie, (u32) page->flags);
 913
 914         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 915         ASSERT(PageFsCache(page));
 916
 917         fscache_stat(&fscache_n_stores);
 918
 919         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 920                 _leave(" = -ENOBUFS [invalidating]");
 921                 return -ENOBUFS;
 922         }
 923
 924         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 925         if (!op)
 926                 goto nomem;
 927
 928         fscache_operation_init(&op->op, fscache_write_op,
 929                                fscache_release_write_op);
 930         op->op.flags = FSCACHE_OP_ASYNC |
 931                 (1 << FSCACHE_OP_WAITING) |
 932                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 933
 934         ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
 935         if (ret < 0)
 936                 goto nomem_free;
 937
 938         ret = -ENOBUFS;
 939         spin_lock(&cookie->lock);
 940
 941         if (hlist_empty(&cookie->backing_objects))
 942                 goto nobufs;
 943         object = hlist_entry(cookie->backing_objects.first,
 944                              struct fscache_object, cookie_link);
 945         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 946                 goto nobufs;
 947
 948         /* add the page to the pending-storage radix tree on the backing
 949          * object */
 950         spin_lock(&object->lock);
 951         spin_lock(&cookie->stores_lock);
 952
 953         _debug("store limit %llx", (unsigned long long) object->store_limit);
 954
 955         ret = radix_tree_insert(&cookie->stores, page->index, page);
 956         if (ret < 0) {
 957                 if (ret == -EEXIST)
 958                         goto already_queued;
 959                 _debug("insert failed %d", ret);
 960                 goto nobufs_unlock_obj;
 961         }
 962
 963         radix_tree_tag_set(&cookie->stores, page->index,
 964                            FSCACHE_COOKIE_PENDING_TAG);
 965         page_cache_get(page);
 966
 967         /* we only want one writer at a time, but we do need to queue new
 968          * writers after exclusive ops */
 969         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 970                 goto already_pending;
 971
 972         spin_unlock(&cookie->stores_lock);
 973         spin_unlock(&object->lock);
 974
 975         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 976         op->store_limit = object->store_limit;
 977
 978         __fscache_use_cookie(cookie);
 979         if (fscache_submit_op(object, &op->op) < 0)
 980                 goto submit_failed;
 981
 982         spin_unlock(&cookie->lock);
 983         radix_tree_preload_end();
 984         fscache_stat(&fscache_n_store_ops);
 985         fscache_stat(&fscache_n_stores_ok);
 986
 987         /* the work queue now carries its own ref on the object */
 988         fscache_put_operation(&op->op);
 989         _leave(" = 0");
 990         return 0;
 991
 992 already_queued:
 993         fscache_stat(&fscache_n_stores_again);
 994 already_pending:
 995         spin_unlock(&cookie->stores_lock);
 996         spin_unlock(&object->lock);
 997         spin_unlock(&cookie->lock);
 998         radix_tree_preload_end();
 999         kfree(op);
1000         fscache_stat(&fscache_n_stores_ok);
1001         _leave(" = 0");
1002         return 0;
1003
1004 submit_failed:
1005         spin_lock(&cookie->stores_lock);
1006         radix_tree_delete(&cookie->stores, page->index);
1007         spin_unlock(&cookie->stores_lock);
1008         wake_cookie = __fscache_unuse_cookie(cookie);
1009         page_cache_release(page);
1010         ret = -ENOBUFS;
1011         goto nobufs;
1012
1013 nobufs_unlock_obj:
1014         spin_unlock(&cookie->stores_lock);
1015         spin_unlock(&object->lock);
1016 nobufs:
1017         spin_unlock(&cookie->lock);
1018         radix_tree_preload_end();
1019         kfree(op);
1020         if (wake_cookie)
1021                 __fscache_wake_unused_cookie(cookie);
1022         fscache_stat(&fscache_n_stores_nobufs);
1023         _leave(" = -ENOBUFS");
1024         return -ENOBUFS;
1025
1026 nomem_free:
1027         kfree(op);
1028 nomem:
1029         fscache_stat(&fscache_n_stores_oom);
1030         _leave(" = -ENOMEM");
1031         return -ENOMEM;
1032 }
1033 EXPORT_SYMBOL(__fscache_write_page);
1034
1035 /*
1036  * remove a page from the cache
1037  */
1038 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1039 {
1040         struct fscache_object *object;
1041
1042         _enter(",%p", page);
1043
1044         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1045         ASSERTCMP(page, !=, NULL);
1046
1047         fscache_stat(&fscache_n_uncaches);
1048
1049         /* cache withdrawal may beat us to it */
1050         if (!PageFsCache(page))
1051                 goto done;
1052
1053         /* get the object */
1054         spin_lock(&cookie->lock);
1055
1056         if (hlist_empty(&cookie->backing_objects)) {
1057                 ClearPageFsCache(page);
1058                 goto done_unlock;
1059         }
1060
1061         object = hlist_entry(cookie->backing_objects.first,
1062                              struct fscache_object, cookie_link);
1063
1064         /* there might now be stuff on disk we could read */
1065         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1066
1067         /* only invoke the cache backend if we managed to mark the page
1068          * uncached here; this deals with synchronisation vs withdrawal */
1069         if (TestClearPageFsCache(page) &&
1070             object->cache->ops->uncache_page) {
1071                 /* the cache backend releases the cookie lock */
1072                 fscache_stat(&fscache_n_cop_uncache_page);
1073                 object->cache->ops->uncache_page(object, page);
1074                 fscache_stat_d(&fscache_n_cop_uncache_page);
1075                 goto done;
1076         }
1077
1078 done_unlock:
1079         spin_unlock(&cookie->lock);
1080 done:
1081         _leave("");
1082 }
1083 EXPORT_SYMBOL(__fscache_uncache_page);
1084
1085 /**
1086  * fscache_mark_page_cached - Mark a page as being cached
1087  * @op: The retrieval op pages are being marked for
1088  * @page: The page to be marked
1089  *
1090  * Mark a netfs page as being cached.  After this is called, the netfs
1091  * must call fscache_uncache_page() to remove the mark.
1092  */
1093 void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1094 {
1095         struct fscache_cookie *cookie = op->op.object->cookie;
1096
1097 #ifdef CONFIG_FSCACHE_STATS
1098         atomic_inc(&fscache_n_marks);
1099 #endif
1100
1101         _debug("- mark %p{%lx}", page, page->index);
1102         if (TestSetPageFsCache(page)) {
1103                 static bool once_only;
1104                 if (!once_only) {
1105                         once_only = true;
1106                         printk(KERN_WARNING "FS-Cache:"
1107                                " Cookie type %s marked page %lx"
1108                                " multiple times\n",
1109                                cookie->def->name, page->index);
1110                 }
1111         }
1112
1113         if (cookie->def->mark_page_cached)
1114                 cookie->def->mark_page_cached(cookie->netfs_data,
1115                                               op->mapping, page);
1116 }
1117 EXPORT_SYMBOL(fscache_mark_page_cached);
1118
1119 /**
1120  * fscache_mark_pages_cached - Mark pages as being cached
1121  * @op: The retrieval op pages are being marked for
1122  * @pagevec: The pages to be marked
1123  *
1124  * Mark a bunch of netfs pages as being cached.  After this is called,
1125  * the netfs must call fscache_uncache_page() to remove the mark.
1126  */
1127 void fscache_mark_pages_cached(struct fscache_retrieval *op,
1128                                struct pagevec *pagevec)
1129 {
1130         unsigned long loop;
1131
1132         for (loop = 0; loop < pagevec->nr; loop++)
1133                 fscache_mark_page_cached(op, pagevec->pages[loop]);
1134
1135         pagevec_reinit(pagevec);
1136 }
1137 EXPORT_SYMBOL(fscache_mark_pages_cached);
1138
1139 /*
1140  * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1141  * to be associated with the given cookie.
1142  */
1143 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1144                                        struct inode *inode)
1145 {
1146         struct address_space *mapping = inode->i_mapping;
1147         struct pagevec pvec;
1148         pgoff_t next;
1149         int i;
1150
1151         _enter("%p,%p", cookie, inode);
1152
1153         if (!mapping || mapping->nrpages == 0) {
1154                 _leave(" [no pages]");
1155                 return;
1156         }
1157
1158         pagevec_init(&pvec, 0);
1159         next = 0;
1160         do {
1161                 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
1162                         break;
1163                 for (i = 0; i < pagevec_count(&pvec); i++) {
1164                         struct page *page = pvec.pages[i];
1165                         next = page->index;
1166                         if (PageFsCache(page)) {
1167                                 __fscache_wait_on_page_write(cookie, page);
1168                                 __fscache_uncache_page(cookie, page);
1169                         }
1170                 }
1171                 pagevec_release(&pvec);
1172                 cond_resched();
1173         } while (++next);
1174
1175         _leave("");
1176 }
1177 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);