fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 /*
  21  * check to see if a page is being written to the cache
  22  */
  23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  24 {
  25         void *val;
  26
  27         rcu_read_lock();
  28         val = radix_tree_lookup(&cookie->stores, page->index);
  29         rcu_read_unlock();
  30
  31         return val != NULL;
  32 }
  33 EXPORT_SYMBOL(__fscache_check_page_write);
  34
  35 /*
  36  * wait for a page to finish being written to the cache
  37  */
  38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  39 {
  40         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  41
  42         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  43 }
  44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  45
  46 /*
  47  * decide whether a page can be released, possibly by cancelling a store to it
  48  * - we're allowed to sleep if __GFP_WAIT is flagged
  49  */
  50 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  51                                   struct page *page,
  52                                   gfp_t gfp)
  53 {
  54         struct page *xpage;
  55         void *val;
  56
  57         _enter("%p,%p,%x", cookie, page, gfp);
  58
  59 try_again:
  60         rcu_read_lock();
  61         val = radix_tree_lookup(&cookie->stores, page->index);
  62         if (!val) {
  63                 rcu_read_unlock();
  64                 fscache_stat(&fscache_n_store_vmscan_not_storing);
  65                 __fscache_uncache_page(cookie, page);
  66                 return true;
  67         }
  68
  69         /* see if the page is actually undergoing storage - if so we can't get
  70          * rid of it till the cache has finished with it */
  71         if (radix_tree_tag_get(&cookie->stores, page->index,
  72                                FSCACHE_COOKIE_STORING_TAG)) {
  73                 rcu_read_unlock();
  74                 goto page_busy;
  75         }
  76
  77         /* the page is pending storage, so we attempt to cancel the store and
  78          * discard the store request so that the page can be reclaimed */
  79         spin_lock(&cookie->stores_lock);
  80         rcu_read_unlock();
  81
  82         if (radix_tree_tag_get(&cookie->stores, page->index,
  83                                FSCACHE_COOKIE_STORING_TAG)) {
  84                 /* the page started to undergo storage whilst we were looking,
  85                  * so now we can only wait or return */
  86                 spin_unlock(&cookie->stores_lock);
  87                 goto page_busy;
  88         }
  89
  90         xpage = radix_tree_delete(&cookie->stores, page->index);
  91         spin_unlock(&cookie->stores_lock);
  92
  93         if (xpage) {
  94                 fscache_stat(&fscache_n_store_vmscan_cancelled);
  95                 fscache_stat(&fscache_n_store_radix_deletes);
  96                 ASSERTCMP(xpage, ==, page);
  97         } else {
  98                 fscache_stat(&fscache_n_store_vmscan_gone);
  99         }
 100
 101         wake_up_bit(&cookie->flags, 0);
 102         if (xpage)
 103                 page_cache_release(xpage);
 104         __fscache_uncache_page(cookie, page);
 105         return true;
 106
 107 page_busy:
 108         /* We will wait here if we're allowed to, but that could deadlock the
 109          * allocator as the work threads writing to the cache may all end up
 110          * sleeping on memory allocation, so we may need to impose a timeout
 111          * too. */
 112         if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
 113                 fscache_stat(&fscache_n_store_vmscan_busy);
 114                 return false;
 115         }
 116
 117         fscache_stat(&fscache_n_store_vmscan_wait);
 118         __fscache_wait_on_page_write(cookie, page);
 119         gfp &= ~__GFP_WAIT;
 120         goto try_again;
 121 }
 122 EXPORT_SYMBOL(__fscache_maybe_release_page);
 123
 124 /*
 125  * note that a page has finished being written to the cache
 126  */
 127 static void fscache_end_page_write(struct fscache_object *object,
 128                                    struct page *page)
 129 {
 130         struct fscache_cookie *cookie;
 131         struct page *xpage = NULL;
 132
 133         spin_lock(&object->lock);
 134         cookie = object->cookie;
 135         if (cookie) {
 136                 /* delete the page from the tree if it is now no longer
 137                  * pending */
 138                 spin_lock(&cookie->stores_lock);
 139                 radix_tree_tag_clear(&cookie->stores, page->index,
 140                                      FSCACHE_COOKIE_STORING_TAG);
 141                 if (!radix_tree_tag_get(&cookie->stores, page->index,
 142                                         FSCACHE_COOKIE_PENDING_TAG)) {
 143                         fscache_stat(&fscache_n_store_radix_deletes);
 144                         xpage = radix_tree_delete(&cookie->stores, page->index);
 145                 }
 146                 spin_unlock(&cookie->stores_lock);
 147                 wake_up_bit(&cookie->flags, 0);
 148         }
 149         spin_unlock(&object->lock);
 150         if (xpage)
 151                 page_cache_release(xpage);
 152 }
 153
 154 /*
 155  * actually apply the changed attributes to a cache object
 156  */
 157 static void fscache_attr_changed_op(struct fscache_operation *op)
 158 {
 159         struct fscache_object *object = op->object;
 160         int ret;
 161
 162         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 163
 164         fscache_stat(&fscache_n_attr_changed_calls);
 165
 166         if (fscache_object_is_active(object)) {
 167                 fscache_stat(&fscache_n_cop_attr_changed);
 168                 ret = object->cache->ops->attr_changed(object);
 169                 fscache_stat_d(&fscache_n_cop_attr_changed);
 170                 if (ret < 0)
 171                         fscache_abort_object(object);
 172         }
 173
 174         fscache_op_complete(op, true);
 175         _leave("");
 176 }
 177
 178 /*
 179  * notification that the attributes on an object have changed
 180  */
 181 int __fscache_attr_changed(struct fscache_cookie *cookie)
 182 {
 183         struct fscache_operation *op;
 184         struct fscache_object *object;
 185         bool wake_cookie;
 186
 187         _enter("%p", cookie);
 188
 189         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 190
 191         fscache_stat(&fscache_n_attr_changed);
 192
 193         op = kzalloc(sizeof(*op), GFP_KERNEL);
 194         if (!op) {
 195                 fscache_stat(&fscache_n_attr_changed_nomem);
 196                 _leave(" = -ENOMEM");
 197                 return -ENOMEM;
 198         }
 199
 200         fscache_operation_init(op, fscache_attr_changed_op, NULL);
 201         op->flags = FSCACHE_OP_ASYNC |
 202                 (1 << FSCACHE_OP_EXCLUSIVE) |
 203                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 204
 205         spin_lock(&cookie->lock);
 206
 207         if (!fscache_cookie_enabled(cookie) ||
 208             hlist_empty(&cookie->backing_objects))
 209                 goto nobufs;
 210         object = hlist_entry(cookie->backing_objects.first,
 211                              struct fscache_object, cookie_link);
 212
 213         __fscache_use_cookie(cookie);
 214         if (fscache_submit_exclusive_op(object, op) < 0)
 215                 goto nobufs;
 216         spin_unlock(&cookie->lock);
 217         fscache_stat(&fscache_n_attr_changed_ok);
 218         fscache_put_operation(op);
 219         _leave(" = 0");
 220         return 0;
 221
 222 nobufs:
 223         wake_cookie = __fscache_unuse_cookie(cookie);
 224         spin_unlock(&cookie->lock);
 225         kfree(op);
 226         if (wake_cookie)
 227                 __fscache_wake_unused_cookie(cookie);
 228         fscache_stat(&fscache_n_attr_changed_nobufs);
 229         _leave(" = %d", -ENOBUFS);
 230         return -ENOBUFS;
 231 }
 232 EXPORT_SYMBOL(__fscache_attr_changed);
 233
 234 /*
 235  * release a retrieval op reference
 236  */
 237 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 238 {
 239         struct fscache_retrieval *op =
 240                 container_of(_op, struct fscache_retrieval, op);
 241
 242         _enter("{OP%x}", op->op.debug_id);
 243
 244         ASSERTCMP(atomic_read(&op->n_pages), ==, 0);
 245
 246         fscache_hist(fscache_retrieval_histogram, op->start_time);
 247         if (op->context)
 248                 fscache_put_context(op->op.object->cookie, op->context);
 249
 250         _leave("");
 251 }
 252
 253 /*
 254  * allocate a retrieval op
 255  */
 256 static struct fscache_retrieval *fscache_alloc_retrieval(
 257         struct fscache_cookie *cookie,
 258         struct address_space *mapping,
 259         fscache_rw_complete_t end_io_func,
 260         void *context)
 261 {
 262         struct fscache_retrieval *op;
 263
 264         /* allocate a retrieval operation and attempt to submit it */
 265         op = kzalloc(sizeof(*op), GFP_NOIO);
 266         if (!op) {
 267                 fscache_stat(&fscache_n_retrievals_nomem);
 268                 return NULL;
 269         }
 270
 271         fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
 272         op->op.flags    = FSCACHE_OP_MYTHREAD |
 273                 (1UL << FSCACHE_OP_WAITING) |
 274                 (1UL << FSCACHE_OP_UNUSE_COOKIE);
 275         op->mapping     = mapping;
 276         op->end_io_func = end_io_func;
 277         op->context     = context;
 278         op->start_time  = jiffies;
 279         INIT_LIST_HEAD(&op->to_do);
 280         return op;
 281 }
 282
 283 /*
 284  * wait for a deferred lookup to complete
 285  */
 286 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 287 {
 288         unsigned long jif;
 289
 290         _enter("");
 291
 292         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 293                 _leave(" = 0 [imm]");
 294                 return 0;
 295         }
 296
 297         fscache_stat(&fscache_n_retrievals_wait);
 298
 299         jif = jiffies;
 300         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 301                         fscache_wait_bit_interruptible,
 302                         TASK_INTERRUPTIBLE) != 0) {
 303                 fscache_stat(&fscache_n_retrievals_intr);
 304                 _leave(" = -ERESTARTSYS");
 305                 return -ERESTARTSYS;
 306         }
 307
 308         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 309
 310         smp_rmb();
 311         fscache_hist(fscache_retrieval_delay_histogram, jif);
 312         _leave(" = 0 [dly]");
 313         return 0;
 314 }
 315
 316 /*
 317  * Handle cancellation of a pending retrieval op
 318  */
 319 static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
 320 {
 321         struct fscache_retrieval *op =
 322                 container_of(_op, struct fscache_retrieval, op);
 323
 324         atomic_set(&op->n_pages, 0);
 325 }
 326
 327 /*
 328  * wait for an object to become active (or dead)
 329  */
 330 int fscache_wait_for_operation_activation(struct fscache_object *object,
 331                                           struct fscache_operation *op,
 332                                           atomic_t *stat_op_waits,
 333                                           atomic_t *stat_object_dead,
 334                                           void (*do_cancel)(struct fscache_operation *))
 335 {
 336         int ret;
 337
 338         if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
 339                 goto check_if_dead;
 340
 341         _debug(">>> WT");
 342         if (stat_op_waits)
 343                 fscache_stat(stat_op_waits);
 344         if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 345                         fscache_wait_bit_interruptible,
 346                         TASK_INTERRUPTIBLE) != 0) {
 347                 ret = fscache_cancel_op(op, do_cancel);
 348                 if (ret == 0)
 349                         return -ERESTARTSYS;
 350
 351                 /* it's been removed from the pending queue by another party,
 352                  * so we should get to run shortly */
 353                 wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 354                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 355         }
 356         _debug("<<< GO");
 357
 358 check_if_dead:
 359         if (op->state == FSCACHE_OP_ST_CANCELLED) {
 360                 if (stat_object_dead)
 361                         fscache_stat(stat_object_dead);
 362                 _leave(" = -ENOBUFS [cancelled]");
 363                 return -ENOBUFS;
 364         }
 365         if (unlikely(fscache_object_is_dead(object))) {
 366                 pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
 367                 fscache_cancel_op(op, do_cancel);
 368                 if (stat_object_dead)
 369                         fscache_stat(stat_object_dead);
 370                 return -ENOBUFS;
 371         }
 372         return 0;
 373 }
 374
 375 /*
 376  * read a page from the cache or allocate a block in which to store it
 377  * - we return:
 378  *   -ENOMEM    - out of memory, nothing done
 379  *   -ERESTARTSYS - interrupted
 380  *   -ENOBUFS   - no backing object available in which to cache the block
 381  *   -ENODATA   - no data available in the backing object for this block
 382  *   0          - dispatched a read - it'll call end_io_func() when finished
 383  */
 384 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 385                                  struct page *page,
 386                                  fscache_rw_complete_t end_io_func,
 387                                  void *context,
 388                                  gfp_t gfp)
 389 {
 390         struct fscache_retrieval *op;
 391         struct fscache_object *object;
 392         bool wake_cookie = false;
 393         int ret;
 394
 395         _enter("%p,%p,,,", cookie, page);
 396
 397         fscache_stat(&fscache_n_retrievals);
 398
 399         if (hlist_empty(&cookie->backing_objects))
 400                 goto nobufs;
 401
 402         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 403                 _leave(" = -ENOBUFS [invalidating]");
 404                 return -ENOBUFS;
 405         }
 406
 407         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 408         ASSERTCMP(page, !=, NULL);
 409
 410         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 411                 return -ERESTARTSYS;
 412
 413         op = fscache_alloc_retrieval(cookie, page->mapping,
 414                                      end_io_func, context);
 415         if (!op) {
 416                 _leave(" = -ENOMEM");
 417                 return -ENOMEM;
 418         }
 419         atomic_set(&op->n_pages, 1);
 420
 421         spin_lock(&cookie->lock);
 422
 423         if (!fscache_cookie_enabled(cookie) ||
 424             hlist_empty(&cookie->backing_objects))
 425                 goto nobufs_unlock;
 426         object = hlist_entry(cookie->backing_objects.first,
 427                              struct fscache_object, cookie_link);
 428
 429         ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
 430
 431         __fscache_use_cookie(cookie);
 432         atomic_inc(&object->n_reads);
 433         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 434
 435         if (fscache_submit_op(object, &op->op) < 0)
 436                 goto nobufs_unlock_dec;
 437         spin_unlock(&cookie->lock);
 438
 439         fscache_stat(&fscache_n_retrieval_ops);
 440
 441         /* pin the netfs read context in case we need to do the actual netfs
 442          * read because we've encountered a cache read failure */
 443         fscache_get_context(object->cookie, op->context);
 444
 445         /* we wait for the operation to become active, and then process it
 446          * *here*, in this thread, and not in the thread pool */
 447         ret = fscache_wait_for_operation_activation(
 448                 object, &op->op,
 449                 __fscache_stat(&fscache_n_retrieval_op_waits),
 450                 __fscache_stat(&fscache_n_retrievals_object_dead),
 451                 fscache_do_cancel_retrieval);
 452         if (ret < 0)
 453                 goto error;
 454
 455         /* ask the cache to honour the operation */
 456         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 457                 fscache_stat(&fscache_n_cop_allocate_page);
 458                 ret = object->cache->ops->allocate_page(op, page, gfp);
 459                 fscache_stat_d(&fscache_n_cop_allocate_page);
 460                 if (ret == 0)
 461                         ret = -ENODATA;
 462         } else {
 463                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 464                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 465                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 466         }
 467
 468 error:
 469         if (ret == -ENOMEM)
 470                 fscache_stat(&fscache_n_retrievals_nomem);
 471         else if (ret == -ERESTARTSYS)
 472                 fscache_stat(&fscache_n_retrievals_intr);
 473         else if (ret == -ENODATA)
 474                 fscache_stat(&fscache_n_retrievals_nodata);
 475         else if (ret < 0)
 476                 fscache_stat(&fscache_n_retrievals_nobufs);
 477         else
 478                 fscache_stat(&fscache_n_retrievals_ok);
 479
 480         fscache_put_retrieval(op);
 481         _leave(" = %d", ret);
 482         return ret;
 483
 484 nobufs_unlock_dec:
 485         atomic_dec(&object->n_reads);
 486         wake_cookie = __fscache_unuse_cookie(cookie);
 487 nobufs_unlock:
 488         spin_unlock(&cookie->lock);
 489         if (wake_cookie)
 490                 __fscache_wake_unused_cookie(cookie);
 491         kfree(op);
 492 nobufs:
 493         fscache_stat(&fscache_n_retrievals_nobufs);
 494         _leave(" = -ENOBUFS");
 495         return -ENOBUFS;
 496 }
 497 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 498
 499 /*
 500  * read a list of page from the cache or allocate a block in which to store
 501  * them
 502  * - we return:
 503  *   -ENOMEM    - out of memory, some pages may be being read
 504  *   -ERESTARTSYS - interrupted, some pages may be being read
 505  *   -ENOBUFS   - no backing object or space available in which to cache any
 506  *                pages not being read
 507  *   -ENODATA   - no data available in the backing object for some or all of
 508  *                the pages
 509  *   0          - dispatched a read on all pages
 510  *
 511  * end_io_func() will be called for each page read from the cache as it is
 512  * finishes being read
 513  *
 514  * any pages for which a read is dispatched will be removed from pages and
 515  * nr_pages
 516  */
 517 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 518                                   struct address_space *mapping,
 519                                   struct list_head *pages,
 520                                   unsigned *nr_pages,
 521                                   fscache_rw_complete_t end_io_func,
 522                                   void *context,
 523                                   gfp_t gfp)
 524 {
 525         struct fscache_retrieval *op;
 526         struct fscache_object *object;
 527         bool wake_cookie = false;
 528         int ret;
 529
 530         _enter("%p,,%d,,,", cookie, *nr_pages);
 531
 532         fscache_stat(&fscache_n_retrievals);
 533
 534         if (hlist_empty(&cookie->backing_objects))
 535                 goto nobufs;
 536
 537         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 538                 _leave(" = -ENOBUFS [invalidating]");
 539                 return -ENOBUFS;
 540         }
 541
 542         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 543         ASSERTCMP(*nr_pages, >, 0);
 544         ASSERT(!list_empty(pages));
 545
 546         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 547                 return -ERESTARTSYS;
 548
 549         op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
 550         if (!op)
 551                 return -ENOMEM;
 552         atomic_set(&op->n_pages, *nr_pages);
 553
 554         spin_lock(&cookie->lock);
 555
 556         if (!fscache_cookie_enabled(cookie) ||
 557             hlist_empty(&cookie->backing_objects))
 558                 goto nobufs_unlock;
 559         object = hlist_entry(cookie->backing_objects.first,
 560                              struct fscache_object, cookie_link);
 561
 562         __fscache_use_cookie(cookie);
 563         atomic_inc(&object->n_reads);
 564         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 565
 566         if (fscache_submit_op(object, &op->op) < 0)
 567                 goto nobufs_unlock_dec;
 568         spin_unlock(&cookie->lock);
 569
 570         fscache_stat(&fscache_n_retrieval_ops);
 571
 572         /* pin the netfs read context in case we need to do the actual netfs
 573          * read because we've encountered a cache read failure */
 574         fscache_get_context(object->cookie, op->context);
 575
 576         /* we wait for the operation to become active, and then process it
 577          * *here*, in this thread, and not in the thread pool */
 578         ret = fscache_wait_for_operation_activation(
 579                 object, &op->op,
 580                 __fscache_stat(&fscache_n_retrieval_op_waits),
 581                 __fscache_stat(&fscache_n_retrievals_object_dead),
 582                 fscache_do_cancel_retrieval);
 583         if (ret < 0)
 584                 goto error;
 585
 586         /* ask the cache to honour the operation */
 587         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 588                 fscache_stat(&fscache_n_cop_allocate_pages);
 589                 ret = object->cache->ops->allocate_pages(
 590                         op, pages, nr_pages, gfp);
 591                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 592         } else {
 593                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 594                 ret = object->cache->ops->read_or_alloc_pages(
 595                         op, pages, nr_pages, gfp);
 596                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 597         }
 598
 599 error:
 600         if (ret == -ENOMEM)
 601                 fscache_stat(&fscache_n_retrievals_nomem);
 602         else if (ret == -ERESTARTSYS)
 603                 fscache_stat(&fscache_n_retrievals_intr);
 604         else if (ret == -ENODATA)
 605                 fscache_stat(&fscache_n_retrievals_nodata);
 606         else if (ret < 0)
 607                 fscache_stat(&fscache_n_retrievals_nobufs);
 608         else
 609                 fscache_stat(&fscache_n_retrievals_ok);
 610
 611         fscache_put_retrieval(op);
 612         _leave(" = %d", ret);
 613         return ret;
 614
 615 nobufs_unlock_dec:
 616         atomic_dec(&object->n_reads);
 617         wake_cookie = __fscache_unuse_cookie(cookie);
 618 nobufs_unlock:
 619         spin_unlock(&cookie->lock);
 620         kfree(op);
 621         if (wake_cookie)
 622                 __fscache_wake_unused_cookie(cookie);
 623 nobufs:
 624         fscache_stat(&fscache_n_retrievals_nobufs);
 625         _leave(" = -ENOBUFS");
 626         return -ENOBUFS;
 627 }
 628 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 629
 630 /*
 631  * allocate a block in the cache on which to store a page
 632  * - we return:
 633  *   -ENOMEM    - out of memory, nothing done
 634  *   -ERESTARTSYS - interrupted
 635  *   -ENOBUFS   - no backing object available in which to cache the block
 636  *   0          - block allocated
 637  */
 638 int __fscache_alloc_page(struct fscache_cookie *cookie,
 639                          struct page *page,
 640                          gfp_t gfp)
 641 {
 642         struct fscache_retrieval *op;
 643         struct fscache_object *object;
 644         bool wake_cookie = false;
 645         int ret;
 646
 647         _enter("%p,%p,,,", cookie, page);
 648
 649         fscache_stat(&fscache_n_allocs);
 650
 651         if (hlist_empty(&cookie->backing_objects))
 652                 goto nobufs;
 653
 654         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 655         ASSERTCMP(page, !=, NULL);
 656
 657         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 658                 _leave(" = -ENOBUFS [invalidating]");
 659                 return -ENOBUFS;
 660         }
 661
 662         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 663                 return -ERESTARTSYS;
 664
 665         op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
 666         if (!op)
 667                 return -ENOMEM;
 668         atomic_set(&op->n_pages, 1);
 669
 670         spin_lock(&cookie->lock);
 671
 672         if (!fscache_cookie_enabled(cookie) ||
 673             hlist_empty(&cookie->backing_objects))
 674                 goto nobufs_unlock;
 675         object = hlist_entry(cookie->backing_objects.first,
 676                              struct fscache_object, cookie_link);
 677
 678         __fscache_use_cookie(cookie);
 679         if (fscache_submit_op(object, &op->op) < 0)
 680                 goto nobufs_unlock_dec;
 681         spin_unlock(&cookie->lock);
 682
 683         fscache_stat(&fscache_n_alloc_ops);
 684
 685         ret = fscache_wait_for_operation_activation(
 686                 object, &op->op,
 687                 __fscache_stat(&fscache_n_alloc_op_waits),
 688                 __fscache_stat(&fscache_n_allocs_object_dead),
 689                 fscache_do_cancel_retrieval);
 690         if (ret < 0)
 691                 goto error;
 692
 693         /* ask the cache to honour the operation */
 694         fscache_stat(&fscache_n_cop_allocate_page);
 695         ret = object->cache->ops->allocate_page(op, page, gfp);
 696         fscache_stat_d(&fscache_n_cop_allocate_page);
 697
 698 error:
 699         if (ret == -ERESTARTSYS)
 700                 fscache_stat(&fscache_n_allocs_intr);
 701         else if (ret < 0)
 702                 fscache_stat(&fscache_n_allocs_nobufs);
 703         else
 704                 fscache_stat(&fscache_n_allocs_ok);
 705
 706         fscache_put_retrieval(op);
 707         _leave(" = %d", ret);
 708         return ret;
 709
 710 nobufs_unlock_dec:
 711         wake_cookie = __fscache_unuse_cookie(cookie);
 712 nobufs_unlock:
 713         spin_unlock(&cookie->lock);
 714         kfree(op);
 715         if (wake_cookie)
 716                 __fscache_wake_unused_cookie(cookie);
 717 nobufs:
 718         fscache_stat(&fscache_n_allocs_nobufs);
 719         _leave(" = -ENOBUFS");
 720         return -ENOBUFS;
 721 }
 722 EXPORT_SYMBOL(__fscache_alloc_page);
 723
 724 /*
 725  * Unmark pages allocate in the readahead code path (via:
 726  * fscache_readpages_or_alloc) after delegating to the base filesystem
 727  */
 728 void __fscache_readpages_cancel(struct fscache_cookie *cookie,
 729                                 struct list_head *pages)
 730 {
 731         struct page *page;
 732
 733         list_for_each_entry(page, pages, lru) {
 734                 if (PageFsCache(page))
 735                         __fscache_uncache_page(cookie, page);
 736         }
 737 }
 738 EXPORT_SYMBOL(__fscache_readpages_cancel);
 739
 740 /*
 741  * release a write op reference
 742  */
 743 static void fscache_release_write_op(struct fscache_operation *_op)
 744 {
 745         _enter("{OP%x}", _op->debug_id);
 746 }
 747
 748 /*
 749  * perform the background storage of a page into the cache
 750  */
 751 static void fscache_write_op(struct fscache_operation *_op)
 752 {
 753         struct fscache_storage *op =
 754                 container_of(_op, struct fscache_storage, op);
 755         struct fscache_object *object = op->op.object;
 756         struct fscache_cookie *cookie;
 757         struct page *page;
 758         unsigned n;
 759         void *results[1];
 760         int ret;
 761
 762         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 763
 764         spin_lock(&object->lock);
 765         cookie = object->cookie;
 766
 767         if (!fscache_object_is_active(object)) {
 768                 /* If we get here, then the on-disk cache object likely longer
 769                  * exists, so we should just cancel this write operation.
 770                  */
 771                 spin_unlock(&object->lock);
 772                 fscache_op_complete(&op->op, false);
 773                 _leave(" [inactive]");
 774                 return;
 775         }
 776
 777         if (!cookie) {
 778                 /* If we get here, then the cookie belonging to the object was
 779                  * detached, probably by the cookie being withdrawn due to
 780                  * memory pressure, which means that the pages we might write
 781                  * to the cache from no longer exist - therefore, we can just
 782                  * cancel this write operation.
 783                  */
 784                 spin_unlock(&object->lock);
 785                 fscache_op_complete(&op->op, false);
 786                 _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
 787                        _op->flags, _op->state, object->state->short_name,
 788                        object->flags);
 789                 return;
 790         }
 791
 792         spin_lock(&cookie->stores_lock);
 793
 794         fscache_stat(&fscache_n_store_calls);
 795
 796         /* find a page to store */
 797         page = NULL;
 798         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 799                                        FSCACHE_COOKIE_PENDING_TAG);
 800         if (n != 1)
 801                 goto superseded;
 802         page = results[0];
 803         _debug("gang %d [%lx]", n, page->index);
 804         if (page->index > op->store_limit) {
 805                 fscache_stat(&fscache_n_store_pages_over_limit);
 806                 goto superseded;
 807         }
 808
 809         radix_tree_tag_set(&cookie->stores, page->index,
 810                            FSCACHE_COOKIE_STORING_TAG);
 811         radix_tree_tag_clear(&cookie->stores, page->index,
 812                              FSCACHE_COOKIE_PENDING_TAG);
 813
 814         spin_unlock(&cookie->stores_lock);
 815         spin_unlock(&object->lock);
 816
 817         fscache_stat(&fscache_n_store_pages);
 818         fscache_stat(&fscache_n_cop_write_page);
 819         ret = object->cache->ops->write_page(op, page);
 820         fscache_stat_d(&fscache_n_cop_write_page);
 821         fscache_end_page_write(object, page);
 822         if (ret < 0) {
 823                 fscache_abort_object(object);
 824                 fscache_op_complete(&op->op, true);
 825         } else {
 826                 fscache_enqueue_operation(&op->op);
 827         }
 828
 829         _leave("");
 830         return;
 831
 832 superseded:
 833         /* this writer is going away and there aren't any more things to
 834          * write */
 835         _debug("cease");
 836         spin_unlock(&cookie->stores_lock);
 837         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 838         spin_unlock(&object->lock);
 839         fscache_op_complete(&op->op, true);
 840         _leave("");
 841 }
 842
 843 /*
 844  * Clear the pages pending writing for invalidation
 845  */
 846 void fscache_invalidate_writes(struct fscache_cookie *cookie)
 847 {
 848         struct page *page;
 849         void *results[16];
 850         int n, i;
 851
 852         _enter("");
 853
 854         for (;;) {
 855                 spin_lock(&cookie->stores_lock);
 856                 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
 857                                                ARRAY_SIZE(results),
 858                                                FSCACHE_COOKIE_PENDING_TAG);
 859                 if (n == 0) {
 860                         spin_unlock(&cookie->stores_lock);
 861                         break;
 862                 }
 863
 864                 for (i = n - 1; i >= 0; i--) {
 865                         page = results[i];
 866                         radix_tree_delete(&cookie->stores, page->index);
 867                 }
 868
 869                 spin_unlock(&cookie->stores_lock);
 870
 871                 for (i = n - 1; i >= 0; i--)
 872                         page_cache_release(results[i]);
 873         }
 874
 875         _leave("");
 876 }
 877
 878 /*
 879  * request a page be stored in the cache
 880  * - returns:
 881  *   -ENOMEM    - out of memory, nothing done
 882  *   -ENOBUFS   - no backing object available in which to cache the page
 883  *   0          - dispatched a write - it'll call end_io_func() when finished
 884  *
 885  * if the cookie still has a backing object at this point, that object can be
 886  * in one of a few states with respect to storage processing:
 887  *
 888  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 889  *      set)
 890  *
 891  *      (a) no writes yet
 892  *
 893  *      (b) writes deferred till post-creation (mark page for writing and
 894  *          return immediately)
 895  *
 896  *  (2) negative lookup, object created, initial fill being made from netfs
 897  *
 898  *      (a) fill point not yet reached this page (mark page for writing and
 899  *          return)
 900  *
 901  *      (b) fill point passed this page (queue op to store this page)
 902  *
 903  *  (3) object extant (queue op to store this page)
 904  *
 905  * any other state is invalid
 906  */
 907 int __fscache_write_page(struct fscache_cookie *cookie,
 908                          struct page *page,
 909                          gfp_t gfp)
 910 {
 911         struct fscache_storage *op;
 912         struct fscache_object *object;
 913         bool wake_cookie = false;
 914         int ret;
 915
 916         _enter("%p,%x,", cookie, (u32) page->flags);
 917
 918         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 919         ASSERT(PageFsCache(page));
 920
 921         fscache_stat(&fscache_n_stores);
 922
 923         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 924                 _leave(" = -ENOBUFS [invalidating]");
 925                 return -ENOBUFS;
 926         }
 927
 928         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 929         if (!op)
 930                 goto nomem;
 931
 932         fscache_operation_init(&op->op, fscache_write_op,
 933                                fscache_release_write_op);
 934         op->op.flags = FSCACHE_OP_ASYNC |
 935                 (1 << FSCACHE_OP_WAITING) |
 936                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 937
 938         ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
 939         if (ret < 0)
 940                 goto nomem_free;
 941
 942         ret = -ENOBUFS;
 943         spin_lock(&cookie->lock);
 944
 945         if (!fscache_cookie_enabled(cookie) ||
 946             hlist_empty(&cookie->backing_objects))
 947                 goto nobufs;
 948         object = hlist_entry(cookie->backing_objects.first,
 949                              struct fscache_object, cookie_link);
 950         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 951                 goto nobufs;
 952
 953         /* add the page to the pending-storage radix tree on the backing
 954          * object */
 955         spin_lock(&object->lock);
 956         spin_lock(&cookie->stores_lock);
 957
 958         _debug("store limit %llx", (unsigned long long) object->store_limit);
 959
 960         ret = radix_tree_insert(&cookie->stores, page->index, page);
 961         if (ret < 0) {
 962                 if (ret == -EEXIST)
 963                         goto already_queued;
 964                 _debug("insert failed %d", ret);
 965                 goto nobufs_unlock_obj;
 966         }
 967
 968         radix_tree_tag_set(&cookie->stores, page->index,
 969                            FSCACHE_COOKIE_PENDING_TAG);
 970         page_cache_get(page);
 971
 972         /* we only want one writer at a time, but we do need to queue new
 973          * writers after exclusive ops */
 974         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 975                 goto already_pending;
 976
 977         spin_unlock(&cookie->stores_lock);
 978         spin_unlock(&object->lock);
 979
 980         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 981         op->store_limit = object->store_limit;
 982
 983         __fscache_use_cookie(cookie);
 984         if (fscache_submit_op(object, &op->op) < 0)
 985                 goto submit_failed;
 986
 987         spin_unlock(&cookie->lock);
 988         radix_tree_preload_end();
 989         fscache_stat(&fscache_n_store_ops);
 990         fscache_stat(&fscache_n_stores_ok);
 991
 992         /* the work queue now carries its own ref on the object */
 993         fscache_put_operation(&op->op);
 994         _leave(" = 0");
 995         return 0;
 996
 997 already_queued:
 998         fscache_stat(&fscache_n_stores_again);
 999 already_pending:
1000         spin_unlock(&cookie->stores_lock);
1001         spin_unlock(&object->lock);
1002         spin_unlock(&cookie->lock);
1003         radix_tree_preload_end();
1004         kfree(op);
1005         fscache_stat(&fscache_n_stores_ok);
1006         _leave(" = 0");
1007         return 0;
1008
1009 submit_failed:
1010         spin_lock(&cookie->stores_lock);
1011         radix_tree_delete(&cookie->stores, page->index);
1012         spin_unlock(&cookie->stores_lock);
1013         wake_cookie = __fscache_unuse_cookie(cookie);
1014         page_cache_release(page);
1015         ret = -ENOBUFS;
1016         goto nobufs;
1017
1018 nobufs_unlock_obj:
1019         spin_unlock(&cookie->stores_lock);
1020         spin_unlock(&object->lock);
1021 nobufs:
1022         spin_unlock(&cookie->lock);
1023         radix_tree_preload_end();
1024         kfree(op);
1025         if (wake_cookie)
1026                 __fscache_wake_unused_cookie(cookie);
1027         fscache_stat(&fscache_n_stores_nobufs);
1028         _leave(" = -ENOBUFS");
1029         return -ENOBUFS;
1030
1031 nomem_free:
1032         kfree(op);
1033 nomem:
1034         fscache_stat(&fscache_n_stores_oom);
1035         _leave(" = -ENOMEM");
1036         return -ENOMEM;
1037 }
1038 EXPORT_SYMBOL(__fscache_write_page);
1039
1040 /*
1041  * remove a page from the cache
1042  */
1043 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1044 {
1045         struct fscache_object *object;
1046
1047         _enter(",%p", page);
1048
1049         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1050         ASSERTCMP(page, !=, NULL);
1051
1052         fscache_stat(&fscache_n_uncaches);
1053
1054         /* cache withdrawal may beat us to it */
1055         if (!PageFsCache(page))
1056                 goto done;
1057
1058         /* get the object */
1059         spin_lock(&cookie->lock);
1060
1061         if (hlist_empty(&cookie->backing_objects)) {
1062                 ClearPageFsCache(page);
1063                 goto done_unlock;
1064         }
1065
1066         object = hlist_entry(cookie->backing_objects.first,
1067                              struct fscache_object, cookie_link);
1068
1069         /* there might now be stuff on disk we could read */
1070         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1071
1072         /* only invoke the cache backend if we managed to mark the page
1073          * uncached here; this deals with synchronisation vs withdrawal */
1074         if (TestClearPageFsCache(page) &&
1075             object->cache->ops->uncache_page) {
1076                 /* the cache backend releases the cookie lock */
1077                 fscache_stat(&fscache_n_cop_uncache_page);
1078                 object->cache->ops->uncache_page(object, page);
1079                 fscache_stat_d(&fscache_n_cop_uncache_page);
1080                 goto done;
1081         }
1082
1083 done_unlock:
1084         spin_unlock(&cookie->lock);
1085 done:
1086         _leave("");
1087 }
1088 EXPORT_SYMBOL(__fscache_uncache_page);
1089
1090 /**
1091  * fscache_mark_page_cached - Mark a page as being cached
1092  * @op: The retrieval op pages are being marked for
1093  * @page: The page to be marked
1094  *
1095  * Mark a netfs page as being cached.  After this is called, the netfs
1096  * must call fscache_uncache_page() to remove the mark.
1097  */
1098 void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1099 {
1100         struct fscache_cookie *cookie = op->op.object->cookie;
1101
1102 #ifdef CONFIG_FSCACHE_STATS
1103         atomic_inc(&fscache_n_marks);
1104 #endif
1105
1106         _debug("- mark %p{%lx}", page, page->index);
1107         if (TestSetPageFsCache(page)) {
1108                 static bool once_only;
1109                 if (!once_only) {
1110                         once_only = true;
1111                         printk(KERN_WARNING "FS-Cache:"
1112                                " Cookie type %s marked page %lx"
1113                                " multiple times\n",
1114                                cookie->def->name, page->index);
1115                 }
1116         }
1117
1118         if (cookie->def->mark_page_cached)
1119                 cookie->def->mark_page_cached(cookie->netfs_data,
1120                                               op->mapping, page);
1121 }
1122 EXPORT_SYMBOL(fscache_mark_page_cached);
1123
1124 /**
1125  * fscache_mark_pages_cached - Mark pages as being cached
1126  * @op: The retrieval op pages are being marked for
1127  * @pagevec: The pages to be marked
1128  *
1129  * Mark a bunch of netfs pages as being cached.  After this is called,
1130  * the netfs must call fscache_uncache_page() to remove the mark.
1131  */
1132 void fscache_mark_pages_cached(struct fscache_retrieval *op,
1133                                struct pagevec *pagevec)
1134 {
1135         unsigned long loop;
1136
1137         for (loop = 0; loop < pagevec->nr; loop++)
1138                 fscache_mark_page_cached(op, pagevec->pages[loop]);
1139
1140         pagevec_reinit(pagevec);
1141 }
1142 EXPORT_SYMBOL(fscache_mark_pages_cached);
1143
1144 /*
1145  * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1146  * to be associated with the given cookie.
1147  */
1148 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1149                                        struct inode *inode)
1150 {
1151         struct address_space *mapping = inode->i_mapping;
1152         struct pagevec pvec;
1153         pgoff_t next;
1154         int i;
1155
1156         _enter("%p,%p", cookie, inode);
1157
1158         if (!mapping || mapping->nrpages == 0) {
1159                 _leave(" [no pages]");
1160                 return;
1161         }
1162
1163         pagevec_init(&pvec, 0);
1164         next = 0;
1165         do {
1166                 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
1167                         break;
1168                 for (i = 0; i < pagevec_count(&pvec); i++) {
1169                         struct page *page = pvec.pages[i];
1170                         next = page->index;
1171                         if (PageFsCache(page)) {
1172                                 __fscache_wait_on_page_write(cookie, page);
1173                                 __fscache_uncache_page(cookie, page);
1174                         }
1175                 }
1176                 pagevec_release(&pvec);
1177                 cond_resched();
1178         } while (++next);
1179
1180         _leave("");
1181 }
1182 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);