mm/swap.c

   1 /*
   2  *  linux/mm/swap.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file contains the default values for the operation of the
   9  * Linux VM subsystem. Fine-tuning documentation can be found in
  10  * Documentation/sysctl/vm.txt.
  11  * Started 18.12.91
  12  * Swap aging added 23.2.95, Stephen Tweedie.
  13  * Buffermem limits added 12.3.98, Rik van Riel.
  14  */
  15
  16 #include <linux/mm.h>
  17 #include <linux/sched.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/swap.h>
  20 #include <linux/mman.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/pagevec.h>
  23 #include <linux/init.h>
  24 #include <linux/export.h>
  25 #include <linux/mm_inline.h>
  26 #include <linux/buffer_head.h>  /* for try_to_release_page() */
  27 #include <linux/percpu_counter.h>
  28 #include <linux/percpu.h>
  29 #include <linux/cpu.h>
  30 #include <linux/notifier.h>
  31 #include <linux/backing-dev.h>
  32 #include <linux/memcontrol.h>
  33 #include <linux/gfp.h>
  34
  35 #include "internal.h"
  36
  37 /* How many pages do we try to swap or page in/out together? */
  38 int page_cluster;
  39
  40 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
  41 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
  42 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
  43
  44 /*
  45  * This path almost never happens for VM activity - pages are normally
  46  * freed via pagevecs.  But it gets used by networking.
  47  */
  48 static void __page_cache_release(struct page *page)
  49 {
  50         if (PageLRU(page)) {
  51                 unsigned long flags;
  52                 struct zone *zone = page_zone(page);
  53
  54                 spin_lock_irqsave(&zone->lru_lock, flags);
  55                 VM_BUG_ON(!PageLRU(page));
  56                 __ClearPageLRU(page);
  57                 del_page_from_lru(zone, page);
  58                 spin_unlock_irqrestore(&zone->lru_lock, flags);
  59         }
  60 }
  61
  62 static void __put_single_page(struct page *page)
  63 {
  64         __page_cache_release(page);
  65         free_hot_cold_page(page, 0);
  66 }
  67
  68 static void __put_compound_page(struct page *page)
  69 {
  70         compound_page_dtor *dtor;
  71
  72         __page_cache_release(page);
  73         dtor = get_compound_page_dtor(page);
  74         (*dtor)(page);
  75 }
  76
  77 static void put_compound_page(struct page *page)
  78 {
  79         if (unlikely(PageTail(page))) {
  80                 /* __split_huge_page_refcount can run under us */
  81                 struct page *page_head = compound_trans_head(page);
  82
  83                 if (likely(page != page_head &&
  84                            get_page_unless_zero(page_head))) {
  85                         unsigned long flags;
  86                         /*
  87                          * page_head wasn't a dangling pointer but it
  88                          * may not be a head page anymore by the time
  89                          * we obtain the lock. That is ok as long as it
  90                          * can't be freed from under us.
  91                          */
  92                         flags = compound_lock_irqsave(page_head);
  93                         if (unlikely(!PageTail(page))) {
  94                                 /* __split_huge_page_refcount run before us */
  95                                 compound_unlock_irqrestore(page_head, flags);
  96                                 VM_BUG_ON(PageHead(page_head));
  97                                 if (put_page_testzero(page_head))
  98                                         __put_single_page(page_head);
  99                         out_put_single:
 100                                 if (put_page_testzero(page))
 101                                         __put_single_page(page);
 102                                 return;
 103                         }
 104                         VM_BUG_ON(page_head != page->first_page);
 105                         /*
 106                          * We can release the refcount taken by
 107                          * get_page_unless_zero() now that
 108                          * __split_huge_page_refcount() is blocked on
 109                          * the compound_lock.
 110                          */
 111                         if (put_page_testzero(page_head))
 112                                 VM_BUG_ON(1);
 113                         /* __split_huge_page_refcount will wait now */
 114                         VM_BUG_ON(page_mapcount(page) <= 0);
 115                         atomic_dec(&page->_mapcount);
 116                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
 117                         VM_BUG_ON(atomic_read(&page->_count) != 0);
 118                         compound_unlock_irqrestore(page_head, flags);
 119                         if (put_page_testzero(page_head)) {
 120                                 if (PageHead(page_head))
 121                                         __put_compound_page(page_head);
 122                                 else
 123                                         __put_single_page(page_head);
 124                         }
 125                 } else {
 126                         /* page_head is a dangling pointer */
 127                         VM_BUG_ON(PageTail(page));
 128                         goto out_put_single;
 129                 }
 130         } else if (put_page_testzero(page)) {
 131                 if (PageHead(page))
 132                         __put_compound_page(page);
 133                 else
 134                         __put_single_page(page);
 135         }
 136 }
 137
 138 void put_page(struct page *page)
 139 {
 140         if (unlikely(PageCompound(page)))
 141                 put_compound_page(page);
 142         else if (put_page_testzero(page))
 143                 __put_single_page(page);
 144 }
 145 EXPORT_SYMBOL(put_page);
 146
 147 /*
 148  * This function is exported but must not be called by anything other
 149  * than get_page(). It implements the slow path of get_page().
 150  */
 151 bool __get_page_tail(struct page *page)
 152 {
 153         /*
 154          * This takes care of get_page() if run on a tail page
 155          * returned by one of the get_user_pages/follow_page variants.
 156          * get_user_pages/follow_page itself doesn't need the compound
 157          * lock because it runs __get_page_tail_foll() under the
 158          * proper PT lock that already serializes against
 159          * split_huge_page().
 160          */
 161         unsigned long flags;
 162         bool got = false;
 163         struct page *page_head = compound_trans_head(page);
 164
 165         if (likely(page != page_head && get_page_unless_zero(page_head))) {
 166                 /*
 167                  * page_head wasn't a dangling pointer but it
 168                  * may not be a head page anymore by the time
 169                  * we obtain the lock. That is ok as long as it
 170                  * can't be freed from under us.
 171                  */
 172                 flags = compound_lock_irqsave(page_head);
 173                 /* here __split_huge_page_refcount won't run anymore */
 174                 if (likely(PageTail(page))) {
 175                         __get_page_tail_foll(page, false);
 176                         got = true;
 177                 }
 178                 compound_unlock_irqrestore(page_head, flags);
 179                 if (unlikely(!got))
 180                         put_page(page_head);
 181         }
 182         return got;
 183 }
 184 EXPORT_SYMBOL(__get_page_tail);
 185
 186 /**
 187  * put_pages_list() - release a list of pages
 188  * @pages: list of pages threaded on page->lru
 189  *
 190  * Release a list of pages which are strung together on page.lru.  Currently
 191  * used by read_cache_pages() and related error recovery code.
 192  */
 193 void put_pages_list(struct list_head *pages)
 194 {
 195         while (!list_empty(pages)) {
 196                 struct page *victim;
 197
 198                 victim = list_entry(pages->prev, struct page, lru);
 199                 list_del(&victim->lru);
 200                 page_cache_release(victim);
 201         }
 202 }
 203 EXPORT_SYMBOL(put_pages_list);
 204
 205 static void pagevec_lru_move_fn(struct pagevec *pvec,
 206                                 void (*move_fn)(struct page *page, void *arg),
 207                                 void *arg)
 208 {
 209         int i;
 210         struct zone *zone = NULL;
 211         unsigned long flags = 0;
 212
 213         for (i = 0; i < pagevec_count(pvec); i++) {
 214                 struct page *page = pvec->pages[i];
 215                 struct zone *pagezone = page_zone(page);
 216
 217                 if (pagezone != zone) {
 218                         if (zone)
 219                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 220                         zone = pagezone;
 221                         spin_lock_irqsave(&zone->lru_lock, flags);
 222                 }
 223
 224                 (*move_fn)(page, arg);
 225         }
 226         if (zone)
 227                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 228         release_pages(pvec->pages, pvec->nr, pvec->cold);
 229         pagevec_reinit(pvec);
 230 }
 231
 232 static void pagevec_move_tail_fn(struct page *page, void *arg)
 233 {
 234         int *pgmoved = arg;
 235
 236         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 237                 enum lru_list lru = page_lru_base_type(page);
 238                 struct lruvec *lruvec;
 239
 240                 lruvec = mem_cgroup_lru_move_lists(page_zone(page),
 241                                                    page, lru, lru);
 242                 list_move_tail(&page->lru, &lruvec->lists[lru]);
 243                 (*pgmoved)++;
 244         }
 245 }
 246
 247 /*
 248  * pagevec_move_tail() must be called with IRQ disabled.
 249  * Otherwise this may cause nasty races.
 250  */
 251 static void pagevec_move_tail(struct pagevec *pvec)
 252 {
 253         int pgmoved = 0;
 254
 255         pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
 256         __count_vm_events(PGROTATED, pgmoved);
 257 }
 258
 259 /*
 260  * Writeback is about to end against a page which has been marked for immediate
 261  * reclaim.  If it still appears to be reclaimable, move it to the tail of the
 262  * inactive list.
 263  */
 264 void rotate_reclaimable_page(struct page *page)
 265 {
 266         if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
 267             !PageUnevictable(page) && PageLRU(page)) {
 268                 struct pagevec *pvec;
 269                 unsigned long flags;
 270
 271                 page_cache_get(page);
 272                 local_irq_save(flags);
 273                 pvec = &__get_cpu_var(lru_rotate_pvecs);
 274                 if (!pagevec_add(pvec, page))
 275                         pagevec_move_tail(pvec);
 276                 local_irq_restore(flags);
 277         }
 278 }
 279
 280 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
 281                                      int file, int rotated)
 282 {
 283         struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
 284         struct zone_reclaim_stat *memcg_reclaim_stat;
 285
 286         memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
 287
 288         reclaim_stat->recent_scanned[file]++;
 289         if (rotated)
 290                 reclaim_stat->recent_rotated[file]++;
 291
 292         if (!memcg_reclaim_stat)
 293                 return;
 294
 295         memcg_reclaim_stat->recent_scanned[file]++;
 296         if (rotated)
 297                 memcg_reclaim_stat->recent_rotated[file]++;
 298 }
 299
 300 static void __activate_page(struct page *page, void *arg)
 301 {
 302         struct zone *zone = page_zone(page);
 303
 304         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 305                 int file = page_is_file_cache(page);
 306                 int lru = page_lru_base_type(page);
 307                 del_page_from_lru_list(zone, page, lru);
 308
 309                 SetPageActive(page);
 310                 lru += LRU_ACTIVE;
 311                 add_page_to_lru_list(zone, page, lru);
 312                 __count_vm_event(PGACTIVATE);
 313
 314                 update_page_reclaim_stat(zone, page, file, 1);
 315         }
 316 }
 317
 318 #ifdef CONFIG_SMP
 319 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 320
 321 static void activate_page_drain(int cpu)
 322 {
 323         struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
 324
 325         if (pagevec_count(pvec))
 326                 pagevec_lru_move_fn(pvec, __activate_page, NULL);
 327 }
 328
 329 void activate_page(struct page *page)
 330 {
 331         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 332                 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
 333
 334                 page_cache_get(page);
 335                 if (!pagevec_add(pvec, page))
 336                         pagevec_lru_move_fn(pvec, __activate_page, NULL);
 337                 put_cpu_var(activate_page_pvecs);
 338         }
 339 }
 340
 341 #else
 342 static inline void activate_page_drain(int cpu)
 343 {
 344 }
 345
 346 void activate_page(struct page *page)
 347 {
 348         struct zone *zone = page_zone(page);
 349
 350         spin_lock_irq(&zone->lru_lock);
 351         __activate_page(page, NULL);
 352         spin_unlock_irq(&zone->lru_lock);
 353 }
 354 #endif
 355
 356 /*
 357  * Mark a page as having seen activity.
 358  *
 359  * inactive,unreferenced        ->      inactive,referenced
 360  * inactive,referenced          ->      active,unreferenced
 361  * active,unreferenced          ->      active,referenced
 362  */
 363 void mark_page_accessed(struct page *page)
 364 {
 365         if (!PageActive(page) && !PageUnevictable(page) &&
 366                         PageReferenced(page) && PageLRU(page)) {
 367                 activate_page(page);
 368                 ClearPageReferenced(page);
 369         } else if (!PageReferenced(page)) {
 370                 SetPageReferenced(page);
 371         }
 372 }
 373
 374 EXPORT_SYMBOL(mark_page_accessed);
 375
 376 void __lru_cache_add(struct page *page, enum lru_list lru)
 377 {
 378         struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
 379
 380         page_cache_get(page);
 381         if (!pagevec_add(pvec, page))
 382                 ____pagevec_lru_add(pvec, lru);
 383         put_cpu_var(lru_add_pvecs);
 384 }
 385 EXPORT_SYMBOL(__lru_cache_add);
 386
 387 /**
 388  * lru_cache_add_lru - add a page to a page list
 389  * @page: the page to be added to the LRU.
 390  * @lru: the LRU list to which the page is added.
 391  */
 392 void lru_cache_add_lru(struct page *page, enum lru_list lru)
 393 {
 394         if (PageActive(page)) {
 395                 VM_BUG_ON(PageUnevictable(page));
 396                 ClearPageActive(page);
 397         } else if (PageUnevictable(page)) {
 398                 VM_BUG_ON(PageActive(page));
 399                 ClearPageUnevictable(page);
 400         }
 401
 402         VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
 403         __lru_cache_add(page, lru);
 404 }
 405
 406 /**
 407  * add_page_to_unevictable_list - add a page to the unevictable list
 408  * @page:  the page to be added to the unevictable list
 409  *
 410  * Add page directly to its zone's unevictable list.  To avoid races with
 411  * tasks that might be making the page evictable, through eg. munlock,
 412  * munmap or exit, while it's not on the lru, we want to add the page
 413  * while it's locked or otherwise "invisible" to other tasks.  This is
 414  * difficult to do when using the pagevec cache, so bypass that.
 415  */
 416 void add_page_to_unevictable_list(struct page *page)
 417 {
 418         struct zone *zone = page_zone(page);
 419
 420         spin_lock_irq(&zone->lru_lock);
 421         SetPageUnevictable(page);
 422         SetPageLRU(page);
 423         add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
 424         spin_unlock_irq(&zone->lru_lock);
 425 }
 426
 427 /*
 428  * If the page can not be invalidated, it is moved to the
 429  * inactive list to speed up its reclaim.  It is moved to the
 430  * head of the list, rather than the tail, to give the flusher
 431  * threads some time to write it out, as this is much more
 432  * effective than the single-page writeout from reclaim.
 433  *
 434  * If the page isn't page_mapped and dirty/writeback, the page
 435  * could reclaim asap using PG_reclaim.
 436  *
 437  * 1. active, mapped page -> none
 438  * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
 439  * 3. inactive, mapped page -> none
 440  * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
 441  * 5. inactive, clean -> inactive, tail
 442  * 6. Others -> none
 443  *
 444  * In 4, why it moves inactive's head, the VM expects the page would
 445  * be write it out by flusher threads as this is much more effective
 446  * than the single-page writeout from reclaim.
 447  */
 448 static void lru_deactivate_fn(struct page *page, void *arg)
 449 {
 450         int lru, file;
 451         bool active;
 452         struct zone *zone = page_zone(page);
 453
 454         if (!PageLRU(page))
 455                 return;
 456
 457         if (PageUnevictable(page))
 458                 return;
 459
 460         /* Some processes are using the page */
 461         if (page_mapped(page))
 462                 return;
 463
 464         active = PageActive(page);
 465
 466         file = page_is_file_cache(page);
 467         lru = page_lru_base_type(page);
 468         del_page_from_lru_list(zone, page, lru + active);
 469         ClearPageActive(page);
 470         ClearPageReferenced(page);
 471         add_page_to_lru_list(zone, page, lru);
 472
 473         if (PageWriteback(page) || PageDirty(page)) {
 474                 /*
 475                  * PG_reclaim could be raced with end_page_writeback
 476                  * It can make readahead confusing.  But race window
 477                  * is _really_ small and  it's non-critical problem.
 478                  */
 479                 SetPageReclaim(page);
 480         } else {
 481                 struct lruvec *lruvec;
 482                 /*
 483                  * The page's writeback ends up during pagevec
 484                  * We moves tha page into tail of inactive.
 485                  */
 486                 lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
 487                 list_move_tail(&page->lru, &lruvec->lists[lru]);
 488                 __count_vm_event(PGROTATED);
 489         }
 490
 491         if (active)
 492                 __count_vm_event(PGDEACTIVATE);
 493         update_page_reclaim_stat(zone, page, file, 0);
 494 }
 495
 496 /*
 497  * Drain pages out of the cpu's pagevecs.
 498  * Either "cpu" is the current CPU, and preemption has already been
 499  * disabled; or "cpu" is being hot-unplugged, and is already dead.
 500  */
 501 static void drain_cpu_pagevecs(int cpu)
 502 {
 503         struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
 504         struct pagevec *pvec;
 505         int lru;
 506
 507         for_each_lru(lru) {
 508                 pvec = &pvecs[lru - LRU_BASE];
 509                 if (pagevec_count(pvec))
 510                         ____pagevec_lru_add(pvec, lru);
 511         }
 512
 513         pvec = &per_cpu(lru_rotate_pvecs, cpu);
 514         if (pagevec_count(pvec)) {
 515                 unsigned long flags;
 516
 517                 /* No harm done if a racing interrupt already did this */
 518                 local_irq_save(flags);
 519                 pagevec_move_tail(pvec);
 520                 local_irq_restore(flags);
 521         }
 522
 523         pvec = &per_cpu(lru_deactivate_pvecs, cpu);
 524         if (pagevec_count(pvec))
 525                 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 526
 527         activate_page_drain(cpu);
 528 }
 529
 530 /**
 531  * deactivate_page - forcefully deactivate a page
 532  * @page: page to deactivate
 533  *
 534  * This function hints the VM that @page is a good reclaim candidate,
 535  * for example if its invalidation fails due to the page being dirty
 536  * or under writeback.
 537  */
 538 void deactivate_page(struct page *page)
 539 {
 540         /*
 541          * In a workload with many unevictable page such as mprotect, unevictable
 542          * page deactivation for accelerating reclaim is pointless.
 543          */
 544         if (PageUnevictable(page))
 545                 return;
 546
 547         if (likely(get_page_unless_zero(page))) {
 548                 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
 549
 550                 if (!pagevec_add(pvec, page))
 551                         pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 552                 put_cpu_var(lru_deactivate_pvecs);
 553         }
 554 }
 555
 556 void lru_add_drain(void)
 557 {
 558         drain_cpu_pagevecs(get_cpu());
 559         put_cpu();
 560 }
 561
 562 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 563 {
 564         lru_add_drain();
 565 }
 566
 567 /*
 568  * Returns 0 for success
 569  */
 570 int lru_add_drain_all(void)
 571 {
 572         return schedule_on_each_cpu(lru_add_drain_per_cpu);
 573 }
 574
 575 /*
 576  * Batched page_cache_release().  Decrement the reference count on all the
 577  * passed pages.  If it fell to zero then remove the page from the LRU and
 578  * free it.
 579  *
 580  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
 581  * for the remainder of the operation.
 582  *
 583  * The locking in this function is against shrink_inactive_list(): we recheck
 584  * the page count inside the lock to see whether shrink_inactive_list()
 585  * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
 586  * will free it.
 587  */
 588 void release_pages(struct page **pages, int nr, int cold)
 589 {
 590         int i;
 591         LIST_HEAD(pages_to_free);
 592         struct zone *zone = NULL;
 593         unsigned long uninitialized_var(flags);
 594
 595         for (i = 0; i < nr; i++) {
 596                 struct page *page = pages[i];
 597
 598                 if (unlikely(PageCompound(page))) {
 599                         if (zone) {
 600                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 601                                 zone = NULL;
 602                         }
 603                         put_compound_page(page);
 604                         continue;
 605                 }
 606
 607                 if (!put_page_testzero(page))
 608                         continue;
 609
 610                 if (PageLRU(page)) {
 611                         struct zone *pagezone = page_zone(page);
 612
 613                         if (pagezone != zone) {
 614                                 if (zone)
 615                                         spin_unlock_irqrestore(&zone->lru_lock,
 616                                                                         flags);
 617                                 zone = pagezone;
 618                                 spin_lock_irqsave(&zone->lru_lock, flags);
 619                         }
 620                         VM_BUG_ON(!PageLRU(page));
 621                         __ClearPageLRU(page);
 622                         del_page_from_lru(zone, page);
 623                 }
 624
 625                 list_add(&page->lru, &pages_to_free);
 626         }
 627         if (zone)
 628                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 629
 630         free_hot_cold_page_list(&pages_to_free, cold);
 631 }
 632 EXPORT_SYMBOL(release_pages);
 633
 634 /*
 635  * The pages which we're about to release may be in the deferred lru-addition
 636  * queues.  That would prevent them from really being freed right now.  That's
 637  * OK from a correctness point of view but is inefficient - those pages may be
 638  * cache-warm and we want to give them back to the page allocator ASAP.
 639  *
 640  * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
 641  * and __pagevec_lru_add_active() call release_pages() directly to avoid
 642  * mutual recursion.
 643  */
 644 void __pagevec_release(struct pagevec *pvec)
 645 {
 646         lru_add_drain();
 647         release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
 648         pagevec_reinit(pvec);
 649 }
 650
 651 EXPORT_SYMBOL(__pagevec_release);
 652
 653 /* used by __split_huge_page_refcount() */
 654 void lru_add_page_tail(struct zone* zone,
 655                        struct page *page, struct page *page_tail)
 656 {
 657         int active;
 658         enum lru_list lru;
 659         const int file = 0;
 660
 661         VM_BUG_ON(!PageHead(page));
 662         VM_BUG_ON(PageCompound(page_tail));
 663         VM_BUG_ON(PageLRU(page_tail));
 664         VM_BUG_ON(!spin_is_locked(&zone->lru_lock));
 665
 666         SetPageLRU(page_tail);
 667
 668         if (page_evictable(page_tail, NULL)) {
 669                 struct lruvec *lruvec;
 670
 671                 if (PageActive(page)) {
 672                         SetPageActive(page_tail);
 673                         active = 1;
 674                         lru = LRU_ACTIVE_ANON;
 675                 } else {
 676                         active = 0;
 677                         lru = LRU_INACTIVE_ANON;
 678                 }
 679                 update_page_reclaim_stat(zone, page_tail, file, active);
 680                 lruvec = mem_cgroup_lru_add_list(zone, page_tail, lru);
 681                 if (likely(PageLRU(page)))
 682                         list_add(&page_tail->lru, page->lru.prev);
 683                 else
 684                         list_add(&page_tail->lru, &lruvec->lists[lru]);
 685                 __mod_zone_page_state(zone, NR_LRU_BASE + lru,
 686                                       hpage_nr_pages(page_tail));
 687         } else {
 688                 SetPageUnevictable(page_tail);
 689                 add_page_to_lru_list(zone, page_tail, LRU_UNEVICTABLE);
 690         }
 691 }
 692
 693 static void ____pagevec_lru_add_fn(struct page *page, void *arg)
 694 {
 695         enum lru_list lru = (enum lru_list)arg;
 696         struct zone *zone = page_zone(page);
 697         int file = is_file_lru(lru);
 698         int active = is_active_lru(lru);
 699
 700         VM_BUG_ON(PageActive(page));
 701         VM_BUG_ON(PageUnevictable(page));
 702         VM_BUG_ON(PageLRU(page));
 703
 704         SetPageLRU(page);
 705         if (active)
 706                 SetPageActive(page);
 707         update_page_reclaim_stat(zone, page, file, active);
 708         add_page_to_lru_list(zone, page, lru);
 709 }
 710
 711 /*
 712  * Add the passed pages to the LRU, then drop the caller's refcount
 713  * on them.  Reinitialises the caller's pagevec.
 714  */
 715 void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
 716 {
 717         VM_BUG_ON(is_unevictable_lru(lru));
 718
 719         pagevec_lru_move_fn(pvec, ____pagevec_lru_add_fn, (void *)lru);
 720 }
 721
 722 EXPORT_SYMBOL(____pagevec_lru_add);
 723
 724 /*
 725  * Try to drop buffers from the pages in a pagevec
 726  */
 727 void pagevec_strip(struct pagevec *pvec)
 728 {
 729         int i;
 730
 731         for (i = 0; i < pagevec_count(pvec); i++) {
 732                 struct page *page = pvec->pages[i];
 733
 734                 if (page_has_private(page) && trylock_page(page)) {
 735                         if (page_has_private(page))
 736                                 try_to_release_page(page, 0);
 737                         unlock_page(page);
 738                 }
 739         }
 740 }
 741
 742 /**
 743  * pagevec_lookup - gang pagecache lookup
 744  * @pvec:       Where the resulting pages are placed
 745  * @mapping:    The address_space to search
 746  * @start:      The starting page index
 747  * @nr_pages:   The maximum number of pages
 748  *
 749  * pagevec_lookup() will search for and return a group of up to @nr_pages pages
 750  * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
 751  * reference against the pages in @pvec.
 752  *
 753  * The search returns a group of mapping-contiguous pages with ascending
 754  * indexes.  There may be holes in the indices due to not-present pages.
 755  *
 756  * pagevec_lookup() returns the number of pages which were found.
 757  */
 758 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
 759                 pgoff_t start, unsigned nr_pages)
 760 {
 761         pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
 762         return pagevec_count(pvec);
 763 }
 764
 765 EXPORT_SYMBOL(pagevec_lookup);
 766
 767 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
 768                 pgoff_t *index, int tag, unsigned nr_pages)
 769 {
 770         pvec->nr = find_get_pages_tag(mapping, index, tag,
 771                                         nr_pages, pvec->pages);
 772         return pagevec_count(pvec);
 773 }
 774
 775 EXPORT_SYMBOL(pagevec_lookup_tag);
 776
 777 /*
 778  * Perform any setup for the swap system
 779  */
 780 void __init swap_setup(void)
 781 {
 782         unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
 783
 784 #ifdef CONFIG_SWAP
 785         bdi_init(swapper_space.backing_dev_info);
 786 #endif
 787
 788         /* Use a smaller cluster for small-memory machines */
 789         if (megs < 16)
 790                 page_cluster = 2;
 791         else
 792                 page_cluster = 3;
 793         /*
 794          * Right now other parts of the system means that we
 795          * _really_ don't want to cluster much more
 796          */
 797 }