mm/swap.c

   1 /*
   2  *  linux/mm/swap.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file contains the default values for the operation of the
   9  * Linux VM subsystem. Fine-tuning documentation can be found in
  10  * Documentation/sysctl/vm.txt.
  11  * Started 18.12.91
  12  * Swap aging added 23.2.95, Stephen Tweedie.
  13  * Buffermem limits added 12.3.98, Rik van Riel.
  14  */
  15
  16 #include <linux/mm.h>
  17 #include <linux/sched.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/swap.h>
  20 #include <linux/mman.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/pagevec.h>
  23 #include <linux/init.h>
  24 #include <linux/export.h>
  25 #include <linux/mm_inline.h>
  26 #include <linux/buffer_head.h>  /* for try_to_release_page() */
  27 #include <linux/percpu_counter.h>
  28 #include <linux/percpu.h>
  29 #include <linux/cpu.h>
  30 #include <linux/notifier.h>
  31 #include <linux/backing-dev.h>
  32 #include <linux/memcontrol.h>
  33 #include <linux/gfp.h>
  34
  35 #include "internal.h"
  36
  37 /* How many pages do we try to swap or page in/out together? */
  38 int page_cluster;
  39
  40 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
  41 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
  42 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
  43
  44 /*
  45  * This path almost never happens for VM activity - pages are normally
  46  * freed via pagevecs.  But it gets used by networking.
  47  */
  48 static void __page_cache_release(struct page *page)
  49 {
  50         if (PageLRU(page)) {
  51                 unsigned long flags;
  52                 struct zone *zone = page_zone(page);
  53
  54                 spin_lock_irqsave(&zone->lru_lock, flags);
  55                 VM_BUG_ON(!PageLRU(page));
  56                 __ClearPageLRU(page);
  57                 del_page_from_lru(zone, page);
  58                 spin_unlock_irqrestore(&zone->lru_lock, flags);
  59         }
  60 }
  61
  62 static void __put_single_page(struct page *page)
  63 {
  64         __page_cache_release(page);
  65         free_hot_cold_page(page, 0);
  66 }
  67
  68 static void __put_compound_page(struct page *page)
  69 {
  70         compound_page_dtor *dtor;
  71
  72         __page_cache_release(page);
  73         dtor = get_compound_page_dtor(page);
  74         (*dtor)(page);
  75 }
  76
  77 static void put_compound_page(struct page *page)
  78 {
  79         if (unlikely(PageTail(page))) {
  80                 /* __split_huge_page_refcount can run under us */
  81                 struct page *page_head = compound_trans_head(page);
  82
  83                 if (likely(page != page_head &&
  84                            get_page_unless_zero(page_head))) {
  85                         unsigned long flags;
  86                         /*
  87                          * page_head wasn't a dangling pointer but it
  88                          * may not be a head page anymore by the time
  89                          * we obtain the lock. That is ok as long as it
  90                          * can't be freed from under us.
  91                          */
  92                         flags = compound_lock_irqsave(page_head);
  93                         if (unlikely(!PageTail(page))) {
  94                                 /* __split_huge_page_refcount run before us */
  95                                 compound_unlock_irqrestore(page_head, flags);
  96                                 VM_BUG_ON(PageHead(page_head));
  97                                 if (put_page_testzero(page_head))
  98                                         __put_single_page(page_head);
  99                         out_put_single:
 100                                 if (put_page_testzero(page))
 101                                         __put_single_page(page);
 102                                 return;
 103                         }
 104                         VM_BUG_ON(page_head != page->first_page);
 105                         /*
 106                          * We can release the refcount taken by
 107                          * get_page_unless_zero() now that
 108                          * __split_huge_page_refcount() is blocked on
 109                          * the compound_lock.
 110                          */
 111                         if (put_page_testzero(page_head))
 112                                 VM_BUG_ON(1);
 113                         /* __split_huge_page_refcount will wait now */
 114                         VM_BUG_ON(page_mapcount(page) <= 0);
 115                         atomic_dec(&page->_mapcount);
 116                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
 117                         VM_BUG_ON(atomic_read(&page->_count) != 0);
 118                         compound_unlock_irqrestore(page_head, flags);
 119                         if (put_page_testzero(page_head)) {
 120                                 if (PageHead(page_head))
 121                                         __put_compound_page(page_head);
 122                                 else
 123                                         __put_single_page(page_head);
 124                         }
 125                 } else {
 126                         /* page_head is a dangling pointer */
 127                         VM_BUG_ON(PageTail(page));
 128                         goto out_put_single;
 129                 }
 130         } else if (put_page_testzero(page)) {
 131                 if (PageHead(page))
 132                         __put_compound_page(page);
 133                 else
 134                         __put_single_page(page);
 135         }
 136 }
 137
 138 void put_page(struct page *page)
 139 {
 140         if (unlikely(PageCompound(page)))
 141                 put_compound_page(page);
 142         else if (put_page_testzero(page))
 143                 __put_single_page(page);
 144 }
 145 EXPORT_SYMBOL(put_page);
 146
 147 /*
 148  * This function is exported but must not be called by anything other
 149  * than get_page(). It implements the slow path of get_page().
 150  */
 151 bool __get_page_tail(struct page *page)
 152 {
 153         /*
 154          * This takes care of get_page() if run on a tail page
 155          * returned by one of the get_user_pages/follow_page variants.
 156          * get_user_pages/follow_page itself doesn't need the compound
 157          * lock because it runs __get_page_tail_foll() under the
 158          * proper PT lock that already serializes against
 159          * split_huge_page().
 160          */
 161         unsigned long flags;
 162         bool got = false;
 163         struct page *page_head = compound_trans_head(page);
 164
 165         if (likely(page != page_head && get_page_unless_zero(page_head))) {
 166                 /*
 167                  * page_head wasn't a dangling pointer but it
 168                  * may not be a head page anymore by the time
 169                  * we obtain the lock. That is ok as long as it
 170                  * can't be freed from under us.
 171                  */
 172                 flags = compound_lock_irqsave(page_head);
 173                 /* here __split_huge_page_refcount won't run anymore */
 174                 if (likely(PageTail(page))) {
 175                         __get_page_tail_foll(page, false);
 176                         got = true;
 177                 }
 178                 compound_unlock_irqrestore(page_head, flags);
 179                 if (unlikely(!got))
 180                         put_page(page_head);
 181         }
 182         return got;
 183 }
 184 EXPORT_SYMBOL(__get_page_tail);
 185
 186 /**
 187  * put_pages_list() - release a list of pages
 188  * @pages: list of pages threaded on page->lru
 189  *
 190  * Release a list of pages which are strung together on page.lru.  Currently
 191  * used by read_cache_pages() and related error recovery code.
 192  */
 193 void put_pages_list(struct list_head *pages)
 194 {
 195         while (!list_empty(pages)) {
 196                 struct page *victim;
 197
 198                 victim = list_entry(pages->prev, struct page, lru);
 199                 list_del(&victim->lru);
 200                 page_cache_release(victim);
 201         }
 202 }
 203 EXPORT_SYMBOL(put_pages_list);
 204
 205 static void pagevec_lru_move_fn(struct pagevec *pvec,
 206                                 void (*move_fn)(struct page *page, void *arg),
 207                                 void *arg)
 208 {
 209         int i;
 210         struct zone *zone = NULL;
 211         unsigned long flags = 0;
 212
 213         for (i = 0; i < pagevec_count(pvec); i++) {
 214                 struct page *page = pvec->pages[i];
 215                 struct zone *pagezone = page_zone(page);
 216
 217                 if (pagezone != zone) {
 218                         if (zone)
 219                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 220                         zone = pagezone;
 221                         spin_lock_irqsave(&zone->lru_lock, flags);
 222                 }
 223
 224                 (*move_fn)(page, arg);
 225         }
 226         if (zone)
 227                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 228         release_pages(pvec->pages, pvec->nr, pvec->cold);
 229         pagevec_reinit(pvec);
 230 }
 231
 232 static void pagevec_move_tail_fn(struct page *page, void *arg)
 233 {
 234         int *pgmoved = arg;
 235         struct zone *zone = page_zone(page);
 236
 237         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 238                 enum lru_list lru = page_lru_base_type(page);
 239                 list_move_tail(&page->lru, &zone->lru[lru].list);
 240                 mem_cgroup_rotate_reclaimable_page(page);
 241                 (*pgmoved)++;
 242         }
 243 }
 244
 245 /*
 246  * pagevec_move_tail() must be called with IRQ disabled.
 247  * Otherwise this may cause nasty races.
 248  */
 249 static void pagevec_move_tail(struct pagevec *pvec)
 250 {
 251         int pgmoved = 0;
 252
 253         pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
 254         __count_vm_events(PGROTATED, pgmoved);
 255 }
 256
 257 /*
 258  * Writeback is about to end against a page which has been marked for immediate
 259  * reclaim.  If it still appears to be reclaimable, move it to the tail of the
 260  * inactive list.
 261  */
 262 void rotate_reclaimable_page(struct page *page)
 263 {
 264         if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
 265             !PageUnevictable(page) && PageLRU(page)) {
 266                 struct pagevec *pvec;
 267                 unsigned long flags;
 268
 269                 page_cache_get(page);
 270                 local_irq_save(flags);
 271                 pvec = &__get_cpu_var(lru_rotate_pvecs);
 272                 if (!pagevec_add(pvec, page))
 273                         pagevec_move_tail(pvec);
 274                 local_irq_restore(flags);
 275         }
 276 }
 277
 278 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
 279                                      int file, int rotated)
 280 {
 281         struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
 282         struct zone_reclaim_stat *memcg_reclaim_stat;
 283
 284         memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
 285
 286         reclaim_stat->recent_scanned[file]++;
 287         if (rotated)
 288                 reclaim_stat->recent_rotated[file]++;
 289
 290         if (!memcg_reclaim_stat)
 291                 return;
 292
 293         memcg_reclaim_stat->recent_scanned[file]++;
 294         if (rotated)
 295                 memcg_reclaim_stat->recent_rotated[file]++;
 296 }
 297
 298 static void __activate_page(struct page *page, void *arg)
 299 {
 300         struct zone *zone = page_zone(page);
 301
 302         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 303                 int file = page_is_file_cache(page);
 304                 int lru = page_lru_base_type(page);
 305                 del_page_from_lru_list(zone, page, lru);
 306
 307                 SetPageActive(page);
 308                 lru += LRU_ACTIVE;
 309                 add_page_to_lru_list(zone, page, lru);
 310                 __count_vm_event(PGACTIVATE);
 311
 312                 update_page_reclaim_stat(zone, page, file, 1);
 313         }
 314 }
 315
 316 #ifdef CONFIG_SMP
 317 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 318
 319 static void activate_page_drain(int cpu)
 320 {
 321         struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
 322
 323         if (pagevec_count(pvec))
 324                 pagevec_lru_move_fn(pvec, __activate_page, NULL);
 325 }
 326
 327 void activate_page(struct page *page)
 328 {
 329         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 330                 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
 331
 332                 page_cache_get(page);
 333                 if (!pagevec_add(pvec, page))
 334                         pagevec_lru_move_fn(pvec, __activate_page, NULL);
 335                 put_cpu_var(activate_page_pvecs);
 336         }
 337 }
 338
 339 #else
 340 static inline void activate_page_drain(int cpu)
 341 {
 342 }
 343
 344 void activate_page(struct page *page)
 345 {
 346         struct zone *zone = page_zone(page);
 347
 348         spin_lock_irq(&zone->lru_lock);
 349         __activate_page(page, NULL);
 350         spin_unlock_irq(&zone->lru_lock);
 351 }
 352 #endif
 353
 354 /*
 355  * Mark a page as having seen activity.
 356  *
 357  * inactive,unreferenced        ->      inactive,referenced
 358  * inactive,referenced          ->      active,unreferenced
 359  * active,unreferenced          ->      active,referenced
 360  */
 361 void mark_page_accessed(struct page *page)
 362 {
 363         if (!PageActive(page) && !PageUnevictable(page) &&
 364                         PageReferenced(page) && PageLRU(page)) {
 365                 activate_page(page);
 366                 ClearPageReferenced(page);
 367         } else if (!PageReferenced(page)) {
 368                 SetPageReferenced(page);
 369         }
 370 }
 371
 372 EXPORT_SYMBOL(mark_page_accessed);
 373
 374 void __lru_cache_add(struct page *page, enum lru_list lru)
 375 {
 376         struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
 377
 378         page_cache_get(page);
 379         if (!pagevec_add(pvec, page))
 380                 ____pagevec_lru_add(pvec, lru);
 381         put_cpu_var(lru_add_pvecs);
 382 }
 383 EXPORT_SYMBOL(__lru_cache_add);
 384
 385 /**
 386  * lru_cache_add_lru - add a page to a page list
 387  * @page: the page to be added to the LRU.
 388  * @lru: the LRU list to which the page is added.
 389  */
 390 void lru_cache_add_lru(struct page *page, enum lru_list lru)
 391 {
 392         if (PageActive(page)) {
 393                 VM_BUG_ON(PageUnevictable(page));
 394                 ClearPageActive(page);
 395         } else if (PageUnevictable(page)) {
 396                 VM_BUG_ON(PageActive(page));
 397                 ClearPageUnevictable(page);
 398         }
 399
 400         VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
 401         __lru_cache_add(page, lru);
 402 }
 403
 404 /**
 405  * add_page_to_unevictable_list - add a page to the unevictable list
 406  * @page:  the page to be added to the unevictable list
 407  *
 408  * Add page directly to its zone's unevictable list.  To avoid races with
 409  * tasks that might be making the page evictable, through eg. munlock,
 410  * munmap or exit, while it's not on the lru, we want to add the page
 411  * while it's locked or otherwise "invisible" to other tasks.  This is
 412  * difficult to do when using the pagevec cache, so bypass that.
 413  */
 414 void add_page_to_unevictable_list(struct page *page)
 415 {
 416         struct zone *zone = page_zone(page);
 417
 418         spin_lock_irq(&zone->lru_lock);
 419         SetPageUnevictable(page);
 420         SetPageLRU(page);
 421         add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
 422         spin_unlock_irq(&zone->lru_lock);
 423 }
 424
 425 /*
 426  * If the page can not be invalidated, it is moved to the
 427  * inactive list to speed up its reclaim.  It is moved to the
 428  * head of the list, rather than the tail, to give the flusher
 429  * threads some time to write it out, as this is much more
 430  * effective than the single-page writeout from reclaim.
 431  *
 432  * If the page isn't page_mapped and dirty/writeback, the page
 433  * could reclaim asap using PG_reclaim.
 434  *
 435  * 1. active, mapped page -> none
 436  * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
 437  * 3. inactive, mapped page -> none
 438  * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
 439  * 5. inactive, clean -> inactive, tail
 440  * 6. Others -> none
 441  *
 442  * In 4, why it moves inactive's head, the VM expects the page would
 443  * be write it out by flusher threads as this is much more effective
 444  * than the single-page writeout from reclaim.
 445  */
 446 static void lru_deactivate_fn(struct page *page, void *arg)
 447 {
 448         int lru, file;
 449         bool active;
 450         struct zone *zone = page_zone(page);
 451
 452         if (!PageLRU(page))
 453                 return;
 454
 455         if (PageUnevictable(page))
 456                 return;
 457
 458         /* Some processes are using the page */
 459         if (page_mapped(page))
 460                 return;
 461
 462         active = PageActive(page);
 463
 464         file = page_is_file_cache(page);
 465         lru = page_lru_base_type(page);
 466         del_page_from_lru_list(zone, page, lru + active);
 467         ClearPageActive(page);
 468         ClearPageReferenced(page);
 469         add_page_to_lru_list(zone, page, lru);
 470
 471         if (PageWriteback(page) || PageDirty(page)) {
 472                 /*
 473                  * PG_reclaim could be raced with end_page_writeback
 474                  * It can make readahead confusing.  But race window
 475                  * is _really_ small and  it's non-critical problem.
 476                  */
 477                 SetPageReclaim(page);
 478         } else {
 479                 /*
 480                  * The page's writeback ends up during pagevec
 481                  * We moves tha page into tail of inactive.
 482                  */
 483                 list_move_tail(&page->lru, &zone->lru[lru].list);
 484                 mem_cgroup_rotate_reclaimable_page(page);
 485                 __count_vm_event(PGROTATED);
 486         }
 487
 488         if (active)
 489                 __count_vm_event(PGDEACTIVATE);
 490         update_page_reclaim_stat(zone, page, file, 0);
 491 }
 492
 493 /*
 494  * Drain pages out of the cpu's pagevecs.
 495  * Either "cpu" is the current CPU, and preemption has already been
 496  * disabled; or "cpu" is being hot-unplugged, and is already dead.
 497  */
 498 static void drain_cpu_pagevecs(int cpu)
 499 {
 500         struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
 501         struct pagevec *pvec;
 502         int lru;
 503
 504         for_each_lru(lru) {
 505                 pvec = &pvecs[lru - LRU_BASE];
 506                 if (pagevec_count(pvec))
 507                         ____pagevec_lru_add(pvec, lru);
 508         }
 509
 510         pvec = &per_cpu(lru_rotate_pvecs, cpu);
 511         if (pagevec_count(pvec)) {
 512                 unsigned long flags;
 513
 514                 /* No harm done if a racing interrupt already did this */
 515                 local_irq_save(flags);
 516                 pagevec_move_tail(pvec);
 517                 local_irq_restore(flags);
 518         }
 519
 520         pvec = &per_cpu(lru_deactivate_pvecs, cpu);
 521         if (pagevec_count(pvec))
 522                 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 523
 524         activate_page_drain(cpu);
 525 }
 526
 527 /**
 528  * deactivate_page - forcefully deactivate a page
 529  * @page: page to deactivate
 530  *
 531  * This function hints the VM that @page is a good reclaim candidate,
 532  * for example if its invalidation fails due to the page being dirty
 533  * or under writeback.
 534  */
 535 void deactivate_page(struct page *page)
 536 {
 537         /*
 538          * In a workload with many unevictable page such as mprotect, unevictable
 539          * page deactivation for accelerating reclaim is pointless.
 540          */
 541         if (PageUnevictable(page))
 542                 return;
 543
 544         if (likely(get_page_unless_zero(page))) {
 545                 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
 546
 547                 if (!pagevec_add(pvec, page))
 548                         pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 549                 put_cpu_var(lru_deactivate_pvecs);
 550         }
 551 }
 552
 553 void lru_add_drain(void)
 554 {
 555         drain_cpu_pagevecs(get_cpu());
 556         put_cpu();
 557 }
 558
 559 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 560 {
 561         lru_add_drain();
 562 }
 563
 564 /*
 565  * Returns 0 for success
 566  */
 567 int lru_add_drain_all(void)
 568 {
 569         return schedule_on_each_cpu(lru_add_drain_per_cpu);
 570 }
 571
 572 /*
 573  * Batched page_cache_release().  Decrement the reference count on all the
 574  * passed pages.  If it fell to zero then remove the page from the LRU and
 575  * free it.
 576  *
 577  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
 578  * for the remainder of the operation.
 579  *
 580  * The locking in this function is against shrink_inactive_list(): we recheck
 581  * the page count inside the lock to see whether shrink_inactive_list()
 582  * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
 583  * will free it.
 584  */
 585 void release_pages(struct page **pages, int nr, int cold)
 586 {
 587         int i;
 588         LIST_HEAD(pages_to_free);
 589         struct zone *zone = NULL;
 590         unsigned long uninitialized_var(flags);
 591
 592         for (i = 0; i < nr; i++) {
 593                 struct page *page = pages[i];
 594
 595                 if (unlikely(PageCompound(page))) {
 596                         if (zone) {
 597                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 598                                 zone = NULL;
 599                         }
 600                         put_compound_page(page);
 601                         continue;
 602                 }
 603
 604                 if (!put_page_testzero(page))
 605                         continue;
 606
 607                 if (PageLRU(page)) {
 608                         struct zone *pagezone = page_zone(page);
 609
 610                         if (pagezone != zone) {
 611                                 if (zone)
 612                                         spin_unlock_irqrestore(&zone->lru_lock,
 613                                                                         flags);
 614                                 zone = pagezone;
 615                                 spin_lock_irqsave(&zone->lru_lock, flags);
 616                         }
 617                         VM_BUG_ON(!PageLRU(page));
 618                         __ClearPageLRU(page);
 619                         del_page_from_lru(zone, page);
 620                 }
 621
 622                 list_add(&page->lru, &pages_to_free);
 623         }
 624         if (zone)
 625                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 626
 627         free_hot_cold_page_list(&pages_to_free, cold);
 628 }
 629 EXPORT_SYMBOL(release_pages);
 630
 631 /*
 632  * The pages which we're about to release may be in the deferred lru-addition
 633  * queues.  That would prevent them from really being freed right now.  That's
 634  * OK from a correctness point of view but is inefficient - those pages may be
 635  * cache-warm and we want to give them back to the page allocator ASAP.
 636  *
 637  * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
 638  * and __pagevec_lru_add_active() call release_pages() directly to avoid
 639  * mutual recursion.
 640  */
 641 void __pagevec_release(struct pagevec *pvec)
 642 {
 643         lru_add_drain();
 644         release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
 645         pagevec_reinit(pvec);
 646 }
 647
 648 EXPORT_SYMBOL(__pagevec_release);
 649
 650 /* used by __split_huge_page_refcount() */
 651 void lru_add_page_tail(struct zone* zone,
 652                        struct page *page, struct page *page_tail)
 653 {
 654         int active;
 655         enum lru_list lru;
 656         const int file = 0;
 657         struct list_head *head;
 658
 659         VM_BUG_ON(!PageHead(page));
 660         VM_BUG_ON(PageCompound(page_tail));
 661         VM_BUG_ON(PageLRU(page_tail));
 662         VM_BUG_ON(!spin_is_locked(&zone->lru_lock));
 663
 664         SetPageLRU(page_tail);
 665
 666         if (page_evictable(page_tail, NULL)) {
 667                 if (PageActive(page)) {
 668                         SetPageActive(page_tail);
 669                         active = 1;
 670                         lru = LRU_ACTIVE_ANON;
 671                 } else {
 672                         active = 0;
 673                         lru = LRU_INACTIVE_ANON;
 674                 }
 675                 update_page_reclaim_stat(zone, page_tail, file, active);
 676                 if (likely(PageLRU(page)))
 677                         head = page->lru.prev;
 678                 else
 679                         head = &zone->lru[lru].list;
 680                 __add_page_to_lru_list(zone, page_tail, lru, head);
 681         } else {
 682                 SetPageUnevictable(page_tail);
 683                 add_page_to_lru_list(zone, page_tail, LRU_UNEVICTABLE);
 684         }
 685 }
 686
 687 static void ____pagevec_lru_add_fn(struct page *page, void *arg)
 688 {
 689         enum lru_list lru = (enum lru_list)arg;
 690         struct zone *zone = page_zone(page);
 691         int file = is_file_lru(lru);
 692         int active = is_active_lru(lru);
 693
 694         VM_BUG_ON(PageActive(page));
 695         VM_BUG_ON(PageUnevictable(page));
 696         VM_BUG_ON(PageLRU(page));
 697
 698         SetPageLRU(page);
 699         if (active)
 700                 SetPageActive(page);
 701         update_page_reclaim_stat(zone, page, file, active);
 702         add_page_to_lru_list(zone, page, lru);
 703 }
 704
 705 /*
 706  * Add the passed pages to the LRU, then drop the caller's refcount
 707  * on them.  Reinitialises the caller's pagevec.
 708  */
 709 void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
 710 {
 711         VM_BUG_ON(is_unevictable_lru(lru));
 712
 713         pagevec_lru_move_fn(pvec, ____pagevec_lru_add_fn, (void *)lru);
 714 }
 715
 716 EXPORT_SYMBOL(____pagevec_lru_add);
 717
 718 /*
 719  * Try to drop buffers from the pages in a pagevec
 720  */
 721 void pagevec_strip(struct pagevec *pvec)
 722 {
 723         int i;
 724
 725         for (i = 0; i < pagevec_count(pvec); i++) {
 726                 struct page *page = pvec->pages[i];
 727
 728                 if (page_has_private(page) && trylock_page(page)) {
 729                         if (page_has_private(page))
 730                                 try_to_release_page(page, 0);
 731                         unlock_page(page);
 732                 }
 733         }
 734 }
 735
 736 /**
 737  * pagevec_lookup - gang pagecache lookup
 738  * @pvec:       Where the resulting pages are placed
 739  * @mapping:    The address_space to search
 740  * @start:      The starting page index
 741  * @nr_pages:   The maximum number of pages
 742  *
 743  * pagevec_lookup() will search for and return a group of up to @nr_pages pages
 744  * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
 745  * reference against the pages in @pvec.
 746  *
 747  * The search returns a group of mapping-contiguous pages with ascending
 748  * indexes.  There may be holes in the indices due to not-present pages.
 749  *
 750  * pagevec_lookup() returns the number of pages which were found.
 751  */
 752 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
 753                 pgoff_t start, unsigned nr_pages)
 754 {
 755         pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
 756         return pagevec_count(pvec);
 757 }
 758
 759 EXPORT_SYMBOL(pagevec_lookup);
 760
 761 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
 762                 pgoff_t *index, int tag, unsigned nr_pages)
 763 {
 764         pvec->nr = find_get_pages_tag(mapping, index, tag,
 765                                         nr_pages, pvec->pages);
 766         return pagevec_count(pvec);
 767 }
 768
 769 EXPORT_SYMBOL(pagevec_lookup_tag);
 770
 771 /*
 772  * Perform any setup for the swap system
 773  */
 774 void __init swap_setup(void)
 775 {
 776         unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
 777
 778 #ifdef CONFIG_SWAP
 779         bdi_init(swapper_space.backing_dev_info);
 780 #endif
 781
 782         /* Use a smaller cluster for small-memory machines */
 783         if (megs < 16)
 784                 page_cluster = 2;
 785         else
 786                 page_cluster = 3;
 787         /*
 788          * Right now other parts of the system means that we
 789          * _really_ don't want to cluster much more
 790          */
 791 }