mm: vmscan: flatten kswapd priority loop
[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / vmscan.c
1 /*
2  *  linux/mm/vmscan.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, Stephen Tweedie.
7  *  kswapd added: 7.1.96  sct
8  *  Removed kswapd_ctl limits, and swap out as many pages as needed
9  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11  *  Multiqueue VM started 5.8.00, Rik van Riel.
12  */
13
14 #include <linux/mm.h>
15 #include <linux/module.h>
16 #include <linux/gfp.h>
17 #include <linux/kernel_stat.h>
18 #include <linux/swap.h>
19 #include <linux/pagemap.h>
20 #include <linux/init.h>
21 #include <linux/highmem.h>
22 #include <linux/vmpressure.h>
23 #include <linux/vmstat.h>
24 #include <linux/file.h>
25 #include <linux/writeback.h>
26 #include <linux/blkdev.h>
27 #include <linux/buffer_head.h>  /* for try_to_release_page(),
28                                         buffer_heads_over_limit */
29 #include <linux/mm_inline.h>
30 #include <linux/backing-dev.h>
31 #include <linux/rmap.h>
32 #include <linux/topology.h>
33 #include <linux/cpu.h>
34 #include <linux/cpuset.h>
35 #include <linux/compaction.h>
36 #include <linux/notifier.h>
37 #include <linux/rwsem.h>
38 #include <linux/delay.h>
39 #include <linux/kthread.h>
40 #include <linux/freezer.h>
41 #include <linux/memcontrol.h>
42 #include <linux/delayacct.h>
43 #include <linux/sysctl.h>
44 #include <linux/oom.h>
45 #include <linux/prefetch.h>
46
47 #include <asm/tlbflush.h>
48 #include <asm/div64.h>
49
50 #include <linux/swapops.h>
51
52 #include "internal.h"
53
54 #define CREATE_TRACE_POINTS
55 #include <trace/events/vmscan.h>
56
57 struct scan_control {
58         /* Incremented by the number of inactive pages that were scanned */
59         unsigned long nr_scanned;
60
61         /* Number of pages freed so far during a call to shrink_zones() */
62         unsigned long nr_reclaimed;
63
64         /* How many pages shrink_list() should reclaim */
65         unsigned long nr_to_reclaim;
66
67         unsigned long hibernation_mode;
68
69         /* This context's GFP mask */
70         gfp_t gfp_mask;
71
72         int may_writepage;
73
74         /* Can mapped pages be reclaimed? */
75         int may_unmap;
76
77         /* Can pages be swapped as part of reclaim? */
78         int may_swap;
79
80         int order;
81
82         /* Scan (total_size >> priority) pages at once */
83         int priority;
84
85         /*
86          * The memory cgroup that hit its limit and as a result is the
87          * primary target of this reclaim invocation.
88          */
89         struct mem_cgroup *target_mem_cgroup;
90
91         /*
92          * Nodemask of nodes allowed by the caller. If NULL, all nodes
93          * are scanned.
94          */
95         nodemask_t      *nodemask;
96 };
97
98 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
99
100 #ifdef ARCH_HAS_PREFETCH
101 #define prefetch_prev_lru_page(_page, _base, _field)                    \
102         do {                                                            \
103                 if ((_page)->lru.prev != _base) {                       \
104                         struct page *prev;                              \
105                                                                         \
106                         prev = lru_to_page(&(_page->lru));              \
107                         prefetch(&prev->_field);                        \
108                 }                                                       \
109         } while (0)
110 #else
111 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
112 #endif
113
114 #ifdef ARCH_HAS_PREFETCHW
115 #define prefetchw_prev_lru_page(_page, _base, _field)                   \
116         do {                                                            \
117                 if ((_page)->lru.prev != _base) {                       \
118                         struct page *prev;                              \
119                                                                         \
120                         prev = lru_to_page(&(_page->lru));              \
121                         prefetchw(&prev->_field);                       \
122                 }                                                       \
123         } while (0)
124 #else
125 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
126 #endif
127
128 /*
129  * From 0 .. 100.  Higher means more swappy.
130  */
131 int vm_swappiness = 60;
132 unsigned long vm_total_pages;   /* The total number of pages which the VM controls */
133
134 static LIST_HEAD(shrinker_list);
135 static DECLARE_RWSEM(shrinker_rwsem);
136
137 #ifdef CONFIG_MEMCG
138 static bool global_reclaim(struct scan_control *sc)
139 {
140         return !sc->target_mem_cgroup;
141 }
142 #else
143 static bool global_reclaim(struct scan_control *sc)
144 {
145         return true;
146 }
147 #endif
148
149 static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru)
150 {
151         if (!mem_cgroup_disabled())
152                 return mem_cgroup_get_lru_size(lruvec, lru);
153
154         return zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru);
155 }
156
157 /*
158  * Add a shrinker callback to be called from the vm
159  */
160 void register_shrinker(struct shrinker *shrinker)
161 {
162         atomic_long_set(&shrinker->nr_in_batch, 0);
163         down_write(&shrinker_rwsem);
164         list_add_tail(&shrinker->list, &shrinker_list);
165         up_write(&shrinker_rwsem);
166 }
167 EXPORT_SYMBOL(register_shrinker);
168
169 /*
170  * Remove one
171  */
172 void unregister_shrinker(struct shrinker *shrinker)
173 {
174         down_write(&shrinker_rwsem);
175         list_del(&shrinker->list);
176         up_write(&shrinker_rwsem);
177 }
178 EXPORT_SYMBOL(unregister_shrinker);
179
180 static inline int do_shrinker_shrink(struct shrinker *shrinker,
181                                      struct shrink_control *sc,
182                                      unsigned long nr_to_scan)
183 {
184         sc->nr_to_scan = nr_to_scan;
185         return (*shrinker->shrink)(shrinker, sc);
186 }
187
188 #define SHRINK_BATCH 128
189 /*
190  * Call the shrink functions to age shrinkable caches
191  *
192  * Here we assume it costs one seek to replace a lru page and that it also
193  * takes a seek to recreate a cache object.  With this in mind we age equal
194  * percentages of the lru and ageable caches.  This should balance the seeks
195  * generated by these structures.
196  *
197  * If the vm encountered mapped pages on the LRU it increase the pressure on
198  * slab to avoid swapping.
199  *
200  * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
201  *
202  * `lru_pages' represents the number of on-LRU pages in all the zones which
203  * are eligible for the caller's allocation attempt.  It is used for balancing
204  * slab reclaim versus page reclaim.
205  *
206  * Returns the number of slab objects which we shrunk.
207  */
208 unsigned long shrink_slab(struct shrink_control *shrink,
209                           unsigned long nr_pages_scanned,
210                           unsigned long lru_pages)
211 {
212         struct shrinker *shrinker;
213         unsigned long ret = 0;
214
215         if (nr_pages_scanned == 0)
216                 nr_pages_scanned = SWAP_CLUSTER_MAX;
217
218         if (!down_read_trylock(&shrinker_rwsem)) {
219                 /* Assume we'll be able to shrink next time */
220                 ret = 1;
221                 goto out;
222         }
223
224         list_for_each_entry(shrinker, &shrinker_list, list) {
225                 unsigned long long delta;
226                 long total_scan;
227                 long max_pass;
228                 int shrink_ret = 0;
229                 long nr;
230                 long new_nr;
231                 long batch_size = shrinker->batch ? shrinker->batch
232                                                   : SHRINK_BATCH;
233
234                 max_pass = do_shrinker_shrink(shrinker, shrink, 0);
235                 if (max_pass <= 0)
236                         continue;
237
238                 /*
239                  * copy the current shrinker scan count into a local variable
240                  * and zero it so that other concurrent shrinker invocations
241                  * don't also do this scanning work.
242                  */
243                 nr = atomic_long_xchg(&shrinker->nr_in_batch, 0);
244
245                 total_scan = nr;
246                 delta = (4 * nr_pages_scanned) / shrinker->seeks;
247                 delta *= max_pass;
248                 do_div(delta, lru_pages + 1);
249                 total_scan += delta;
250                 if (total_scan < 0) {
251                         printk(KERN_ERR "shrink_slab: %pF negative objects to "
252                                "delete nr=%ld\n",
253                                shrinker->shrink, total_scan);
254                         total_scan = max_pass;
255                 }
256
257                 /*
258                  * We need to avoid excessive windup on filesystem shrinkers
259                  * due to large numbers of GFP_NOFS allocations causing the
260                  * shrinkers to return -1 all the time. This results in a large
261                  * nr being built up so when a shrink that can do some work
262                  * comes along it empties the entire cache due to nr >>>
263                  * max_pass.  This is bad for sustaining a working set in
264                  * memory.
265                  *
266                  * Hence only allow the shrinker to scan the entire cache when
267                  * a large delta change is calculated directly.
268                  */
269                 if (delta < max_pass / 4)
270                         total_scan = min(total_scan, max_pass / 2);
271
272                 /*
273                  * Avoid risking looping forever due to too large nr value:
274                  * never try to free more than twice the estimate number of
275                  * freeable entries.
276                  */
277                 if (total_scan > max_pass * 2)
278                         total_scan = max_pass * 2;
279
280                 trace_mm_shrink_slab_start(shrinker, shrink, nr,
281                                         nr_pages_scanned, lru_pages,
282                                         max_pass, delta, total_scan);
283
284                 while (total_scan >= batch_size) {
285                         int nr_before;
286
287                         nr_before = do_shrinker_shrink(shrinker, shrink, 0);
288                         shrink_ret = do_shrinker_shrink(shrinker, shrink,
289                                                         batch_size);
290                         if (shrink_ret == -1)
291                                 break;
292                         if (shrink_ret < nr_before)
293                                 ret += nr_before - shrink_ret;
294                         count_vm_events(SLABS_SCANNED, batch_size);
295                         total_scan -= batch_size;
296
297                         cond_resched();
298                 }
299
300                 /*
301                  * move the unused scan count back into the shrinker in a
302                  * manner that handles concurrent updates. If we exhausted the
303                  * scan, there is no need to do an update.
304                  */
305                 if (total_scan > 0)
306                         new_nr = atomic_long_add_return(total_scan,
307                                         &shrinker->nr_in_batch);
308                 else
309                         new_nr = atomic_long_read(&shrinker->nr_in_batch);
310
311                 trace_mm_shrink_slab_end(shrinker, shrink_ret, nr, new_nr);
312         }
313         up_read(&shrinker_rwsem);
314 out:
315         cond_resched();
316         return ret;
317 }
318
319 static inline int is_page_cache_freeable(struct page *page)
320 {
321         /*
322          * A freeable page cache page is referenced only by the caller
323          * that isolated the page, the page cache radix tree and
324          * optional buffer heads at page->private.
325          */
326         return page_count(page) - page_has_private(page) == 2;
327 }
328
329 static int may_write_to_queue(struct backing_dev_info *bdi,
330                               struct scan_control *sc)
331 {
332         if (current->flags & PF_SWAPWRITE)
333                 return 1;
334         if (!bdi_write_congested(bdi))
335                 return 1;
336         if (bdi == current->backing_dev_info)
337                 return 1;
338         return 0;
339 }
340
341 /*
342  * We detected a synchronous write error writing a page out.  Probably
343  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
344  * fsync(), msync() or close().
345  *
346  * The tricky part is that after writepage we cannot touch the mapping: nothing
347  * prevents it from being freed up.  But we have a ref on the page and once
348  * that page is locked, the mapping is pinned.
349  *
350  * We're allowed to run sleeping lock_page() here because we know the caller has
351  * __GFP_FS.
352  */
353 static void handle_write_error(struct address_space *mapping,
354                                 struct page *page, int error)
355 {
356         lock_page(page);
357         if (page_mapping(page) == mapping)
358                 mapping_set_error(mapping, error);
359         unlock_page(page);
360 }
361
362 /* possible outcome of pageout() */
363 typedef enum {
364         /* failed to write page out, page is locked */
365         PAGE_KEEP,
366         /* move page to the active list, page is locked */
367         PAGE_ACTIVATE,
368         /* page has been sent to the disk successfully, page is unlocked */
369         PAGE_SUCCESS,
370         /* page is clean and locked */
371         PAGE_CLEAN,
372 } pageout_t;
373
374 /*
375  * pageout is called by shrink_page_list() for each dirty page.
376  * Calls ->writepage().
377  */
378 static pageout_t pageout(struct page *page, struct address_space *mapping,
379                          struct scan_control *sc)
380 {
381         /*
382          * If the page is dirty, only perform writeback if that write
383          * will be non-blocking.  To prevent this allocation from being
384          * stalled by pagecache activity.  But note that there may be
385          * stalls if we need to run get_block().  We could test
386          * PagePrivate for that.
387          *
388          * If this process is currently in __generic_file_aio_write() against
389          * this page's queue, we can perform writeback even if that
390          * will block.
391          *
392          * If the page is swapcache, write it back even if that would
393          * block, for some throttling. This happens by accident, because
394          * swap_backing_dev_info is bust: it doesn't reflect the
395          * congestion state of the swapdevs.  Easy to fix, if needed.
396          */
397         if (!is_page_cache_freeable(page))
398                 return PAGE_KEEP;
399         if (!mapping) {
400                 /*
401                  * Some data journaling orphaned pages can have
402                  * page->mapping == NULL while being dirty with clean buffers.
403                  */
404                 if (page_has_private(page)) {
405                         if (try_to_free_buffers(page)) {
406                                 ClearPageDirty(page);
407                                 printk("%s: orphaned page\n", __func__);
408                                 return PAGE_CLEAN;
409                         }
410                 }
411                 return PAGE_KEEP;
412         }
413         if (mapping->a_ops->writepage == NULL)
414                 return PAGE_ACTIVATE;
415         if (!may_write_to_queue(mapping->backing_dev_info, sc))
416                 return PAGE_KEEP;
417
418         if (clear_page_dirty_for_io(page)) {
419                 int res;
420                 struct writeback_control wbc = {
421                         .sync_mode = WB_SYNC_NONE,
422                         .nr_to_write = SWAP_CLUSTER_MAX,
423                         .range_start = 0,
424                         .range_end = LLONG_MAX,
425                         .for_reclaim = 1,
426                 };
427
428                 SetPageReclaim(page);
429                 res = mapping->a_ops->writepage(page, &wbc);
430                 if (res < 0)
431                         handle_write_error(mapping, page, res);
432                 if (res == AOP_WRITEPAGE_ACTIVATE) {
433                         ClearPageReclaim(page);
434                         return PAGE_ACTIVATE;
435                 }
436
437                 if (!PageWriteback(page)) {
438                         /* synchronous write or broken a_ops? */
439                         ClearPageReclaim(page);
440                 }
441                 trace_mm_vmscan_writepage(page, trace_reclaim_flags(page));
442                 inc_zone_page_state(page, NR_VMSCAN_WRITE);
443                 return PAGE_SUCCESS;
444         }
445
446         return PAGE_CLEAN;
447 }
448
449 /*
450  * Same as remove_mapping, but if the page is removed from the mapping, it
451  * gets returned with a refcount of 0.
452  */
453 static int __remove_mapping(struct address_space *mapping, struct page *page)
454 {
455         BUG_ON(!PageLocked(page));
456         BUG_ON(mapping != page_mapping(page));
457
458         spin_lock_irq(&mapping->tree_lock);
459         /*
460          * The non racy check for a busy page.
461          *
462          * Must be careful with the order of the tests. When someone has
463          * a ref to the page, it may be possible that they dirty it then
464          * drop the reference. So if PageDirty is tested before page_count
465          * here, then the following race may occur:
466          *
467          * get_user_pages(&page);
468          * [user mapping goes away]
469          * write_to(page);
470          *                              !PageDirty(page)    [good]
471          * SetPageDirty(page);
472          * put_page(page);
473          *                              !page_count(page)   [good, discard it]
474          *
475          * [oops, our write_to data is lost]
476          *
477          * Reversing the order of the tests ensures such a situation cannot
478          * escape unnoticed. The smp_rmb is needed to ensure the page->flags
479          * load is not satisfied before that of page->_count.
480          *
481          * Note that if SetPageDirty is always performed via set_page_dirty,
482          * and thus under tree_lock, then this ordering is not required.
483          */
484         if (!page_freeze_refs(page, 2))
485                 goto cannot_free;
486         /* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
487         if (unlikely(PageDirty(page))) {
488                 page_unfreeze_refs(page, 2);
489                 goto cannot_free;
490         }
491
492         if (PageSwapCache(page)) {
493                 swp_entry_t swap = { .val = page_private(page) };
494                 __delete_from_swap_cache(page);
495                 spin_unlock_irq(&mapping->tree_lock);
496                 swapcache_free(swap, page);
497         } else {
498                 void (*freepage)(struct page *);
499
500                 freepage = mapping->a_ops->freepage;
501
502                 __delete_from_page_cache(page);
503                 spin_unlock_irq(&mapping->tree_lock);
504                 mem_cgroup_uncharge_cache_page(page);
505
506                 if (freepage != NULL)
507                         freepage(page);
508         }
509
510         return 1;
511
512 cannot_free:
513         spin_unlock_irq(&mapping->tree_lock);
514         return 0;
515 }
516
517 /*
518  * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
519  * someone else has a ref on the page, abort and return 0.  If it was
520  * successfully detached, return 1.  Assumes the caller has a single ref on
521  * this page.
522  */
523 int remove_mapping(struct address_space *mapping, struct page *page)
524 {
525         if (__remove_mapping(mapping, page)) {
526                 /*
527                  * Unfreezing the refcount with 1 rather than 2 effectively
528                  * drops the pagecache ref for us without requiring another
529                  * atomic operation.
530                  */
531                 page_unfreeze_refs(page, 1);
532                 return 1;
533         }
534         return 0;
535 }
536
537 /**
538  * putback_lru_page - put previously isolated page onto appropriate LRU list
539  * @page: page to be put back to appropriate lru list
540  *
541  * Add previously isolated @page to appropriate LRU list.
542  * Page may still be unevictable for other reasons.
543  *
544  * lru_lock must not be held, interrupts must be enabled.
545  */
546 void putback_lru_page(struct page *page)
547 {
548         int lru;
549         int active = !!TestClearPageActive(page);
550         int was_unevictable = PageUnevictable(page);
551
552         VM_BUG_ON(PageLRU(page));
553
554 redo:
555         ClearPageUnevictable(page);
556
557         if (page_evictable(page)) {
558                 /*
559                  * For evictable pages, we can use the cache.
560                  * In event of a race, worst case is we end up with an
561                  * unevictable page on [in]active list.
562                  * We know how to handle that.
563                  */
564                 lru = active + page_lru_base_type(page);
565                 lru_cache_add_lru(page, lru);
566         } else {
567                 /*
568                  * Put unevictable pages directly on zone's unevictable
569                  * list.
570                  */
571                 lru = LRU_UNEVICTABLE;
572                 add_page_to_unevictable_list(page);
573                 /*
574                  * When racing with an mlock or AS_UNEVICTABLE clearing
575                  * (page is unlocked) make sure that if the other thread
576                  * does not observe our setting of PG_lru and fails
577                  * isolation/check_move_unevictable_pages,
578                  * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
579                  * the page back to the evictable list.
580                  *
581                  * The other side is TestClearPageMlocked() or shmem_lock().
582                  */
583                 smp_mb();
584         }
585
586         /*
587          * page's status can change while we move it among lru. If an evictable
588          * page is on unevictable list, it never be freed. To avoid that,
589          * check after we added it to the list, again.
590          */
591         if (lru == LRU_UNEVICTABLE && page_evictable(page)) {
592                 if (!isolate_lru_page(page)) {
593                         put_page(page);
594                         goto redo;
595                 }
596                 /* This means someone else dropped this page from LRU
597                  * So, it will be freed or putback to LRU again. There is
598                  * nothing to do here.
599                  */
600         }
601
602         if (was_unevictable && lru != LRU_UNEVICTABLE)
603                 count_vm_event(UNEVICTABLE_PGRESCUED);
604         else if (!was_unevictable && lru == LRU_UNEVICTABLE)
605                 count_vm_event(UNEVICTABLE_PGCULLED);
606
607         put_page(page);         /* drop ref from isolate */
608 }
609
610 enum page_references {
611         PAGEREF_RECLAIM,
612         PAGEREF_RECLAIM_CLEAN,
613         PAGEREF_KEEP,
614         PAGEREF_ACTIVATE,
615 };
616
617 static enum page_references page_check_references(struct page *page,
618                                                   struct scan_control *sc)
619 {
620         int referenced_ptes, referenced_page;
621         unsigned long vm_flags;
622
623         referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
624                                           &vm_flags);
625         referenced_page = TestClearPageReferenced(page);
626
627         /*
628          * Mlock lost the isolation race with us.  Let try_to_unmap()
629          * move the page to the unevictable list.
630          */
631         if (vm_flags & VM_LOCKED)
632                 return PAGEREF_RECLAIM;
633
634         if (referenced_ptes) {
635                 if (PageSwapBacked(page))
636                         return PAGEREF_ACTIVATE;
637                 /*
638                  * All mapped pages start out with page table
639                  * references from the instantiating fault, so we need
640                  * to look twice if a mapped file page is used more
641                  * than once.
642                  *
643                  * Mark it and spare it for another trip around the
644                  * inactive list.  Another page table reference will
645                  * lead to its activation.
646                  *
647                  * Note: the mark is set for activated pages as well
648                  * so that recently deactivated but used pages are
649                  * quickly recovered.
650                  */
651                 SetPageReferenced(page);
652
653                 if (referenced_page || referenced_ptes > 1)
654                         return PAGEREF_ACTIVATE;
655
656                 /*
657                  * Activate file-backed executable pages after first usage.
658                  */
659                 if (vm_flags & VM_EXEC)
660                         return PAGEREF_ACTIVATE;
661
662                 return PAGEREF_KEEP;
663         }
664
665         /* Reclaim if clean, defer dirty pages to writeback */
666         if (referenced_page && !PageSwapBacked(page))
667                 return PAGEREF_RECLAIM_CLEAN;
668
669         return PAGEREF_RECLAIM;
670 }
671
672 /*
673  * shrink_page_list() returns the number of reclaimed pages
674  */
675 static unsigned long shrink_page_list(struct list_head *page_list,
676                                       struct zone *zone,
677                                       struct scan_control *sc,
678                                       enum ttu_flags ttu_flags,
679                                       unsigned long *ret_nr_dirty,
680                                       unsigned long *ret_nr_writeback,
681                                       bool force_reclaim)
682 {
683         LIST_HEAD(ret_pages);
684         LIST_HEAD(free_pages);
685         int pgactivate = 0;
686         unsigned long nr_dirty = 0;
687         unsigned long nr_congested = 0;
688         unsigned long nr_reclaimed = 0;
689         unsigned long nr_writeback = 0;
690
691         cond_resched();
692
693         mem_cgroup_uncharge_start();
694         while (!list_empty(page_list)) {
695                 struct address_space *mapping;
696                 struct page *page;
697                 int may_enter_fs;
698                 enum page_references references = PAGEREF_RECLAIM_CLEAN;
699
700                 cond_resched();
701
702                 page = lru_to_page(page_list);
703                 list_del(&page->lru);
704
705                 if (!trylock_page(page))
706                         goto keep;
707
708                 VM_BUG_ON(PageActive(page));
709                 VM_BUG_ON(page_zone(page) != zone);
710
711                 sc->nr_scanned++;
712
713                 if (unlikely(!page_evictable(page)))
714                         goto cull_mlocked;
715
716                 if (!sc->may_unmap && page_mapped(page))
717                         goto keep_locked;
718
719                 /* Double the slab pressure for mapped and swapcache pages */
720                 if (page_mapped(page) || PageSwapCache(page))
721                         sc->nr_scanned++;
722
723                 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
724                         (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
725
726                 if (PageWriteback(page)) {
727                         /*
728                          * memcg doesn't have any dirty pages throttling so we
729                          * could easily OOM just because too many pages are in
730                          * writeback and there is nothing else to reclaim.
731                          *
732                          * Check __GFP_IO, certainly because a loop driver
733                          * thread might enter reclaim, and deadlock if it waits
734                          * on a page for which it is needed to do the write
735                          * (loop masks off __GFP_IO|__GFP_FS for this reason);
736                          * but more thought would probably show more reasons.
737                          *
738                          * Don't require __GFP_FS, since we're not going into
739                          * the FS, just waiting on its writeback completion.
740                          * Worryingly, ext4 gfs2 and xfs allocate pages with
741                          * grab_cache_page_write_begin(,,AOP_FLAG_NOFS), so
742                          * testing may_enter_fs here is liable to OOM on them.
743                          */
744                         if (global_reclaim(sc) ||
745                             !PageReclaim(page) || !(sc->gfp_mask & __GFP_IO)) {
746                                 /*
747                                  * This is slightly racy - end_page_writeback()
748                                  * might have just cleared PageReclaim, then
749                                  * setting PageReclaim here end up interpreted
750                                  * as PageReadahead - but that does not matter
751                                  * enough to care.  What we do want is for this
752                                  * page to have PageReclaim set next time memcg
753                                  * reclaim reaches the tests above, so it will
754                                  * then wait_on_page_writeback() to avoid OOM;
755                                  * and it's also appropriate in global reclaim.
756                                  */
757                                 SetPageReclaim(page);
758                                 nr_writeback++;
759                                 goto keep_locked;
760                         }
761                         wait_on_page_writeback(page);
762                 }
763
764                 if (!force_reclaim)
765                         references = page_check_references(page, sc);
766
767                 switch (references) {
768                 case PAGEREF_ACTIVATE:
769                         goto activate_locked;
770                 case PAGEREF_KEEP:
771                         goto keep_locked;
772                 case PAGEREF_RECLAIM:
773                 case PAGEREF_RECLAIM_CLEAN:
774                         ; /* try to reclaim the page below */
775                 }
776
777                 /*
778                  * Anonymous process memory has backing store?
779                  * Try to allocate it some swap space here.
780                  */
781                 if (PageAnon(page) && !PageSwapCache(page)) {
782                         if (!(sc->gfp_mask & __GFP_IO))
783                                 goto keep_locked;
784                         if (!add_to_swap(page, page_list))
785                                 goto activate_locked;
786                         may_enter_fs = 1;
787                 }
788
789                 mapping = page_mapping(page);
790
791                 /*
792                  * The page is mapped into the page tables of one or more
793                  * processes. Try to unmap it here.
794                  */
795                 if (page_mapped(page) && mapping) {
796                         switch (try_to_unmap(page, ttu_flags)) {
797                         case SWAP_FAIL:
798                                 goto activate_locked;
799                         case SWAP_AGAIN:
800                                 goto keep_locked;
801                         case SWAP_MLOCK:
802                                 goto cull_mlocked;
803                         case SWAP_SUCCESS:
804                                 ; /* try to free the page below */
805                         }
806                 }
807
808                 if (PageDirty(page)) {
809                         nr_dirty++;
810
811                         /*
812                          * Only kswapd can writeback filesystem pages to
813                          * avoid risk of stack overflow but do not writeback
814                          * unless under significant pressure.
815                          */
816                         if (page_is_file_cache(page) &&
817                                         (!current_is_kswapd() ||
818                                          sc->priority >= DEF_PRIORITY - 2)) {
819                                 /*
820                                  * Immediately reclaim when written back.
821                                  * Similar in principal to deactivate_page()
822                                  * except we already have the page isolated
823                                  * and know it's dirty
824                                  */
825                                 inc_zone_page_state(page, NR_VMSCAN_IMMEDIATE);
826                                 SetPageReclaim(page);
827
828                                 goto keep_locked;
829                         }
830
831                         if (references == PAGEREF_RECLAIM_CLEAN)
832                                 goto keep_locked;
833                         if (!may_enter_fs)
834                                 goto keep_locked;
835                         if (!sc->may_writepage)
836                                 goto keep_locked;
837
838                         /* Page is dirty, try to write it out here */
839                         switch (pageout(page, mapping, sc)) {
840                         case PAGE_KEEP:
841                                 nr_congested++;
842                                 goto keep_locked;
843                         case PAGE_ACTIVATE:
844                                 goto activate_locked;
845                         case PAGE_SUCCESS:
846                                 if (PageWriteback(page))
847                                         goto keep;
848                                 if (PageDirty(page))
849                                         goto keep;
850
851                                 /*
852                                  * A synchronous write - probably a ramdisk.  Go
853                                  * ahead and try to reclaim the page.
854                                  */
855                                 if (!trylock_page(page))
856                                         goto keep;
857                                 if (PageDirty(page) || PageWriteback(page))
858                                         goto keep_locked;
859                                 mapping = page_mapping(page);
860                         case PAGE_CLEAN:
861                                 ; /* try to free the page below */
862                         }
863                 }
864
865                 /*
866                  * If the page has buffers, try to free the buffer mappings
867                  * associated with this page. If we succeed we try to free
868                  * the page as well.
869                  *
870                  * We do this even if the page is PageDirty().
871                  * try_to_release_page() does not perform I/O, but it is
872                  * possible for a page to have PageDirty set, but it is actually
873                  * clean (all its buffers are clean).  This happens if the
874                  * buffers were written out directly, with submit_bh(). ext3
875                  * will do this, as well as the blockdev mapping.
876                  * try_to_release_page() will discover that cleanness and will
877                  * drop the buffers and mark the page clean - it can be freed.
878                  *
879                  * Rarely, pages can have buffers and no ->mapping.  These are
880                  * the pages which were not successfully invalidated in
881                  * truncate_complete_page().  We try to drop those buffers here
882                  * and if that worked, and the page is no longer mapped into
883                  * process address space (page_count == 1) it can be freed.
884                  * Otherwise, leave the page on the LRU so it is swappable.
885                  */
886                 if (page_has_private(page)) {
887                         if (!try_to_release_page(page, sc->gfp_mask))
888                                 goto activate_locked;
889                         if (!mapping && page_count(page) == 1) {
890                                 unlock_page(page);
891                                 if (put_page_testzero(page))
892                                         goto free_it;
893                                 else {
894                                         /*
895                                          * rare race with speculative reference.
896                                          * the speculative reference will free
897                                          * this page shortly, so we may
898                                          * increment nr_reclaimed here (and
899                                          * leave it off the LRU).
900                                          */
901                                         nr_reclaimed++;
902                                         continue;
903                                 }
904                         }
905                 }
906
907                 if (!mapping || !__remove_mapping(mapping, page))
908                         goto keep_locked;
909
910                 /*
911                  * At this point, we have no other references and there is
912                  * no way to pick any more up (removed from LRU, removed
913                  * from pagecache). Can use non-atomic bitops now (and
914                  * we obviously don't have to worry about waking up a process
915                  * waiting on the page lock, because there are no references.
916                  */
917                 __clear_page_locked(page);
918 free_it:
919                 nr_reclaimed++;
920
921                 /*
922                  * Is there need to periodically free_page_list? It would
923                  * appear not as the counts should be low
924                  */
925                 list_add(&page->lru, &free_pages);
926                 continue;
927
928 cull_mlocked:
929                 if (PageSwapCache(page))
930                         try_to_free_swap(page);
931                 unlock_page(page);
932                 putback_lru_page(page);
933                 continue;
934
935 activate_locked:
936                 /* Not a candidate for swapping, so reclaim swap space. */
937                 if (PageSwapCache(page) && vm_swap_full())
938                         try_to_free_swap(page);
939                 VM_BUG_ON(PageActive(page));
940                 SetPageActive(page);
941                 pgactivate++;
942 keep_locked:
943                 unlock_page(page);
944 keep:
945                 list_add(&page->lru, &ret_pages);
946                 VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
947         }
948
949         /*
950          * Tag a zone as congested if all the dirty pages encountered were
951          * backed by a congested BDI. In this case, reclaimers should just
952          * back off and wait for congestion to clear because further reclaim
953          * will encounter the same problem
954          */
955         if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
956                 zone_set_flag(zone, ZONE_CONGESTED);
957
958         free_hot_cold_page_list(&free_pages, 1);
959
960         list_splice(&ret_pages, page_list);
961         count_vm_events(PGACTIVATE, pgactivate);
962         mem_cgroup_uncharge_end();
963         *ret_nr_dirty += nr_dirty;
964         *ret_nr_writeback += nr_writeback;
965         return nr_reclaimed;
966 }
967
968 unsigned long reclaim_clean_pages_from_list(struct zone *zone,
969                                             struct list_head *page_list)
970 {
971         struct scan_control sc = {
972                 .gfp_mask = GFP_KERNEL,
973                 .priority = DEF_PRIORITY,
974                 .may_unmap = 1,
975         };
976         unsigned long ret, dummy1, dummy2;
977         struct page *page, *next;
978         LIST_HEAD(clean_pages);
979
980         list_for_each_entry_safe(page, next, page_list, lru) {
981                 if (page_is_file_cache(page) && !PageDirty(page)) {
982                         ClearPageActive(page);
983                         list_move(&page->lru, &clean_pages);
984                 }
985         }
986
987         ret = shrink_page_list(&clean_pages, zone, &sc,
988                                 TTU_UNMAP|TTU_IGNORE_ACCESS,
989                                 &dummy1, &dummy2, true);
990         list_splice(&clean_pages, page_list);
991         __mod_zone_page_state(zone, NR_ISOLATED_FILE, -ret);
992         return ret;
993 }
994
995 /*
996  * Attempt to remove the specified page from its LRU.  Only take this page
997  * if it is of the appropriate PageActive status.  Pages which are being
998  * freed elsewhere are also ignored.
999  *
1000  * page:        page to consider
1001  * mode:        one of the LRU isolation modes defined above
1002  *
1003  * returns 0 on success, -ve errno on failure.
1004  */
1005 int __isolate_lru_page(struct page *page, isolate_mode_t mode)
1006 {
1007         int ret = -EINVAL;
1008
1009         /* Only take pages on the LRU. */
1010         if (!PageLRU(page))
1011                 return ret;
1012
1013         /* Compaction should not handle unevictable pages but CMA can do so */
1014         if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
1015                 return ret;
1016
1017         ret = -EBUSY;
1018
1019         /*
1020          * To minimise LRU disruption, the caller can indicate that it only
1021          * wants to isolate pages it will be able to operate on without
1022          * blocking - clean pages for the most part.
1023          *
1024          * ISOLATE_CLEAN means that only clean pages should be isolated. This
1025          * is used by reclaim when it is cannot write to backing storage
1026          *
1027          * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1028          * that it is possible to migrate without blocking
1029          */
1030         if (mode & (ISOLATE_CLEAN|ISOLATE_ASYNC_MIGRATE)) {
1031                 /* All the caller can do on PageWriteback is block */
1032                 if (PageWriteback(page))
1033                         return ret;
1034
1035                 if (PageDirty(page)) {
1036                         struct address_space *mapping;
1037
1038                         /* ISOLATE_CLEAN means only clean pages */
1039                         if (mode & ISOLATE_CLEAN)
1040                                 return ret;
1041
1042                         /*
1043                          * Only pages without mappings or that have a
1044                          * ->migratepage callback are possible to migrate
1045                          * without blocking
1046                          */
1047                         mapping = page_mapping(page);
1048                         if (mapping && !mapping->a_ops->migratepage)
1049                                 return ret;
1050                 }
1051         }
1052
1053         if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1054                 return ret;
1055
1056         if (likely(get_page_unless_zero(page))) {
1057                 /*
1058                  * Be careful not to clear PageLRU until after we're
1059                  * sure the page is not being freed elsewhere -- the
1060                  * page release code relies on it.
1061                  */
1062                 ClearPageLRU(page);
1063                 ret = 0;
1064         }
1065
1066         return ret;
1067 }
1068
1069 /*
1070  * zone->lru_lock is heavily contended.  Some of the functions that
1071  * shrink the lists perform better by taking out a batch of pages
1072  * and working on them outside the LRU lock.
1073  *
1074  * For pagecache intensive workloads, this function is the hottest
1075  * spot in the kernel (apart from copy_*_user functions).
1076  *
1077  * Appropriate locks must be held before calling this function.
1078  *
1079  * @nr_to_scan: The number of pages to look through on the list.
1080  * @lruvec:     The LRU vector to pull pages from.
1081  * @dst:        The temp list to put pages on to.
1082  * @nr_scanned: The number of pages that were scanned.
1083  * @sc:         The scan_control struct for this reclaim session
1084  * @mode:       One of the LRU isolation modes
1085  * @lru:        LRU list id for isolating
1086  *
1087  * returns how many pages were moved onto *@dst.
1088  */
1089 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1090                 struct lruvec *lruvec, struct list_head *dst,
1091                 unsigned long *nr_scanned, struct scan_control *sc,
1092                 isolate_mode_t mode, enum lru_list lru)
1093 {
1094         struct list_head *src = &lruvec->lists[lru];
1095         unsigned long nr_taken = 0;
1096         unsigned long scan;
1097
1098         for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
1099                 struct page *page;
1100                 int nr_pages;
1101
1102                 page = lru_to_page(src);
1103                 prefetchw_prev_lru_page(page, src, flags);
1104
1105                 VM_BUG_ON(!PageLRU(page));
1106
1107                 switch (__isolate_lru_page(page, mode)) {
1108                 case 0:
1109                         nr_pages = hpage_nr_pages(page);
1110                         mem_cgroup_update_lru_size(lruvec, lru, -nr_pages);
1111                         list_move(&page->lru, dst);
1112                         nr_taken += nr_pages;
1113                         break;
1114
1115                 case -EBUSY:
1116                         /* else it is being freed elsewhere */
1117                         list_move(&page->lru, src);
1118                         continue;
1119
1120                 default:
1121                         BUG();
1122                 }
1123         }
1124
1125         *nr_scanned = scan;
1126         trace_mm_vmscan_lru_isolate(sc->order, nr_to_scan, scan,
1127                                     nr_taken, mode, is_file_lru(lru));
1128         return nr_taken;
1129 }
1130
1131 /**
1132  * isolate_lru_page - tries to isolate a page from its LRU list
1133  * @page: page to isolate from its LRU list
1134  *
1135  * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1136  * vmstat statistic corresponding to whatever LRU list the page was on.
1137  *
1138  * Returns 0 if the page was removed from an LRU list.
1139  * Returns -EBUSY if the page was not on an LRU list.
1140  *
1141  * The returned page will have PageLRU() cleared.  If it was found on
1142  * the active list, it will have PageActive set.  If it was found on
1143  * the unevictable list, it will have the PageUnevictable bit set. That flag
1144  * may need to be cleared by the caller before letting the page go.
1145  *
1146  * The vmstat statistic corresponding to the list on which the page was
1147  * found will be decremented.
1148  *
1149  * Restrictions:
1150  * (1) Must be called with an elevated refcount on the page. This is a
1151  *     fundamentnal difference from isolate_lru_pages (which is called
1152  *     without a stable reference).
1153  * (2) the lru_lock must not be held.
1154  * (3) interrupts must be enabled.
1155  */
1156 int isolate_lru_page(struct page *page)
1157 {
1158         int ret = -EBUSY;
1159
1160         VM_BUG_ON(!page_count(page));
1161
1162         if (PageLRU(page)) {
1163                 struct zone *zone = page_zone(page);
1164                 struct lruvec *lruvec;
1165
1166                 spin_lock_irq(&zone->lru_lock);
1167                 lruvec = mem_cgroup_page_lruvec(page, zone);
1168                 if (PageLRU(page)) {
1169                         int lru = page_lru(page);
1170                         get_page(page);
1171                         ClearPageLRU(page);
1172                         del_page_from_lru_list(page, lruvec, lru);
1173                         ret = 0;
1174                 }
1175                 spin_unlock_irq(&zone->lru_lock);
1176         }
1177         return ret;
1178 }
1179
1180 /*
1181  * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1182  * then get resheduled. When there are massive number of tasks doing page
1183  * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1184  * the LRU list will go small and be scanned faster than necessary, leading to
1185  * unnecessary swapping, thrashing and OOM.
1186  */
1187 static int too_many_isolated(struct zone *zone, int file,
1188                 struct scan_control *sc)
1189 {
1190         unsigned long inactive, isolated;
1191
1192         if (current_is_kswapd())
1193                 return 0;
1194
1195         if (!global_reclaim(sc))
1196                 return 0;
1197
1198         if (file) {
1199                 inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1200                 isolated = zone_page_state(zone, NR_ISOLATED_FILE);
1201         } else {
1202                 inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1203                 isolated = zone_page_state(zone, NR_ISOLATED_ANON);
1204         }
1205
1206         /*
1207          * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1208          * won't get blocked by normal direct-reclaimers, forming a circular
1209          * deadlock.
1210          */
1211         if ((sc->gfp_mask & GFP_IOFS) == GFP_IOFS)
1212                 inactive >>= 3;
1213
1214         return isolated > inactive;
1215 }
1216
1217 static noinline_for_stack void
1218 putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
1219 {
1220         struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1221         struct zone *zone = lruvec_zone(lruvec);
1222         LIST_HEAD(pages_to_free);
1223
1224         /*
1225          * Put back any unfreeable pages.
1226          */
1227         while (!list_empty(page_list)) {
1228                 struct page *page = lru_to_page(page_list);
1229                 int lru;
1230
1231                 VM_BUG_ON(PageLRU(page));
1232                 list_del(&page->lru);
1233                 if (unlikely(!page_evictable(page))) {
1234                         spin_unlock_irq(&zone->lru_lock);
1235                         putback_lru_page(page);
1236                         spin_lock_irq(&zone->lru_lock);
1237                         continue;
1238                 }
1239
1240                 lruvec = mem_cgroup_page_lruvec(page, zone);
1241
1242                 SetPageLRU(page);
1243                 lru = page_lru(page);
1244                 add_page_to_lru_list(page, lruvec, lru);
1245
1246                 if (is_active_lru(lru)) {
1247                         int file = is_file_lru(lru);
1248                         int numpages = hpage_nr_pages(page);
1249                         reclaim_stat->recent_rotated[file] += numpages;
1250                 }
1251                 if (put_page_testzero(page)) {
1252                         __ClearPageLRU(page);
1253                         __ClearPageActive(page);
1254                         del_page_from_lru_list(page, lruvec, lru);
1255
1256                         if (unlikely(PageCompound(page))) {
1257                                 spin_unlock_irq(&zone->lru_lock);
1258                                 (*get_compound_page_dtor(page))(page);
1259                                 spin_lock_irq(&zone->lru_lock);
1260                         } else
1261                                 list_add(&page->lru, &pages_to_free);
1262                 }
1263         }
1264
1265         /*
1266          * To save our caller's stack, now use input list for pages to free.
1267          */
1268         list_splice(&pages_to_free, page_list);
1269 }
1270
1271 /*
1272  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
1273  * of reclaimed pages
1274  */
1275 static noinline_for_stack unsigned long
1276 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
1277                      struct scan_control *sc, enum lru_list lru)
1278 {
1279         LIST_HEAD(page_list);
1280         unsigned long nr_scanned;
1281         unsigned long nr_reclaimed = 0;
1282         unsigned long nr_taken;
1283         unsigned long nr_dirty = 0;
1284         unsigned long nr_writeback = 0;
1285         isolate_mode_t isolate_mode = 0;
1286         int file = is_file_lru(lru);
1287         struct zone *zone = lruvec_zone(lruvec);
1288         struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1289
1290         while (unlikely(too_many_isolated(zone, file, sc))) {
1291                 congestion_wait(BLK_RW_ASYNC, HZ/10);
1292
1293                 /* We are about to die and free our memory. Return now. */
1294                 if (fatal_signal_pending(current))
1295                         return SWAP_CLUSTER_MAX;
1296         }
1297
1298         lru_add_drain();
1299
1300         if (!sc->may_unmap)
1301                 isolate_mode |= ISOLATE_UNMAPPED;
1302         if (!sc->may_writepage)
1303                 isolate_mode |= ISOLATE_CLEAN;
1304
1305         spin_lock_irq(&zone->lru_lock);
1306
1307         nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1308                                      &nr_scanned, sc, isolate_mode, lru);
1309
1310         __mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
1311         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1312
1313         if (global_reclaim(sc)) {
1314                 zone->pages_scanned += nr_scanned;
1315                 if (current_is_kswapd())
1316                         __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scanned);
1317                 else
1318                         __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scanned);
1319         }
1320         spin_unlock_irq(&zone->lru_lock);
1321
1322         if (nr_taken == 0)
1323                 return 0;
1324
1325         nr_reclaimed = shrink_page_list(&page_list, zone, sc, TTU_UNMAP,
1326                                         &nr_dirty, &nr_writeback, false);
1327
1328         spin_lock_irq(&zone->lru_lock);
1329
1330         reclaim_stat->recent_scanned[file] += nr_taken;
1331
1332         if (global_reclaim(sc)) {
1333                 if (current_is_kswapd())
1334                         __count_zone_vm_events(PGSTEAL_KSWAPD, zone,
1335                                                nr_reclaimed);
1336                 else
1337                         __count_zone_vm_events(PGSTEAL_DIRECT, zone,
1338                                                nr_reclaimed);
1339         }
1340
1341         putback_inactive_pages(lruvec, &page_list);
1342
1343         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
1344
1345         spin_unlock_irq(&zone->lru_lock);
1346
1347         free_hot_cold_page_list(&page_list, 1);
1348
1349         /*
1350          * If reclaim is isolating dirty pages under writeback, it implies
1351          * that the long-lived page allocation rate is exceeding the page
1352          * laundering rate. Either the global limits are not being effective
1353          * at throttling processes due to the page distribution throughout
1354          * zones or there is heavy usage of a slow backing device. The
1355          * only option is to throttle from reclaim context which is not ideal
1356          * as there is no guarantee the dirtying process is throttled in the
1357          * same way balance_dirty_pages() manages.
1358          *
1359          * This scales the number of dirty pages that must be under writeback
1360          * before throttling depending on priority. It is a simple backoff
1361          * function that has the most effect in the range DEF_PRIORITY to
1362          * DEF_PRIORITY-2 which is the priority reclaim is considered to be
1363          * in trouble and reclaim is considered to be in trouble.
1364          *
1365          * DEF_PRIORITY   100% isolated pages must be PageWriteback to throttle
1366          * DEF_PRIORITY-1  50% must be PageWriteback
1367          * DEF_PRIORITY-2  25% must be PageWriteback, kswapd in trouble
1368          * ...
1369          * DEF_PRIORITY-6 For SWAP_CLUSTER_MAX isolated pages, throttle if any
1370          *                     isolated page is PageWriteback
1371          */
1372         if (nr_writeback && nr_writeback >=
1373                         (nr_taken >> (DEF_PRIORITY - sc->priority)))
1374                 wait_iff_congested(zone, BLK_RW_ASYNC, HZ/10);
1375
1376         trace_mm_vmscan_lru_shrink_inactive(zone->zone_pgdat->node_id,
1377                 zone_idx(zone),
1378                 nr_scanned, nr_reclaimed,
1379                 sc->priority,
1380                 trace_shrink_flags(file));
1381         return nr_reclaimed;
1382 }
1383
1384 /*
1385  * This moves pages from the active list to the inactive list.
1386  *
1387  * We move them the other way if the page is referenced by one or more
1388  * processes, from rmap.
1389  *
1390  * If the pages are mostly unmapped, the processing is fast and it is
1391  * appropriate to hold zone->lru_lock across the whole operation.  But if
1392  * the pages are mapped, the processing is slow (page_referenced()) so we
1393  * should drop zone->lru_lock around each page.  It's impossible to balance
1394  * this, so instead we remove the pages from the LRU while processing them.
1395  * It is safe to rely on PG_active against the non-LRU pages in here because
1396  * nobody will play with that bit on a non-LRU page.
1397  *
1398  * The downside is that we have to touch page->_count against each page.
1399  * But we had to alter page->flags anyway.
1400  */
1401
1402 static void move_active_pages_to_lru(struct lruvec *lruvec,
1403                                      struct list_head *list,
1404                                      struct list_head *pages_to_free,
1405                                      enum lru_list lru)
1406 {
1407         struct zone *zone = lruvec_zone(lruvec);
1408         unsigned long pgmoved = 0;
1409         struct page *page;
1410         int nr_pages;
1411
1412         while (!list_empty(list)) {
1413                 page = lru_to_page(list);
1414                 lruvec = mem_cgroup_page_lruvec(page, zone);
1415
1416                 VM_BUG_ON(PageLRU(page));
1417                 SetPageLRU(page);
1418
1419                 nr_pages = hpage_nr_pages(page);
1420                 mem_cgroup_update_lru_size(lruvec, lru, nr_pages);
1421                 list_move(&page->lru, &lruvec->lists[lru]);
1422                 pgmoved += nr_pages;
1423
1424                 if (put_page_testzero(page)) {
1425                         __ClearPageLRU(page);
1426                         __ClearPageActive(page);
1427                         del_page_from_lru_list(page, lruvec, lru);
1428
1429                         if (unlikely(PageCompound(page))) {
1430                                 spin_unlock_irq(&zone->lru_lock);
1431                                 (*get_compound_page_dtor(page))(page);
1432                                 spin_lock_irq(&zone->lru_lock);
1433                         } else
1434                                 list_add(&page->lru, pages_to_free);
1435                 }
1436         }
1437         __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
1438         if (!is_active_lru(lru))
1439                 __count_vm_events(PGDEACTIVATE, pgmoved);
1440 }
1441
1442 static void shrink_active_list(unsigned long nr_to_scan,
1443                                struct lruvec *lruvec,
1444                                struct scan_control *sc,
1445                                enum lru_list lru)
1446 {
1447         unsigned long nr_taken;
1448         unsigned long nr_scanned;
1449         unsigned long vm_flags;
1450         LIST_HEAD(l_hold);      /* The pages which were snipped off */
1451         LIST_HEAD(l_active);
1452         LIST_HEAD(l_inactive);
1453         struct page *page;
1454         struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1455         unsigned long nr_rotated = 0;
1456         isolate_mode_t isolate_mode = 0;
1457         int file = is_file_lru(lru);
1458         struct zone *zone = lruvec_zone(lruvec);
1459
1460         lru_add_drain();
1461
1462         if (!sc->may_unmap)
1463                 isolate_mode |= ISOLATE_UNMAPPED;
1464         if (!sc->may_writepage)
1465                 isolate_mode |= ISOLATE_CLEAN;
1466
1467         spin_lock_irq(&zone->lru_lock);
1468
1469         nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
1470                                      &nr_scanned, sc, isolate_mode, lru);
1471         if (global_reclaim(sc))
1472                 zone->pages_scanned += nr_scanned;
1473
1474         reclaim_stat->recent_scanned[file] += nr_taken;
1475
1476         __count_zone_vm_events(PGREFILL, zone, nr_scanned);
1477         __mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
1478         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1479         spin_unlock_irq(&zone->lru_lock);
1480
1481         while (!list_empty(&l_hold)) {
1482                 cond_resched();
1483                 page = lru_to_page(&l_hold);
1484                 list_del(&page->lru);
1485
1486                 if (unlikely(!page_evictable(page))) {
1487                         putback_lru_page(page);
1488                         continue;
1489                 }
1490
1491                 if (unlikely(buffer_heads_over_limit)) {
1492                         if (page_has_private(page) && trylock_page(page)) {
1493                                 if (page_has_private(page))
1494                                         try_to_release_page(page, 0);
1495                                 unlock_page(page);
1496                         }
1497                 }
1498
1499                 if (page_referenced(page, 0, sc->target_mem_cgroup,
1500                                     &vm_flags)) {
1501                         nr_rotated += hpage_nr_pages(page);
1502                         /*
1503                          * Identify referenced, file-backed active pages and
1504                          * give them one more trip around the active list. So
1505                          * that executable code get better chances to stay in
1506                          * memory under moderate memory pressure.  Anon pages
1507                          * are not likely to be evicted by use-once streaming
1508                          * IO, plus JVM can create lots of anon VM_EXEC pages,
1509                          * so we ignore them here.
1510                          */
1511                         if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
1512                                 list_add(&page->lru, &l_active);
1513                                 continue;
1514                         }
1515                 }
1516
1517                 ClearPageActive(page);  /* we are de-activating */
1518                 list_add(&page->lru, &l_inactive);
1519         }
1520
1521         /*
1522          * Move pages back to the lru list.
1523          */
1524         spin_lock_irq(&zone->lru_lock);
1525         /*
1526          * Count referenced pages from currently used mappings as rotated,
1527          * even though only some of them are actually re-activated.  This
1528          * helps balance scan pressure between file and anonymous pages in
1529          * get_scan_ratio.
1530          */
1531         reclaim_stat->recent_rotated[file] += nr_rotated;
1532
1533         move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
1534         move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
1535         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
1536         spin_unlock_irq(&zone->lru_lock);
1537
1538         free_hot_cold_page_list(&l_hold, 1);
1539 }
1540
1541 #ifdef CONFIG_SWAP
1542 static int inactive_anon_is_low_global(struct zone *zone)
1543 {
1544         unsigned long active, inactive;
1545
1546         active = zone_page_state(zone, NR_ACTIVE_ANON);
1547         inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1548
1549         if (inactive * zone->inactive_ratio < active)
1550                 return 1;
1551
1552         return 0;
1553 }
1554
1555 /**
1556  * inactive_anon_is_low - check if anonymous pages need to be deactivated
1557  * @lruvec: LRU vector to check
1558  *
1559  * Returns true if the zone does not have enough inactive anon pages,
1560  * meaning some active anon pages need to be deactivated.
1561  */
1562 static int inactive_anon_is_low(struct lruvec *lruvec)
1563 {
1564         /*
1565          * If we don't have swap space, anonymous page deactivation
1566          * is pointless.
1567          */
1568         if (!total_swap_pages)
1569                 return 0;
1570
1571         if (!mem_cgroup_disabled())
1572                 return mem_cgroup_inactive_anon_is_low(lruvec);
1573
1574         return inactive_anon_is_low_global(lruvec_zone(lruvec));
1575 }
1576 #else
1577 static inline int inactive_anon_is_low(struct lruvec *lruvec)
1578 {
1579         return 0;
1580 }
1581 #endif
1582
1583 /**
1584  * inactive_file_is_low - check if file pages need to be deactivated
1585  * @lruvec: LRU vector to check
1586  *
1587  * When the system is doing streaming IO, memory pressure here
1588  * ensures that active file pages get deactivated, until more
1589  * than half of the file pages are on the inactive list.
1590  *
1591  * Once we get to that situation, protect the system's working
1592  * set from being evicted by disabling active file page aging.
1593  *
1594  * This uses a different ratio than the anonymous pages, because
1595  * the page cache uses a use-once replacement algorithm.
1596  */
1597 static int inactive_file_is_low(struct lruvec *lruvec)
1598 {
1599         unsigned long inactive;
1600         unsigned long active;
1601
1602         inactive = get_lru_size(lruvec, LRU_INACTIVE_FILE);
1603         active = get_lru_size(lruvec, LRU_ACTIVE_FILE);
1604
1605         return active > inactive;
1606 }
1607
1608 static int inactive_list_is_low(struct lruvec *lruvec, enum lru_list lru)
1609 {
1610         if (is_file_lru(lru))
1611                 return inactive_file_is_low(lruvec);
1612         else
1613                 return inactive_anon_is_low(lruvec);
1614 }
1615
1616 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
1617                                  struct lruvec *lruvec, struct scan_control *sc)
1618 {
1619         if (is_active_lru(lru)) {
1620                 if (inactive_list_is_low(lruvec, lru))
1621                         shrink_active_list(nr_to_scan, lruvec, sc, lru);
1622                 return 0;
1623         }
1624
1625         return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
1626 }
1627
1628 static int vmscan_swappiness(struct scan_control *sc)
1629 {
1630         if (global_reclaim(sc))
1631                 return vm_swappiness;
1632         return mem_cgroup_swappiness(sc->target_mem_cgroup);
1633 }
1634
1635 enum scan_balance {
1636         SCAN_EQUAL,
1637         SCAN_FRACT,
1638         SCAN_ANON,
1639         SCAN_FILE,
1640 };
1641
1642 /*
1643  * Determine how aggressively the anon and file LRU lists should be
1644  * scanned.  The relative value of each set of LRU lists is determined
1645  * by looking at the fraction of the pages scanned we did rotate back
1646  * onto the active list instead of evict.
1647  *
1648  * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
1649  * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
1650  */
1651 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
1652                            unsigned long *nr)
1653 {
1654         struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1655         u64 fraction[2];
1656         u64 denominator = 0;    /* gcc */
1657         struct zone *zone = lruvec_zone(lruvec);
1658         unsigned long anon_prio, file_prio;
1659         enum scan_balance scan_balance;
1660         unsigned long anon, file, free;
1661         bool force_scan = false;
1662         unsigned long ap, fp;
1663         enum lru_list lru;
1664
1665         /*
1666          * If the zone or memcg is small, nr[l] can be 0.  This
1667          * results in no scanning on this priority and a potential
1668          * priority drop.  Global direct reclaim can go to the next
1669          * zone and tends to have no problems. Global kswapd is for
1670          * zone balancing and it needs to scan a minimum amount. When
1671          * reclaiming for a memcg, a priority drop can cause high
1672          * latencies, so it's better to scan a minimum amount there as
1673          * well.
1674          */
1675         if (current_is_kswapd() && zone->all_unreclaimable)
1676                 force_scan = true;
1677         if (!global_reclaim(sc))
1678                 force_scan = true;
1679
1680         /* If we have no swap space, do not bother scanning anon pages. */
1681         if (!sc->may_swap || (get_nr_swap_pages() <= 0)) {
1682                 scan_balance = SCAN_FILE;
1683                 goto out;
1684         }
1685
1686         /*
1687          * Global reclaim will swap to prevent OOM even with no
1688          * swappiness, but memcg users want to use this knob to
1689          * disable swapping for individual groups completely when
1690          * using the memory controller's swap limit feature would be
1691          * too expensive.
1692          */
1693         if (!global_reclaim(sc) && !vmscan_swappiness(sc)) {
1694                 scan_balance = SCAN_FILE;
1695                 goto out;
1696         }
1697
1698         /*
1699          * Do not apply any pressure balancing cleverness when the
1700          * system is close to OOM, scan both anon and file equally
1701          * (unless the swappiness setting disagrees with swapping).
1702          */
1703         if (!sc->priority && vmscan_swappiness(sc)) {
1704                 scan_balance = SCAN_EQUAL;
1705                 goto out;
1706         }
1707
1708         anon  = get_lru_size(lruvec, LRU_ACTIVE_ANON) +
1709                 get_lru_size(lruvec, LRU_INACTIVE_ANON);
1710         file  = get_lru_size(lruvec, LRU_ACTIVE_FILE) +
1711                 get_lru_size(lruvec, LRU_INACTIVE_FILE);
1712
1713         /*
1714          * If it's foreseeable that reclaiming the file cache won't be
1715          * enough to get the zone back into a desirable shape, we have
1716          * to swap.  Better start now and leave the - probably heavily
1717          * thrashing - remaining file pages alone.
1718          */
1719         if (global_reclaim(sc)) {
1720                 free = zone_page_state(zone, NR_FREE_PAGES);
1721                 if (unlikely(file + free <= high_wmark_pages(zone))) {
1722                         scan_balance = SCAN_ANON;
1723                         goto out;
1724                 }
1725         }
1726
1727         /*
1728          * There is enough inactive page cache, do not reclaim
1729          * anything from the anonymous working set right now.
1730          */
1731         if (!inactive_file_is_low(lruvec)) {
1732                 scan_balance = SCAN_FILE;
1733                 goto out;
1734         }
1735
1736         scan_balance = SCAN_FRACT;
1737
1738         /*
1739          * With swappiness at 100, anonymous and file have the same priority.
1740          * This scanning priority is essentially the inverse of IO cost.
1741          */
1742         anon_prio = vmscan_swappiness(sc);
1743         file_prio = 200 - anon_prio;
1744
1745         /*
1746          * OK, so we have swap space and a fair amount of page cache
1747          * pages.  We use the recently rotated / recently scanned
1748          * ratios to determine how valuable each cache is.
1749          *
1750          * Because workloads change over time (and to avoid overflow)
1751          * we keep these statistics as a floating average, which ends
1752          * up weighing recent references more than old ones.
1753          *
1754          * anon in [0], file in [1]
1755          */
1756         spin_lock_irq(&zone->lru_lock);
1757         if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
1758                 reclaim_stat->recent_scanned[0] /= 2;
1759                 reclaim_stat->recent_rotated[0] /= 2;
1760         }
1761
1762         if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
1763                 reclaim_stat->recent_scanned[1] /= 2;
1764                 reclaim_stat->recent_rotated[1] /= 2;
1765         }
1766
1767         /*
1768          * The amount of pressure on anon vs file pages is inversely
1769          * proportional to the fraction of recently scanned pages on
1770          * each list that were recently referenced and in active use.
1771          */
1772         ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
1773         ap /= reclaim_stat->recent_rotated[0] + 1;
1774
1775         fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
1776         fp /= reclaim_stat->recent_rotated[1] + 1;
1777         spin_unlock_irq(&zone->lru_lock);
1778
1779         fraction[0] = ap;
1780         fraction[1] = fp;
1781         denominator = ap + fp + 1;
1782 out:
1783         for_each_evictable_lru(lru) {
1784                 int file = is_file_lru(lru);
1785                 unsigned long size;
1786                 unsigned long scan;
1787
1788                 size = get_lru_size(lruvec, lru);
1789                 scan = size >> sc->priority;
1790
1791                 if (!scan && force_scan)
1792                         scan = min(size, SWAP_CLUSTER_MAX);
1793
1794                 switch (scan_balance) {
1795                 case SCAN_EQUAL:
1796                         /* Scan lists relative to size */
1797                         break;
1798                 case SCAN_FRACT:
1799                         /*
1800                          * Scan types proportional to swappiness and
1801                          * their relative recent reclaim efficiency.
1802                          */
1803                         scan = div64_u64(scan * fraction[file], denominator);
1804                         break;
1805                 case SCAN_FILE:
1806                 case SCAN_ANON:
1807                         /* Scan one type exclusively */
1808                         if ((scan_balance == SCAN_FILE) != file)
1809                                 scan = 0;
1810                         break;
1811                 default:
1812                         /* Look ma, no brain */
1813                         BUG();
1814                 }
1815                 nr[lru] = scan;
1816         }
1817 }
1818
1819 /*
1820  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
1821  */
1822 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
1823 {
1824         unsigned long nr[NR_LRU_LISTS];
1825         unsigned long targets[NR_LRU_LISTS];
1826         unsigned long nr_to_scan;
1827         enum lru_list lru;
1828         unsigned long nr_reclaimed = 0;
1829         unsigned long nr_to_reclaim = sc->nr_to_reclaim;
1830         struct blk_plug plug;
1831         bool scan_adjusted = false;
1832
1833         get_scan_count(lruvec, sc, nr);
1834
1835         /* Record the original scan target for proportional adjustments later */
1836         memcpy(targets, nr, sizeof(nr));
1837
1838         blk_start_plug(&plug);
1839         while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
1840                                         nr[LRU_INACTIVE_FILE]) {
1841                 unsigned long nr_anon, nr_file, percentage;
1842                 unsigned long nr_scanned;
1843
1844                 for_each_evictable_lru(lru) {
1845                         if (nr[lru]) {
1846                                 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
1847                                 nr[lru] -= nr_to_scan;
1848
1849                                 nr_reclaimed += shrink_list(lru, nr_to_scan,
1850                                                             lruvec, sc);
1851                         }
1852                 }
1853
1854                 if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
1855                         continue;
1856
1857                 /*
1858                  * For global direct reclaim, reclaim only the number of pages
1859                  * requested. Less care is taken to scan proportionally as it
1860                  * is more important to minimise direct reclaim stall latency
1861                  * than it is to properly age the LRU lists.
1862                  */
1863                 if (global_reclaim(sc) && !current_is_kswapd())
1864                         break;
1865
1866                 /*
1867                  * For kswapd and memcg, reclaim at least the number of pages
1868                  * requested. Ensure that the anon and file LRUs shrink
1869                  * proportionally what was requested by get_scan_count(). We
1870                  * stop reclaiming one LRU and reduce the amount scanning
1871                  * proportional to the original scan target.
1872                  */
1873                 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
1874                 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
1875
1876                 if (nr_file > nr_anon) {
1877                         unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
1878                                                 targets[LRU_ACTIVE_ANON] + 1;
1879                         lru = LRU_BASE;
1880                         percentage = nr_anon * 100 / scan_target;
1881                 } else {
1882                         unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
1883                                                 targets[LRU_ACTIVE_FILE] + 1;
1884                         lru = LRU_FILE;
1885                         percentage = nr_file * 100 / scan_target;
1886                 }
1887
1888                 /* Stop scanning the smaller of the LRU */
1889                 nr[lru] = 0;
1890                 nr[lru + LRU_ACTIVE] = 0;
1891
1892                 /*
1893                  * Recalculate the other LRU scan count based on its original
1894                  * scan target and the percentage scanning already complete
1895                  */
1896                 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
1897                 nr_scanned = targets[lru] - nr[lru];
1898                 nr[lru] = targets[lru] * (100 - percentage) / 100;
1899                 nr[lru] -= min(nr[lru], nr_scanned);
1900
1901                 lru += LRU_ACTIVE;
1902                 nr_scanned = targets[lru] - nr[lru];
1903                 nr[lru] = targets[lru] * (100 - percentage) / 100;
1904                 nr[lru] -= min(nr[lru], nr_scanned);
1905
1906                 scan_adjusted = true;
1907         }
1908         blk_finish_plug(&plug);
1909         sc->nr_reclaimed += nr_reclaimed;
1910
1911         /*
1912          * Even if we did not try to evict anon pages at all, we want to
1913          * rebalance the anon lru active/inactive ratio.
1914          */
1915         if (inactive_anon_is_low(lruvec))
1916                 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
1917                                    sc, LRU_ACTIVE_ANON);
1918
1919         throttle_vm_writeout(sc->gfp_mask);
1920 }
1921
1922 /* Use reclaim/compaction for costly allocs or under memory pressure */
1923 static bool in_reclaim_compaction(struct scan_control *sc)
1924 {
1925         if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
1926                         (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
1927                          sc->priority < DEF_PRIORITY - 2))
1928                 return true;
1929
1930         return false;
1931 }
1932
1933 /*
1934  * Reclaim/compaction is used for high-order allocation requests. It reclaims
1935  * order-0 pages before compacting the zone. should_continue_reclaim() returns
1936  * true if more pages should be reclaimed such that when the page allocator
1937  * calls try_to_compact_zone() that it will have enough free pages to succeed.
1938  * It will give up earlier than that if there is difficulty reclaiming pages.
1939  */
1940 static inline bool should_continue_reclaim(struct zone *zone,
1941                                         unsigned long nr_reclaimed,
1942                                         unsigned long nr_scanned,
1943                                         struct scan_control *sc)
1944 {
1945         unsigned long pages_for_compaction;
1946         unsigned long inactive_lru_pages;
1947
1948         /* If not in reclaim/compaction mode, stop */
1949         if (!in_reclaim_compaction(sc))
1950                 return false;
1951
1952         /* Consider stopping depending on scan and reclaim activity */
1953         if (sc->gfp_mask & __GFP_REPEAT) {
1954                 /*
1955                  * For __GFP_REPEAT allocations, stop reclaiming if the
1956                  * full LRU list has been scanned and we are still failing
1957                  * to reclaim pages. This full LRU scan is potentially
1958                  * expensive but a __GFP_REPEAT caller really wants to succeed
1959                  */
1960                 if (!nr_reclaimed && !nr_scanned)
1961                         return false;
1962         } else {
1963                 /*
1964                  * For non-__GFP_REPEAT allocations which can presumably
1965                  * fail without consequence, stop if we failed to reclaim
1966                  * any pages from the last SWAP_CLUSTER_MAX number of
1967                  * pages that were scanned. This will return to the
1968                  * caller faster at the risk reclaim/compaction and
1969                  * the resulting allocation attempt fails
1970                  */
1971                 if (!nr_reclaimed)
1972                         return false;
1973         }
1974
1975         /*
1976          * If we have not reclaimed enough pages for compaction and the
1977          * inactive lists are large enough, continue reclaiming
1978          */
1979         pages_for_compaction = (2UL << sc->order);
1980         inactive_lru_pages = zone_page_state(zone, NR_INACTIVE_FILE);
1981         if (get_nr_swap_pages() > 0)
1982                 inactive_lru_pages += zone_page_state(zone, NR_INACTIVE_ANON);
1983         if (sc->nr_reclaimed < pages_for_compaction &&
1984                         inactive_lru_pages > pages_for_compaction)
1985                 return true;
1986
1987         /* If compaction would go ahead or the allocation would succeed, stop */
1988         switch (compaction_suitable(zone, sc->order)) {
1989         case COMPACT_PARTIAL:
1990         case COMPACT_CONTINUE:
1991                 return false;
1992         default:
1993                 return true;
1994         }
1995 }
1996
1997 static void shrink_zone(struct zone *zone, struct scan_control *sc)
1998 {
1999         unsigned long nr_reclaimed, nr_scanned;
2000
2001         do {
2002                 struct mem_cgroup *root = sc->target_mem_cgroup;
2003                 struct mem_cgroup_reclaim_cookie reclaim = {
2004                         .zone = zone,
2005                         .priority = sc->priority,
2006                 };
2007                 struct mem_cgroup *memcg;
2008
2009                 nr_reclaimed = sc->nr_reclaimed;
2010                 nr_scanned = sc->nr_scanned;
2011
2012                 memcg = mem_cgroup_iter(root, NULL, &reclaim);
2013                 do {
2014                         struct lruvec *lruvec;
2015
2016                         lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2017
2018                         shrink_lruvec(lruvec, sc);
2019
2020                         /*
2021                          * Direct reclaim and kswapd have to scan all memory
2022                          * cgroups to fulfill the overall scan target for the
2023                          * zone.
2024                          *
2025                          * Limit reclaim, on the other hand, only cares about
2026                          * nr_to_reclaim pages to be reclaimed and it will
2027                          * retry with decreasing priority if one round over the
2028                          * whole hierarchy is not sufficient.
2029                          */
2030                         if (!global_reclaim(sc) &&
2031                                         sc->nr_reclaimed >= sc->nr_to_reclaim) {
2032                                 mem_cgroup_iter_break(root, memcg);
2033                                 break;
2034                         }
2035                         memcg = mem_cgroup_iter(root, memcg, &reclaim);
2036                 } while (memcg);
2037
2038                 vmpressure(sc->gfp_mask, sc->target_mem_cgroup,
2039                            sc->nr_scanned - nr_scanned,
2040                            sc->nr_reclaimed - nr_reclaimed);
2041
2042         } while (should_continue_reclaim(zone, sc->nr_reclaimed - nr_reclaimed,
2043                                          sc->nr_scanned - nr_scanned, sc));
2044 }
2045
2046 /* Returns true if compaction should go ahead for a high-order request */
2047 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
2048 {
2049         unsigned long balance_gap, watermark;
2050         bool watermark_ok;
2051
2052         /* Do not consider compaction for orders reclaim is meant to satisfy */
2053         if (sc->order <= PAGE_ALLOC_COSTLY_ORDER)
2054                 return false;
2055
2056         /*
2057          * Compaction takes time to run and there are potentially other
2058          * callers using the pages just freed. Continue reclaiming until
2059          * there is a buffer of free pages available to give compaction
2060          * a reasonable chance of completing and allocating the page
2061          */
2062         balance_gap = min(low_wmark_pages(zone),
2063                 (zone->managed_pages + KSWAPD_ZONE_BALANCE_GAP_RATIO-1) /
2064                         KSWAPD_ZONE_BALANCE_GAP_RATIO);
2065         watermark = high_wmark_pages(zone) + balance_gap + (2UL << sc->order);
2066         watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, 0, 0);
2067
2068         /*
2069          * If compaction is deferred, reclaim up to a point where
2070          * compaction will have a chance of success when re-enabled
2071          */
2072         if (compaction_deferred(zone, sc->order))
2073                 return watermark_ok;
2074
2075         /* If compaction is not ready to start, keep reclaiming */
2076         if (!compaction_suitable(zone, sc->order))
2077                 return false;
2078
2079         return watermark_ok;
2080 }
2081
2082 /*
2083  * This is the direct reclaim path, for page-allocating processes.  We only
2084  * try to reclaim pages from zones which will satisfy the caller's allocation
2085  * request.
2086  *
2087  * We reclaim from a zone even if that zone is over high_wmark_pages(zone).
2088  * Because:
2089  * a) The caller may be trying to free *extra* pages to satisfy a higher-order
2090  *    allocation or
2091  * b) The target zone may be at high_wmark_pages(zone) but the lower zones
2092  *    must go *over* high_wmark_pages(zone) to satisfy the `incremental min'
2093  *    zone defense algorithm.
2094  *
2095  * If a zone is deemed to be full of pinned pages then just give it a light
2096  * scan then give up on it.
2097  *
2098  * This function returns true if a zone is being reclaimed for a costly
2099  * high-order allocation and compaction is ready to begin. This indicates to
2100  * the caller that it should consider retrying the allocation instead of
2101  * further reclaim.
2102  */
2103 static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
2104 {
2105         struct zoneref *z;
2106         struct zone *zone;
2107         unsigned long nr_soft_reclaimed;
2108         unsigned long nr_soft_scanned;
2109         bool aborted_reclaim = false;
2110
2111         /*
2112          * If the number of buffer_heads in the machine exceeds the maximum
2113          * allowed level, force direct reclaim to scan the highmem zone as
2114          * highmem pages could be pinning lowmem pages storing buffer_heads
2115          */
2116         if (buffer_heads_over_limit)
2117                 sc->gfp_mask |= __GFP_HIGHMEM;
2118
2119         for_each_zone_zonelist_nodemask(zone, z, zonelist,
2120                                         gfp_zone(sc->gfp_mask), sc->nodemask) {
2121                 if (!populated_zone(zone))
2122                         continue;
2123                 /*
2124                  * Take care memory controller reclaiming has small influence
2125                  * to global LRU.
2126                  */
2127                 if (global_reclaim(sc)) {
2128                         if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2129                                 continue;
2130                         if (zone->all_unreclaimable &&
2131                                         sc->priority != DEF_PRIORITY)
2132                                 continue;       /* Let kswapd poll it */
2133                         if (IS_ENABLED(CONFIG_COMPACTION)) {
2134                                 /*
2135                                  * If we already have plenty of memory free for
2136                                  * compaction in this zone, don't free any more.
2137                                  * Even though compaction is invoked for any
2138                                  * non-zero order, only frequent costly order
2139                                  * reclamation is disruptive enough to become a
2140                                  * noticeable problem, like transparent huge
2141                                  * page allocations.
2142                                  */
2143                                 if (compaction_ready(zone, sc)) {
2144                                         aborted_reclaim = true;
2145                                         continue;
2146                                 }
2147                         }
2148                         /*
2149                          * This steals pages from memory cgroups over softlimit
2150                          * and returns the number of reclaimed pages and
2151                          * scanned pages. This works for global memory pressure
2152                          * and balancing, not for a memcg's limit.
2153                          */
2154                         nr_soft_scanned = 0;
2155                         nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2156                                                 sc->order, sc->gfp_mask,
2157                                                 &nr_soft_scanned);
2158                         sc->nr_reclaimed += nr_soft_reclaimed;
2159                         sc->nr_scanned += nr_soft_scanned;
2160                         /* need some check for avoid more shrink_zone() */
2161                 }
2162
2163                 shrink_zone(zone, sc);
2164         }
2165
2166         return aborted_reclaim;
2167 }
2168
2169 static bool zone_reclaimable(struct zone *zone)
2170 {
2171         return zone->pages_scanned < zone_reclaimable_pages(zone) * 6;
2172 }
2173
2174 /* All zones in zonelist are unreclaimable? */
2175 static bool all_unreclaimable(struct zonelist *zonelist,
2176                 struct scan_control *sc)
2177 {
2178         struct zoneref *z;
2179         struct zone *zone;
2180
2181         for_each_zone_zonelist_nodemask(zone, z, zonelist,
2182                         gfp_zone(sc->gfp_mask), sc->nodemask) {
2183                 if (!populated_zone(zone))
2184                         continue;
2185                 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2186                         continue;
2187                 if (!zone->all_unreclaimable)
2188                         return false;
2189         }
2190
2191         return true;
2192 }
2193
2194 /*
2195  * This is the main entry point to direct page reclaim.
2196  *
2197  * If a full scan of the inactive list fails to free enough memory then we
2198  * are "out of memory" and something needs to be killed.
2199  *
2200  * If the caller is !__GFP_FS then the probability of a failure is reasonably
2201  * high - the zone may be full of dirty or under-writeback pages, which this
2202  * caller can't do much about.  We kick the writeback threads and take explicit
2203  * naps in the hope that some of these pages can be written.  But if the
2204  * allocating task holds filesystem locks which prevent writeout this might not
2205  * work, and the allocation attempt will fail.
2206  *
2207  * returns:     0, if no pages reclaimed
2208  *              else, the number of pages reclaimed
2209  */
2210 static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
2211                                         struct scan_control *sc,
2212                                         struct shrink_control *shrink)
2213 {
2214         unsigned long total_scanned = 0;
2215         struct reclaim_state *reclaim_state = current->reclaim_state;
2216         struct zoneref *z;
2217         struct zone *zone;
2218         unsigned long writeback_threshold;
2219         bool aborted_reclaim;
2220
2221         delayacct_freepages_start();
2222
2223         if (global_reclaim(sc))
2224                 count_vm_event(ALLOCSTALL);
2225
2226         do {
2227                 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
2228                                 sc->priority);
2229                 sc->nr_scanned = 0;
2230                 aborted_reclaim = shrink_zones(zonelist, sc);
2231
2232                 /*
2233                  * Don't shrink slabs when reclaiming memory from
2234                  * over limit cgroups
2235                  */
2236                 if (global_reclaim(sc)) {
2237                         unsigned long lru_pages = 0;
2238                         for_each_zone_zonelist(zone, z, zonelist,
2239                                         gfp_zone(sc->gfp_mask)) {
2240                                 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2241                                         continue;
2242
2243                                 lru_pages += zone_reclaimable_pages(zone);
2244                         }
2245
2246                         shrink_slab(shrink, sc->nr_scanned, lru_pages);
2247                         if (reclaim_state) {
2248                                 sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2249                                 reclaim_state->reclaimed_slab = 0;
2250                         }
2251                 }
2252                 total_scanned += sc->nr_scanned;
2253                 if (sc->nr_reclaimed >= sc->nr_to_reclaim)
2254                         goto out;
2255
2256                 /*
2257                  * If we're getting trouble reclaiming, start doing
2258                  * writepage even in laptop mode.
2259                  */
2260                 if (sc->priority < DEF_PRIORITY - 2)
2261                         sc->may_writepage = 1;
2262
2263                 /*
2264                  * Try to write back as many pages as we just scanned.  This
2265                  * tends to cause slow streaming writers to write data to the
2266                  * disk smoothly, at the dirtying rate, which is nice.   But
2267                  * that's undesirable in laptop mode, where we *want* lumpy
2268                  * writeout.  So in laptop mode, write out the whole world.
2269                  */
2270                 writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
2271                 if (total_scanned > writeback_threshold) {
2272                         wakeup_flusher_threads(laptop_mode ? 0 : total_scanned,
2273                                                 WB_REASON_TRY_TO_FREE_PAGES);
2274                         sc->may_writepage = 1;
2275                 }
2276
2277                 /* Take a nap, wait for some writeback to complete */
2278                 if (!sc->hibernation_mode && sc->nr_scanned &&
2279                     sc->priority < DEF_PRIORITY - 2) {
2280                         struct zone *preferred_zone;
2281
2282                         first_zones_zonelist(zonelist, gfp_zone(sc->gfp_mask),
2283                                                 &cpuset_current_mems_allowed,
2284                                                 &preferred_zone);
2285                         wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/10);
2286                 }
2287         } while (--sc->priority >= 0);
2288
2289 out:
2290         delayacct_freepages_end();
2291
2292         if (sc->nr_reclaimed)
2293                 return sc->nr_reclaimed;
2294
2295         /*
2296          * As hibernation is going on, kswapd is freezed so that it can't mark
2297          * the zone into all_unreclaimable. Thus bypassing all_unreclaimable
2298          * check.
2299          */
2300         if (oom_killer_disabled)
2301                 return 0;
2302
2303         /* Aborted reclaim to try compaction? don't OOM, then */
2304         if (aborted_reclaim)
2305                 return 1;
2306
2307         /* top priority shrink_zones still had more to do? don't OOM, then */
2308         if (global_reclaim(sc) && !all_unreclaimable(zonelist, sc))
2309                 return 1;
2310
2311         return 0;
2312 }
2313
2314 static bool pfmemalloc_watermark_ok(pg_data_t *pgdat)
2315 {
2316         struct zone *zone;
2317         unsigned long pfmemalloc_reserve = 0;
2318         unsigned long free_pages = 0;
2319         int i;
2320         bool wmark_ok;
2321
2322         for (i = 0; i <= ZONE_NORMAL; i++) {
2323                 zone = &pgdat->node_zones[i];
2324                 pfmemalloc_reserve += min_wmark_pages(zone);
2325                 free_pages += zone_page_state(zone, NR_FREE_PAGES);
2326         }
2327
2328         wmark_ok = free_pages > pfmemalloc_reserve / 2;
2329
2330         /* kswapd must be awake if processes are being throttled */
2331         if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
2332                 pgdat->classzone_idx = min(pgdat->classzone_idx,
2333                                                 (enum zone_type)ZONE_NORMAL);
2334                 wake_up_interruptible(&pgdat->kswapd_wait);
2335         }
2336
2337         return wmark_ok;
2338 }
2339
2340 /*
2341  * Throttle direct reclaimers if backing storage is backed by the network
2342  * and the PFMEMALLOC reserve for the preferred node is getting dangerously
2343  * depleted. kswapd will continue to make progress and wake the processes
2344  * when the low watermark is reached.
2345  *
2346  * Returns true if a fatal signal was delivered during throttling. If this
2347  * happens, the page allocator should not consider triggering the OOM killer.
2348  */
2349 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
2350                                         nodemask_t *nodemask)
2351 {
2352         struct zone *zone;
2353         int high_zoneidx = gfp_zone(gfp_mask);
2354         pg_data_t *pgdat;
2355
2356         /*
2357          * Kernel threads should not be throttled as they may be indirectly
2358          * responsible for cleaning pages necessary for reclaim to make forward
2359          * progress. kjournald for example may enter direct reclaim while
2360          * committing a transaction where throttling it could forcing other
2361          * processes to block on log_wait_commit().
2362          */
2363         if (current->flags & PF_KTHREAD)
2364                 goto out;
2365
2366         /*
2367          * If a fatal signal is pending, this process should not throttle.
2368          * It should return quickly so it can exit and free its memory
2369          */
2370         if (fatal_signal_pending(current))
2371                 goto out;
2372
2373         /* Check if the pfmemalloc reserves are ok */
2374         first_zones_zonelist(zonelist, high_zoneidx, NULL, &zone);
2375         pgdat = zone->zone_pgdat;
2376         if (pfmemalloc_watermark_ok(pgdat))
2377                 goto out;
2378
2379         /* Account for the throttling */
2380         count_vm_event(PGSCAN_DIRECT_THROTTLE);
2381
2382         /*
2383          * If the caller cannot enter the filesystem, it's possible that it
2384          * is due to the caller holding an FS lock or performing a journal
2385          * transaction in the case of a filesystem like ext[3|4]. In this case,
2386          * it is not safe to block on pfmemalloc_wait as kswapd could be
2387          * blocked waiting on the same lock. Instead, throttle for up to a
2388          * second before continuing.
2389          */
2390         if (!(gfp_mask & __GFP_FS)) {
2391                 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
2392                         pfmemalloc_watermark_ok(pgdat), HZ);
2393
2394                 goto check_pending;
2395         }
2396
2397         /* Throttle until kswapd wakes the process */
2398         wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
2399                 pfmemalloc_watermark_ok(pgdat));
2400
2401 check_pending:
2402         if (fatal_signal_pending(current))
2403                 return true;
2404
2405 out:
2406         return false;
2407 }
2408
2409 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
2410                                 gfp_t gfp_mask, nodemask_t *nodemask)
2411 {
2412         unsigned long nr_reclaimed;
2413         struct scan_control sc = {
2414                 .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
2415                 .may_writepage = !laptop_mode,
2416                 .nr_to_reclaim = SWAP_CLUSTER_MAX,
2417                 .may_unmap = 1,
2418                 .may_swap = 1,
2419                 .order = order,
2420                 .priority = DEF_PRIORITY,
2421                 .target_mem_cgroup = NULL,
2422                 .nodemask = nodemask,
2423         };
2424         struct shrink_control shrink = {
2425                 .gfp_mask = sc.gfp_mask,
2426         };
2427
2428         /*
2429          * Do not enter reclaim if fatal signal was delivered while throttled.
2430          * 1 is returned so that the page allocator does not OOM kill at this
2431          * point.
2432          */
2433         if (throttle_direct_reclaim(gfp_mask, zonelist, nodemask))
2434                 return 1;
2435
2436         trace_mm_vmscan_direct_reclaim_begin(order,
2437                                 sc.may_writepage,
2438                                 gfp_mask);
2439
2440         nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
2441
2442         trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
2443
2444         return nr_reclaimed;
2445 }
2446
2447 #ifdef CONFIG_MEMCG
2448
2449 unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
2450                                                 gfp_t gfp_mask, bool noswap,
2451                                                 struct zone *zone,
2452                                                 unsigned long *nr_scanned)
2453 {
2454         struct scan_control sc = {
2455                 .nr_scanned = 0,
2456                 .nr_to_reclaim = SWAP_CLUSTER_MAX,
2457                 .may_writepage = !laptop_mode,
2458                 .may_unmap = 1,
2459                 .may_swap = !noswap,
2460                 .order = 0,
2461                 .priority = 0,
2462                 .target_mem_cgroup = memcg,
2463         };
2464         struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2465
2466         sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2467                         (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
2468
2469         trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
2470                                                       sc.may_writepage,
2471                                                       sc.gfp_mask);
2472
2473         /*
2474          * NOTE: Although we can get the priority field, using it
2475          * here is not a good idea, since it limits the pages we can scan.
2476          * if we don't reclaim here, the shrink_zone from balance_pgdat
2477          * will pick up pages from other mem cgroup's as well. We hack
2478          * the priority and make it zero.
2479          */
2480         shrink_lruvec(lruvec, &sc);
2481
2482         trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
2483
2484         *nr_scanned = sc.nr_scanned;
2485         return sc.nr_reclaimed;
2486 }
2487
2488 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
2489                                            gfp_t gfp_mask,
2490                                            bool noswap)
2491 {
2492         struct zonelist *zonelist;
2493         unsigned long nr_reclaimed;
2494         int nid;
2495         struct scan_control sc = {
2496                 .may_writepage = !laptop_mode,
2497                 .may_unmap = 1,
2498                 .may_swap = !noswap,
2499                 .nr_to_reclaim = SWAP_CLUSTER_MAX,
2500                 .order = 0,
2501                 .priority = DEF_PRIORITY,
2502                 .target_mem_cgroup = memcg,
2503                 .nodemask = NULL, /* we don't care the placement */
2504                 .gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2505                                 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
2506         };
2507         struct shrink_control shrink = {
2508                 .gfp_mask = sc.gfp_mask,
2509         };
2510
2511         /*
2512          * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
2513          * take care of from where we get pages. So the node where we start the
2514          * scan does not need to be the current node.
2515          */
2516         nid = mem_cgroup_select_victim_node(memcg);
2517
2518         zonelist = NODE_DATA(nid)->node_zonelists;
2519
2520         trace_mm_vmscan_memcg_reclaim_begin(0,
2521                                             sc.may_writepage,
2522                                             sc.gfp_mask);
2523
2524         nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
2525
2526         trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
2527
2528         return nr_reclaimed;
2529 }
2530 #endif
2531
2532 static void age_active_anon(struct zone *zone, struct scan_control *sc)
2533 {
2534         struct mem_cgroup *memcg;
2535
2536         if (!total_swap_pages)
2537                 return;
2538
2539         memcg = mem_cgroup_iter(NULL, NULL, NULL);
2540         do {
2541                 struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2542
2543                 if (inactive_anon_is_low(lruvec))
2544                         shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2545                                            sc, LRU_ACTIVE_ANON);
2546
2547                 memcg = mem_cgroup_iter(NULL, memcg, NULL);
2548         } while (memcg);
2549 }
2550
2551 static bool zone_balanced(struct zone *zone, int order,
2552                           unsigned long balance_gap, int classzone_idx)
2553 {
2554         if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone) +
2555                                     balance_gap, classzone_idx, 0))
2556                 return false;
2557
2558         if (IS_ENABLED(CONFIG_COMPACTION) && order &&
2559             !compaction_suitable(zone, order))
2560                 return false;
2561
2562         return true;
2563 }
2564
2565 /*
2566  * pgdat_balanced() is used when checking if a node is balanced.
2567  *
2568  * For order-0, all zones must be balanced!
2569  *
2570  * For high-order allocations only zones that meet watermarks and are in a
2571  * zone allowed by the callers classzone_idx are added to balanced_pages. The
2572  * total of balanced pages must be at least 25% of the zones allowed by
2573  * classzone_idx for the node to be considered balanced. Forcing all zones to
2574  * be balanced for high orders can cause excessive reclaim when there are
2575  * imbalanced zones.
2576  * The choice of 25% is due to
2577  *   o a 16M DMA zone that is balanced will not balance a zone on any
2578  *     reasonable sized machine
2579  *   o On all other machines, the top zone must be at least a reasonable
2580  *     percentage of the middle zones. For example, on 32-bit x86, highmem
2581  *     would need to be at least 256M for it to be balance a whole node.
2582  *     Similarly, on x86-64 the Normal zone would need to be at least 1G
2583  *     to balance a node on its own. These seemed like reasonable ratios.
2584  */
2585 static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
2586 {
2587         unsigned long managed_pages = 0;
2588         unsigned long balanced_pages = 0;
2589         int i;
2590
2591         /* Check the watermark levels */
2592         for (i = 0; i <= classzone_idx; i++) {
2593                 struct zone *zone = pgdat->node_zones + i;
2594
2595                 if (!populated_zone(zone))
2596                         continue;
2597
2598                 managed_pages += zone->managed_pages;
2599
2600                 /*
2601                  * A special case here:
2602                  *
2603                  * balance_pgdat() skips over all_unreclaimable after
2604                  * DEF_PRIORITY. Effectively, it considers them balanced so
2605                  * they must be considered balanced here as well!
2606                  */
2607                 if (zone->all_unreclaimable) {
2608                         balanced_pages += zone->managed_pages;
2609                         continue;
2610                 }
2611
2612                 if (zone_balanced(zone, order, 0, i))
2613                         balanced_pages += zone->managed_pages;
2614                 else if (!order)
2615                         return false;
2616         }
2617
2618         if (order)
2619                 return balanced_pages >= (managed_pages >> 2);
2620         else
2621                 return true;
2622 }
2623
2624 /*
2625  * Prepare kswapd for sleeping. This verifies that there are no processes
2626  * waiting in throttle_direct_reclaim() and that watermarks have been met.
2627  *
2628  * Returns true if kswapd is ready to sleep
2629  */
2630 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, long remaining,
2631                                         int classzone_idx)
2632 {
2633         /* If a direct reclaimer woke kswapd within HZ/10, it's premature */
2634         if (remaining)
2635                 return false;
2636
2637         /*
2638          * There is a potential race between when kswapd checks its watermarks
2639          * and a process gets throttled. There is also a potential race if
2640          * processes get throttled, kswapd wakes, a large process exits therby
2641          * balancing the zones that causes kswapd to miss a wakeup. If kswapd
2642          * is going to sleep, no process should be sleeping on pfmemalloc_wait
2643          * so wake them now if necessary. If necessary, processes will wake
2644          * kswapd and get throttled again
2645          */
2646         if (waitqueue_active(&pgdat->pfmemalloc_wait)) {
2647                 wake_up(&pgdat->pfmemalloc_wait);
2648                 return false;
2649         }
2650
2651         return pgdat_balanced(pgdat, order, classzone_idx);
2652 }
2653
2654 /*
2655  * kswapd shrinks the zone by the number of pages required to reach
2656  * the high watermark.
2657  *
2658  * Returns true if kswapd scanned at least the requested number of pages to
2659  * reclaim. This is used to determine if the scanning priority needs to be
2660  * raised.
2661  */
2662 static bool kswapd_shrink_zone(struct zone *zone,
2663                                struct scan_control *sc,
2664                                unsigned long lru_pages)
2665 {
2666         unsigned long nr_slab;
2667         struct reclaim_state *reclaim_state = current->reclaim_state;
2668         struct shrink_control shrink = {
2669                 .gfp_mask = sc->gfp_mask,
2670         };
2671
2672         /* Reclaim above the high watermark. */
2673         sc->nr_to_reclaim = max(SWAP_CLUSTER_MAX, high_wmark_pages(zone));
2674         shrink_zone(zone, sc);
2675
2676         reclaim_state->reclaimed_slab = 0;
2677         nr_slab = shrink_slab(&shrink, sc->nr_scanned, lru_pages);
2678         sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2679
2680         if (nr_slab == 0 && !zone_reclaimable(zone))
2681                 zone->all_unreclaimable = 1;
2682
2683         return sc->nr_scanned >= sc->nr_to_reclaim;
2684 }
2685
2686 /*
2687  * For kswapd, balance_pgdat() will work across all this node's zones until
2688  * they are all at high_wmark_pages(zone).
2689  *
2690  * Returns the final order kswapd was reclaiming at
2691  *
2692  * There is special handling here for zones which are full of pinned pages.
2693  * This can happen if the pages are all mlocked, or if they are all used by
2694  * device drivers (say, ZONE_DMA).  Or if they are all in use by hugetlb.
2695  * What we do is to detect the case where all pages in the zone have been
2696  * scanned twice and there has been zero successful reclaim.  Mark the zone as
2697  * dead and from now on, only perform a short scan.  Basically we're polling
2698  * the zone for when the problem goes away.
2699  *
2700  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
2701  * zones which have free_pages > high_wmark_pages(zone), but once a zone is
2702  * found to have free_pages <= high_wmark_pages(zone), we scan that zone and the
2703  * lower zones regardless of the number of free pages in the lower zones. This
2704  * interoperates with the page allocator fallback scheme to ensure that aging
2705  * of pages is balanced across the zones.
2706  */
2707 static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
2708                                                         int *classzone_idx)
2709 {
2710         int i;
2711         int end_zone = 0;       /* Inclusive.  0 = ZONE_DMA */
2712         unsigned long nr_soft_reclaimed;
2713         unsigned long nr_soft_scanned;
2714         struct scan_control sc = {
2715                 .gfp_mask = GFP_KERNEL,
2716                 .priority = DEF_PRIORITY,
2717                 .may_unmap = 1,
2718                 .may_swap = 1,
2719                 .may_writepage = !laptop_mode,
2720                 .order = order,
2721                 .target_mem_cgroup = NULL,
2722         };
2723         count_vm_event(PAGEOUTRUN);
2724
2725         do {
2726                 unsigned long lru_pages = 0;
2727                 bool raise_priority = true;
2728
2729                 sc.nr_reclaimed = 0;
2730
2731                 /*
2732                  * Scan in the highmem->dma direction for the highest
2733                  * zone which needs scanning
2734                  */
2735                 for (i = pgdat->nr_zones - 1; i >= 0; i--) {
2736                         struct zone *zone = pgdat->node_zones + i;
2737
2738                         if (!populated_zone(zone))
2739                                 continue;
2740
2741                         if (zone->all_unreclaimable &&
2742                             sc.priority != DEF_PRIORITY)
2743                                 continue;
2744
2745                         /*
2746                          * Do some background aging of the anon list, to give
2747                          * pages a chance to be referenced before reclaiming.
2748                          */
2749                         age_active_anon(zone, &sc);
2750
2751                         /*
2752                          * If the number of buffer_heads in the machine
2753                          * exceeds the maximum allowed level and this node
2754                          * has a highmem zone, force kswapd to reclaim from
2755                          * it to relieve lowmem pressure.
2756                          */
2757                         if (buffer_heads_over_limit && is_highmem_idx(i)) {
2758                                 end_zone = i;
2759                                 break;
2760                         }
2761
2762                         if (!zone_balanced(zone, order, 0, 0)) {
2763                                 end_zone = i;
2764                                 break;
2765                         } else {
2766                                 /* If balanced, clear the congested flag */
2767                                 zone_clear_flag(zone, ZONE_CONGESTED);
2768                         }
2769                 }
2770
2771                 if (i < 0)
2772                         goto out;
2773
2774                 for (i = 0; i <= end_zone; i++) {
2775                         struct zone *zone = pgdat->node_zones + i;
2776
2777                         lru_pages += zone_reclaimable_pages(zone);
2778                 }
2779
2780                 /*
2781                  * Now scan the zone in the dma->highmem direction, stopping
2782                  * at the last zone which needs scanning.
2783                  *
2784                  * We do this because the page allocator works in the opposite
2785                  * direction.  This prevents the page allocator from allocating
2786                  * pages behind kswapd's direction of progress, which would
2787                  * cause too much scanning of the lower zones.
2788                  */
2789                 for (i = 0; i <= end_zone; i++) {
2790                         struct zone *zone = pgdat->node_zones + i;
2791                         int testorder;
2792                         unsigned long balance_gap;
2793
2794                         if (!populated_zone(zone))
2795                                 continue;
2796
2797                         if (zone->all_unreclaimable &&
2798                             sc.priority != DEF_PRIORITY)
2799                                 continue;
2800
2801                         sc.nr_scanned = 0;
2802
2803                         nr_soft_scanned = 0;
2804                         /*
2805                          * Call soft limit reclaim before calling shrink_zone.
2806                          */
2807                         nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2808                                                         order, sc.gfp_mask,
2809                                                         &nr_soft_scanned);
2810                         sc.nr_reclaimed += nr_soft_reclaimed;
2811
2812                         /*
2813                          * We put equal pressure on every zone, unless
2814                          * one zone has way too many pages free
2815                          * already. The "too many pages" is defined
2816                          * as the high wmark plus a "gap" where the
2817                          * gap is either the low watermark or 1%
2818                          * of the zone, whichever is smaller.
2819                          */
2820                         balance_gap = min(low_wmark_pages(zone),
2821                                 (zone->managed_pages +
2822                                         KSWAPD_ZONE_BALANCE_GAP_RATIO-1) /
2823                                 KSWAPD_ZONE_BALANCE_GAP_RATIO);
2824                         /*
2825                          * Kswapd reclaims only single pages with compaction
2826                          * enabled. Trying too hard to reclaim until contiguous
2827                          * free pages have become available can hurt performance
2828                          * by evicting too much useful data from memory.
2829                          * Do not reclaim more than needed for compaction.
2830                          */
2831                         testorder = order;
2832                         if (IS_ENABLED(CONFIG_COMPACTION) && order &&
2833                                         compaction_suitable(zone, order) !=
2834                                                 COMPACT_SKIPPED)
2835                                 testorder = 0;
2836
2837                         if ((buffer_heads_over_limit && is_highmem_idx(i)) ||
2838                             !zone_balanced(zone, testorder,
2839                                            balance_gap, end_zone)) {
2840                                 /*
2841                                  * There should be no need to raise the
2842                                  * scanning priority if enough pages are
2843                                  * already being scanned that high
2844                                  * watermark would be met at 100% efficiency.
2845                                  */
2846                                 if (kswapd_shrink_zone(zone, &sc, lru_pages))
2847                                         raise_priority = false;
2848                         }
2849
2850                         /*
2851                          * If we're getting trouble reclaiming, start doing
2852                          * writepage even in laptop mode.
2853                          */
2854                         if (sc.priority < DEF_PRIORITY - 2)
2855                                 sc.may_writepage = 1;
2856
2857                         if (zone->all_unreclaimable) {
2858                                 if (end_zone && end_zone == i)
2859                                         end_zone--;
2860                                 continue;
2861                         }
2862
2863                         if (zone_balanced(zone, testorder, 0, end_zone))
2864                                 /*
2865                                  * If a zone reaches its high watermark,
2866                                  * consider it to be no longer congested. It's
2867                                  * possible there are dirty pages backed by
2868                                  * congested BDIs but as pressure is relieved,
2869                                  * speculatively avoid congestion waits
2870                                  */
2871                                 zone_clear_flag(zone, ZONE_CONGESTED);
2872                 }
2873
2874                 /*
2875                  * If the low watermark is met there is no need for processes
2876                  * to be throttled on pfmemalloc_wait as they should not be
2877                  * able to safely make forward progress. Wake them
2878                  */
2879                 if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
2880                                 pfmemalloc_watermark_ok(pgdat))
2881                         wake_up(&pgdat->pfmemalloc_wait);
2882
2883                 /*
2884                  * Fragmentation may mean that the system cannot be rebalanced
2885                  * for high-order allocations in all zones. If twice the
2886                  * allocation size has been reclaimed and the zones are still
2887                  * not balanced then recheck the watermarks at order-0 to
2888                  * prevent kswapd reclaiming excessively. Assume that a
2889                  * process requested a high-order can direct reclaim/compact.
2890                  */
2891                 if (order && sc.nr_reclaimed >= 2UL << order)
2892                         order = sc.order = 0;
2893
2894                 /* Check if kswapd should be suspending */
2895                 if (try_to_freeze() || kthread_should_stop())
2896                         break;
2897
2898                 /*
2899                  * Raise priority if scanning rate is too low or there was no
2900                  * progress in reclaiming pages
2901                  */
2902                 if (raise_priority || !sc.nr_reclaimed)
2903                         sc.priority--;
2904         } while (sc.priority >= 0 &&
2905                  !pgdat_balanced(pgdat, order, *classzone_idx));
2906
2907         /*
2908          * If kswapd was reclaiming at a higher order, it has the option of
2909          * sleeping without all zones being balanced. Before it does, it must
2910          * ensure that the watermarks for order-0 on *all* zones are met and
2911          * that the congestion flags are cleared. The congestion flag must
2912          * be cleared as kswapd is the only mechanism that clears the flag
2913          * and it is potentially going to sleep here.
2914          */
2915         if (order) {
2916                 int zones_need_compaction = 1;
2917
2918                 for (i = 0; i <= end_zone; i++) {
2919                         struct zone *zone = pgdat->node_zones + i;
2920
2921                         if (!populated_zone(zone))
2922                                 continue;
2923
2924                         /* Check if the memory needs to be defragmented. */
2925                         if (zone_watermark_ok(zone, order,
2926                                     low_wmark_pages(zone), *classzone_idx, 0))
2927                                 zones_need_compaction = 0;
2928                 }
2929
2930                 if (zones_need_compaction)
2931                         compact_pgdat(pgdat, order);
2932         }
2933
2934 out:
2935         /*
2936          * Return the order we were reclaiming at so prepare_kswapd_sleep()
2937          * makes a decision on the order we were last reclaiming at. However,
2938          * if another caller entered the allocator slow path while kswapd
2939          * was awake, order will remain at the higher level
2940          */
2941         *classzone_idx = end_zone;
2942         return order;
2943 }
2944
2945 static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
2946 {
2947         long remaining = 0;
2948         DEFINE_WAIT(wait);
2949
2950         if (freezing(current) || kthread_should_stop())
2951                 return;
2952
2953         prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
2954
2955         /* Try to sleep for a short interval */
2956         if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
2957                 remaining = schedule_timeout(HZ/10);
2958                 finish_wait(&pgdat->kswapd_wait, &wait);
2959                 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
2960         }
2961
2962         /*
2963          * After a short sleep, check if it was a premature sleep. If not, then
2964          * go fully to sleep until explicitly woken up.
2965          */
2966         if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
2967                 trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
2968
2969                 /*
2970                  * vmstat counters are not perfectly accurate and the estimated
2971                  * value for counters such as NR_FREE_PAGES can deviate from the
2972                  * true value by nr_online_cpus * threshold. To avoid the zone
2973                  * watermarks being breached while under pressure, we reduce the
2974                  * per-cpu vmstat threshold while kswapd is awake and restore
2975                  * them before going back to sleep.
2976                  */
2977                 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
2978
2979                 /*
2980                  * Compaction records what page blocks it recently failed to
2981                  * isolate pages from and skips them in the future scanning.
2982                  * When kswapd is going to sleep, it is reasonable to assume
2983                  * that pages and compaction may succeed so reset the cache.
2984                  */
2985                 reset_isolation_suitable(pgdat);
2986
2987                 if (!kthread_should_stop())
2988                         schedule();
2989
2990                 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
2991         } else {
2992                 if (remaining)
2993                         count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
2994                 else
2995                         count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
2996         }
2997         finish_wait(&pgdat->kswapd_wait, &wait);
2998 }
2999
3000 /*
3001  * The background pageout daemon, started as a kernel thread
3002  * from the init process.
3003  *
3004  * This basically trickles out pages so that we have _some_
3005  * free memory available even if there is no other activity
3006  * that frees anything up. This is needed for things like routing
3007  * etc, where we otherwise might have all activity going on in
3008  * asynchronous contexts that cannot page things out.
3009  *
3010  * If there are applications that are active memory-allocators
3011  * (most normal use), this basically shouldn't matter.
3012  */
3013 static int kswapd(void *p)
3014 {
3015         unsigned long order, new_order;
3016         unsigned balanced_order;
3017         int classzone_idx, new_classzone_idx;
3018         int balanced_classzone_idx;
3019         pg_data_t *pgdat = (pg_data_t*)p;
3020         struct task_struct *tsk = current;
3021
3022         struct reclaim_state reclaim_state = {
3023                 .reclaimed_slab = 0,
3024         };
3025         const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3026
3027         lockdep_set_current_reclaim_state(GFP_KERNEL);
3028
3029         if (!cpumask_empty(cpumask))
3030                 set_cpus_allowed_ptr(tsk, cpumask);
3031         current->reclaim_state = &reclaim_state;
3032
3033         /*
3034          * Tell the memory management that we're a "memory allocator",
3035          * and that if we need more memory we should get access to it
3036          * regardless (see "__alloc_pages()"). "kswapd" should
3037          * never get caught in the normal page freeing logic.
3038          *
3039          * (Kswapd normally doesn't need memory anyway, but sometimes
3040          * you need a small amount of memory in order to be able to
3041          * page out something else, and this flag essentially protects
3042          * us from recursively trying to free more memory as we're
3043          * trying to free the first piece of memory in the first place).
3044          */
3045         tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
3046         set_freezable();
3047
3048         order = new_order = 0;
3049         balanced_order = 0;
3050         classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
3051         balanced_classzone_idx = classzone_idx;
3052         for ( ; ; ) {
3053                 bool ret;
3054
3055                 /*
3056                  * If the last balance_pgdat was unsuccessful it's unlikely a
3057                  * new request of a similar or harder type will succeed soon
3058                  * so consider going to sleep on the basis we reclaimed at
3059                  */
3060                 if (balanced_classzone_idx >= new_classzone_idx &&
3061                                         balanced_order == new_order) {
3062                         new_order = pgdat->kswapd_max_order;
3063                         new_classzone_idx = pgdat->classzone_idx;
3064                         pgdat->kswapd_max_order =  0;
3065                         pgdat->classzone_idx = pgdat->nr_zones - 1;
3066                 }
3067
3068                 if (order < new_order || classzone_idx > new_classzone_idx) {
3069                         /*
3070                          * Don't sleep if someone wants a larger 'order'
3071                          * allocation or has tigher zone constraints
3072                          */
3073                         order = new_order;
3074                         classzone_idx = new_classzone_idx;
3075                 } else {
3076                         kswapd_try_to_sleep(pgdat, balanced_order,
3077                                                 balanced_classzone_idx);
3078                         order = pgdat->kswapd_max_order;
3079                         classzone_idx = pgdat->classzone_idx;
3080                         new_order = order;
3081                         new_classzone_idx = classzone_idx;
3082                         pgdat->kswapd_max_order = 0;
3083                         pgdat->classzone_idx = pgdat->nr_zones - 1;
3084                 }
3085
3086                 ret = try_to_freeze();
3087                 if (kthread_should_stop())
3088                         break;
3089
3090                 /*
3091                  * We can speed up thawing tasks if we don't call balance_pgdat
3092                  * after returning from the refrigerator
3093                  */
3094                 if (!ret) {
3095                         trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
3096                         balanced_classzone_idx = classzone_idx;
3097                         balanced_order = balance_pgdat(pgdat, order,
3098                                                 &balanced_classzone_idx);
3099                 }
3100         }
3101
3102         current->reclaim_state = NULL;
3103         return 0;
3104 }
3105
3106 /*
3107  * A zone is low on free memory, so wake its kswapd task to service it.
3108  */
3109 void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
3110 {
3111         pg_data_t *pgdat;
3112
3113         if (!populated_zone(zone))
3114                 return;
3115
3116         if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
3117                 return;
3118         pgdat = zone->zone_pgdat;
3119         if (pgdat->kswapd_max_order < order) {
3120                 pgdat->kswapd_max_order = order;
3121                 pgdat->classzone_idx = min(pgdat->classzone_idx, classzone_idx);
3122         }
3123         if (!waitqueue_active(&pgdat->kswapd_wait))
3124                 return;
3125         if (zone_watermark_ok_safe(zone, order, low_wmark_pages(zone), 0, 0))
3126                 return;
3127
3128         trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
3129         wake_up_interruptible(&pgdat->kswapd_wait);
3130 }
3131
3132 /*
3133  * The reclaimable count would be mostly accurate.
3134  * The less reclaimable pages may be
3135  * - mlocked pages, which will be moved to unevictable list when encountered
3136  * - mapped pages, which may require several travels to be reclaimed
3137  * - dirty pages, which is not "instantly" reclaimable
3138  */
3139 unsigned long global_reclaimable_pages(void)
3140 {
3141         int nr;
3142
3143         nr = global_page_state(NR_ACTIVE_FILE) +
3144              global_page_state(NR_INACTIVE_FILE);
3145
3146         if (get_nr_swap_pages() > 0)
3147                 nr += global_page_state(NR_ACTIVE_ANON) +
3148                       global_page_state(NR_INACTIVE_ANON);
3149
3150         return nr;
3151 }
3152
3153 unsigned long zone_reclaimable_pages(struct zone *zone)
3154 {
3155         int nr;
3156
3157         nr = zone_page_state(zone, NR_ACTIVE_FILE) +
3158              zone_page_state(zone, NR_INACTIVE_FILE);
3159
3160         if (get_nr_swap_pages() > 0)
3161                 nr += zone_page_state(zone, NR_ACTIVE_ANON) +
3162                       zone_page_state(zone, NR_INACTIVE_ANON);
3163
3164         return nr;
3165 }
3166
3167 #ifdef CONFIG_HIBERNATION
3168 /*
3169  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3170  * freed pages.
3171  *
3172  * Rather than trying to age LRUs the aim is to preserve the overall
3173  * LRU order by reclaiming preferentially
3174  * inactive > active > active referenced > active mapped
3175  */
3176 unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3177 {
3178         struct reclaim_state reclaim_state;
3179         struct scan_control sc = {
3180                 .gfp_mask = GFP_HIGHUSER_MOVABLE,
3181                 .may_swap = 1,
3182                 .may_unmap = 1,
3183                 .may_writepage = 1,
3184                 .nr_to_reclaim = nr_to_reclaim,
3185                 .hibernation_mode = 1,
3186                 .order = 0,
3187                 .priority = DEF_PRIORITY,
3188         };
3189         struct shrink_control shrink = {
3190                 .gfp_mask = sc.gfp_mask,
3191         };
3192         struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3193         struct task_struct *p = current;
3194         unsigned long nr_reclaimed;
3195
3196         p->flags |= PF_MEMALLOC;
3197         lockdep_set_current_reclaim_state(sc.gfp_mask);
3198         reclaim_state.reclaimed_slab = 0;
3199         p->reclaim_state = &reclaim_state;
3200
3201         nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
3202
3203         p->reclaim_state = NULL;
3204         lockdep_clear_current_reclaim_state();
3205         p->flags &= ~PF_MEMALLOC;
3206
3207         return nr_reclaimed;
3208 }
3209 #endif /* CONFIG_HIBERNATION */
3210
3211 /* It's optimal to keep kswapds on the same CPUs as their memory, but
3212    not required for correctness.  So if the last cpu in a node goes
3213    away, we get changed to run anywhere: as the first one comes back,
3214    restore their cpu bindings. */
3215 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
3216                         void *hcpu)
3217 {
3218         int nid;
3219
3220         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
3221                 for_each_node_state(nid, N_MEMORY) {
3222                         pg_data_t *pgdat = NODE_DATA(nid);
3223                         const struct cpumask *mask;
3224
3225                         mask = cpumask_of_node(pgdat->node_id);
3226
3227                         if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3228                                 /* One of our CPUs online: restore mask */
3229                                 set_cpus_allowed_ptr(pgdat->kswapd, mask);
3230                 }
3231         }
3232         return NOTIFY_OK;
3233 }
3234
3235 /*
3236  * This kswapd start function will be called by init and node-hot-add.
3237  * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3238  */
3239 int kswapd_run(int nid)
3240 {
3241         pg_data_t *pgdat = NODE_DATA(nid);
3242         int ret = 0;
3243
3244         if (pgdat->kswapd)
3245                 return 0;
3246
3247         pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3248         if (IS_ERR(pgdat->kswapd)) {
3249                 /* failure at boot is fatal */
3250                 BUG_ON(system_state == SYSTEM_BOOTING);
3251                 pr_err("Failed to start kswapd on node %d\n", nid);
3252                 ret = PTR_ERR(pgdat->kswapd);
3253                 pgdat->kswapd = NULL;
3254         }
3255         return ret;
3256 }
3257
3258 /*
3259  * Called by memory hotplug when all memory in a node is offlined.  Caller must
3260  * hold lock_memory_hotplug().
3261  */
3262 void kswapd_stop(int nid)
3263 {
3264         struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3265
3266         if (kswapd) {
3267                 kthread_stop(kswapd);
3268                 NODE_DATA(nid)->kswapd = NULL;
3269         }
3270 }
3271
3272 static int __init kswapd_init(void)
3273 {
3274         int nid;
3275
3276         swap_setup();
3277         for_each_node_state(nid, N_MEMORY)
3278                 kswapd_run(nid);
3279         hotcpu_notifier(cpu_callback, 0);
3280         return 0;
3281 }
3282
3283 module_init(kswapd_init)
3284
3285 #ifdef CONFIG_NUMA
3286 /*
3287  * Zone reclaim mode
3288  *
3289  * If non-zero call zone_reclaim when the number of free pages falls below
3290  * the watermarks.
3291  */
3292 int zone_reclaim_mode __read_mostly;
3293
3294 #define RECLAIM_OFF 0
3295 #define RECLAIM_ZONE (1<<0)     /* Run shrink_inactive_list on the zone */
3296 #define RECLAIM_WRITE (1<<1)    /* Writeout pages during reclaim */
3297 #define RECLAIM_SWAP (1<<2)     /* Swap pages out during reclaim */
3298
3299 /*
3300  * Priority for ZONE_RECLAIM. This determines the fraction of pages
3301  * of a node considered for each zone_reclaim. 4 scans 1/16th of
3302  * a zone.
3303  */
3304 #define ZONE_RECLAIM_PRIORITY 4
3305
3306 /*
3307  * Percentage of pages in a zone that must be unmapped for zone_reclaim to
3308  * occur.
3309  */
3310 int sysctl_min_unmapped_ratio = 1;
3311
3312 /*
3313  * If the number of slab pages in a zone grows beyond this percentage then
3314  * slab reclaim needs to occur.
3315  */
3316 int sysctl_min_slab_ratio = 5;
3317
3318 static inline unsigned long zone_unmapped_file_pages(struct zone *zone)
3319 {
3320         unsigned long file_mapped = zone_page_state(zone, NR_FILE_MAPPED);
3321         unsigned long file_lru = zone_page_state(zone, NR_INACTIVE_FILE) +
3322                 zone_page_state(zone, NR_ACTIVE_FILE);
3323
3324         /*
3325          * It's possible for there to be more file mapped pages than
3326          * accounted for by the pages on the file LRU lists because
3327          * tmpfs pages accounted for as ANON can also be FILE_MAPPED
3328          */
3329         return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
3330 }
3331
3332 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
3333 static long zone_pagecache_reclaimable(struct zone *zone)
3334 {
3335         long nr_pagecache_reclaimable;
3336         long delta = 0;
3337
3338         /*
3339          * If RECLAIM_SWAP is set, then all file pages are considered
3340          * potentially reclaimable. Otherwise, we have to worry about
3341          * pages like swapcache and zone_unmapped_file_pages() provides
3342          * a better estimate
3343          */
3344         if (zone_reclaim_mode & RECLAIM_SWAP)
3345                 nr_pagecache_reclaimable = zone_page_state(zone, NR_FILE_PAGES);
3346         else
3347                 nr_pagecache_reclaimable = zone_unmapped_file_pages(zone);
3348
3349         /* If we can't clean pages, remove dirty pages from consideration */
3350         if (!(zone_reclaim_mode & RECLAIM_WRITE))
3351                 delta += zone_page_state(zone, NR_FILE_DIRTY);
3352
3353         /* Watch for any possible underflows due to delta */
3354         if (unlikely(delta > nr_pagecache_reclaimable))
3355                 delta = nr_pagecache_reclaimable;
3356
3357         return nr_pagecache_reclaimable - delta;
3358 }
3359
3360 /*
3361  * Try to free up some pages from this zone through reclaim.
3362  */
3363 static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3364 {
3365         /* Minimum pages needed in order to stay on node */
3366         const unsigned long nr_pages = 1 << order;
3367         struct task_struct *p = current;
3368         struct reclaim_state reclaim_state;
3369         struct scan_control sc = {
3370                 .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
3371                 .may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP),
3372                 .may_swap = 1,
3373                 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3374                 .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
3375                 .order = order,
3376                 .priority = ZONE_RECLAIM_PRIORITY,
3377         };
3378         struct shrink_control shrink = {
3379                 .gfp_mask = sc.gfp_mask,
3380         };
3381         unsigned long nr_slab_pages0, nr_slab_pages1;
3382
3383         cond_resched();
3384         /*
3385          * We need to be able to allocate from the reserves for RECLAIM_SWAP
3386          * and we also need to be able to write out pages for RECLAIM_WRITE
3387          * and RECLAIM_SWAP.
3388          */
3389         p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
3390         lockdep_set_current_reclaim_state(gfp_mask);
3391         reclaim_state.reclaimed_slab = 0;
3392         p->reclaim_state = &reclaim_state;
3393
3394         if (zone_pagecache_reclaimable(zone) > zone->min_unmapped_pages) {
3395                 /*
3396                  * Free memory by calling shrink zone with increasing
3397                  * priorities until we have enough memory freed.
3398                  */
3399                 do {
3400                         shrink_zone(zone, &sc);
3401                 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
3402         }
3403
3404         nr_slab_pages0 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3405         if (nr_slab_pages0 > zone->min_slab_pages) {
3406                 /*
3407                  * shrink_slab() does not currently allow us to determine how
3408                  * many pages were freed in this zone. So we take the current
3409                  * number of slab pages and shake the slab until it is reduced
3410                  * by the same nr_pages that we used for reclaiming unmapped
3411                  * pages.
3412                  *
3413                  * Note that shrink_slab will free memory on all zones and may
3414                  * take a long time.
3415                  */
3416                 for (;;) {
3417                         unsigned long lru_pages = zone_reclaimable_pages(zone);
3418
3419                         /* No reclaimable slab or very low memory pressure */
3420                         if (!shrink_slab(&shrink, sc.nr_scanned, lru_pages))
3421                                 break;
3422
3423                         /* Freed enough memory */
3424                         nr_slab_pages1 = zone_page_state(zone,
3425                                                         NR_SLAB_RECLAIMABLE);
3426                         if (nr_slab_pages1 + nr_pages <= nr_slab_pages0)
3427                                 break;
3428                 }
3429
3430                 /*
3431                  * Update nr_reclaimed by the number of slab pages we
3432                  * reclaimed from this zone.
3433                  */
3434                 nr_slab_pages1 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3435                 if (nr_slab_pages1 < nr_slab_pages0)
3436                         sc.nr_reclaimed += nr_slab_pages0 - nr_slab_pages1;
3437         }
3438
3439         p->reclaim_state = NULL;
3440         current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
3441         lockdep_clear_current_reclaim_state();
3442         return sc.nr_reclaimed >= nr_pages;
3443 }
3444
3445 int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3446 {
3447         int node_id;
3448         int ret;
3449
3450         /*
3451          * Zone reclaim reclaims unmapped file backed pages and
3452          * slab pages if we are over the defined limits.
3453          *
3454          * A small portion of unmapped file backed pages is needed for
3455          * file I/O otherwise pages read by file I/O will be immediately
3456          * thrown out if the zone is overallocated. So we do not reclaim
3457          * if less than a specified percentage of the zone is used by
3458          * unmapped file backed pages.
3459          */
3460         if (zone_pagecache_reclaimable(zone) <= zone->min_unmapped_pages &&
3461             zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
3462                 return ZONE_RECLAIM_FULL;
3463
3464         if (zone->all_unreclaimable)
3465                 return ZONE_RECLAIM_FULL;
3466
3467         /*
3468          * Do not scan if the allocation should not be delayed.
3469          */
3470         if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
3471                 return ZONE_RECLAIM_NOSCAN;
3472
3473         /*
3474          * Only run zone reclaim on the local zone or on zones that do not
3475          * have associated processors. This will favor the local processor
3476          * over remote processors and spread off node memory allocations
3477          * as wide as possible.
3478          */
3479         node_id = zone_to_nid(zone);
3480         if (node_state(node_id, N_CPU) && node_id != numa_node_id())
3481                 return ZONE_RECLAIM_NOSCAN;
3482
3483         if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED))
3484                 return ZONE_RECLAIM_NOSCAN;
3485
3486         ret = __zone_reclaim(zone, gfp_mask, order);
3487         zone_clear_flag(zone, ZONE_RECLAIM_LOCKED);
3488
3489         if (!ret)
3490                 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
3491
3492         return ret;
3493 }
3494 #endif
3495
3496 /*
3497  * page_evictable - test whether a page is evictable
3498  * @page: the page to test
3499  *
3500  * Test whether page is evictable--i.e., should be placed on active/inactive
3501  * lists vs unevictable list.
3502  *
3503  * Reasons page might not be evictable:
3504  * (1) page's mapping marked unevictable
3505  * (2) page is part of an mlocked VMA
3506  *
3507  */
3508 int page_evictable(struct page *page)
3509 {
3510         return !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
3511 }
3512
3513 #ifdef CONFIG_SHMEM
3514 /**
3515  * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
3516  * @pages:      array of pages to check
3517  * @nr_pages:   number of pages to check
3518  *
3519  * Checks pages for evictability and moves them to the appropriate lru list.
3520  *
3521  * This function is only used for SysV IPC SHM_UNLOCK.
3522  */
3523 void check_move_unevictable_pages(struct page **pages, int nr_pages)
3524 {
3525         struct lruvec *lruvec;
3526         struct zone *zone = NULL;
3527         int pgscanned = 0;
3528         int pgrescued = 0;
3529         int i;
3530
3531         for (i = 0; i < nr_pages; i++) {
3532                 struct page *page = pages[i];
3533                 struct zone *pagezone;
3534
3535                 pgscanned++;
3536                 pagezone = page_zone(page);
3537                 if (pagezone != zone) {
3538                         if (zone)
3539                                 spin_unlock_irq(&zone->lru_lock);
3540                         zone = pagezone;
3541                         spin_lock_irq(&zone->lru_lock);
3542                 }
3543                 lruvec = mem_cgroup_page_lruvec(page, zone);
3544
3545                 if (!PageLRU(page) || !PageUnevictable(page))
3546                         continue;
3547
3548                 if (page_evictable(page)) {
3549                         enum lru_list lru = page_lru_base_type(page);
3550
3551                         VM_BUG_ON(PageActive(page));
3552                         ClearPageUnevictable(page);
3553                         del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
3554                         add_page_to_lru_list(page, lruvec, lru);
3555                         pgrescued++;
3556                 }
3557         }
3558
3559         if (zone) {
3560                 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
3561                 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
3562                 spin_unlock_irq(&zone->lru_lock);
3563         }
3564 }
3565 #endif /* CONFIG_SHMEM */
3566
3567 static void warn_scan_unevictable_pages(void)
3568 {
3569         printk_once(KERN_WARNING
3570                     "%s: The scan_unevictable_pages sysctl/node-interface has been "
3571                     "disabled for lack of a legitimate use case.  If you have "
3572                     "one, please send an email to linux-mm@kvack.org.\n",
3573                     current->comm);
3574 }
3575
3576 /*
3577  * scan_unevictable_pages [vm] sysctl handler.  On demand re-scan of
3578  * all nodes' unevictable lists for evictable pages
3579  */
3580 unsigned long scan_unevictable_pages;
3581
3582 int scan_unevictable_handler(struct ctl_table *table, int write,
3583                            void __user *buffer,
3584                            size_t *length, loff_t *ppos)
3585 {
3586         warn_scan_unevictable_pages();
3587         proc_doulongvec_minmax(table, write, buffer, length, ppos);
3588         scan_unevictable_pages = 0;
3589         return 0;
3590 }
3591
3592 #ifdef CONFIG_NUMA
3593 /*
3594  * per node 'scan_unevictable_pages' attribute.  On demand re-scan of
3595  * a specified node's per zone unevictable lists for evictable pages.
3596  */
3597
3598 static ssize_t read_scan_unevictable_node(struct device *dev,
3599                                           struct device_attribute *attr,
3600                                           char *buf)
3601 {
3602         warn_scan_unevictable_pages();
3603         return sprintf(buf, "0\n");     /* always zero; should fit... */
3604 }
3605
3606 static ssize_t write_scan_unevictable_node(struct device *dev,
3607                                            struct device_attribute *attr,
3608                                         const char *buf, size_t count)
3609 {
3610         warn_scan_unevictable_pages();
3611         return 1;
3612 }
3613
3614
3615 static DEVICE_ATTR(scan_unevictable_pages, S_IRUGO | S_IWUSR,
3616                         read_scan_unevictable_node,
3617                         write_scan_unevictable_node);
3618
3619 int scan_unevictable_register_node(struct node *node)
3620 {
3621         return device_create_file(&node->dev, &dev_attr_scan_unevictable_pages);
3622 }
3623
3624 void scan_unevictable_unregister_node(struct node *node)
3625 {
3626         device_remove_file(&node->dev, &dev_attr_scan_unevictable_pages);
3627 }
3628 #endif