34eff4dfae8067d5a03ac97ffbe331958154b01b
[platform/kernel/linux-starfive.git] / mm / khugepaged.c
1 // SPDX-License-Identifier: GPL-2.0
2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3
4 #include <linux/mm.h>
5 #include <linux/sched.h>
6 #include <linux/sched/mm.h>
7 #include <linux/sched/coredump.h>
8 #include <linux/mmu_notifier.h>
9 #include <linux/rmap.h>
10 #include <linux/swap.h>
11 #include <linux/mm_inline.h>
12 #include <linux/kthread.h>
13 #include <linux/khugepaged.h>
14 #include <linux/freezer.h>
15 #include <linux/mman.h>
16 #include <linux/hashtable.h>
17 #include <linux/userfaultfd_k.h>
18 #include <linux/page_idle.h>
19 #include <linux/swapops.h>
20 #include <linux/shmem_fs.h>
21
22 #include <asm/tlb.h>
23 #include <asm/pgalloc.h>
24 #include "internal.h"
25
26 enum scan_result {
27         SCAN_FAIL,
28         SCAN_SUCCEED,
29         SCAN_PMD_NULL,
30         SCAN_EXCEED_NONE_PTE,
31         SCAN_EXCEED_SWAP_PTE,
32         SCAN_EXCEED_SHARED_PTE,
33         SCAN_PTE_NON_PRESENT,
34         SCAN_PTE_UFFD_WP,
35         SCAN_PAGE_RO,
36         SCAN_LACK_REFERENCED_PAGE,
37         SCAN_PAGE_NULL,
38         SCAN_SCAN_ABORT,
39         SCAN_PAGE_COUNT,
40         SCAN_PAGE_LRU,
41         SCAN_PAGE_LOCK,
42         SCAN_PAGE_ANON,
43         SCAN_PAGE_COMPOUND,
44         SCAN_ANY_PROCESS,
45         SCAN_VMA_NULL,
46         SCAN_VMA_CHECK,
47         SCAN_ADDRESS_RANGE,
48         SCAN_SWAP_CACHE_PAGE,
49         SCAN_DEL_PAGE_LRU,
50         SCAN_ALLOC_HUGE_PAGE_FAIL,
51         SCAN_CGROUP_CHARGE_FAIL,
52         SCAN_TRUNCATED,
53         SCAN_PAGE_HAS_PRIVATE,
54 };
55
56 #define CREATE_TRACE_POINTS
57 #include <trace/events/huge_memory.h>
58
59 /* default scan 8*512 pte (or vmas) every 30 second */
60 static unsigned int khugepaged_pages_to_scan __read_mostly;
61 static unsigned int khugepaged_pages_collapsed;
62 static unsigned int khugepaged_full_scans;
63 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
64 /* during fragmentation poll the hugepage allocator once every minute */
65 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
66 static unsigned long khugepaged_sleep_expire;
67 static DEFINE_SPINLOCK(khugepaged_mm_lock);
68 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
69 /*
70  * default collapse hugepages if there is at least one pte mapped like
71  * it would have happened if the vma was large enough during page
72  * fault.
73  */
74 static unsigned int khugepaged_max_ptes_none __read_mostly;
75 static unsigned int khugepaged_max_ptes_swap __read_mostly;
76 static unsigned int khugepaged_max_ptes_shared __read_mostly;
77
78 #define MM_SLOTS_HASH_BITS 10
79 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
80
81 static struct kmem_cache *mm_slot_cache __read_mostly;
82
83 #define MAX_PTE_MAPPED_THP 8
84
85 /**
86  * struct mm_slot - hash lookup from mm to mm_slot
87  * @hash: hash collision list
88  * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
89  * @mm: the mm that this information is valid for
90  */
91 struct mm_slot {
92         struct hlist_node hash;
93         struct list_head mm_node;
94         struct mm_struct *mm;
95
96         /* pte-mapped THP in this mm */
97         int nr_pte_mapped_thp;
98         unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP];
99 };
100
101 /**
102  * struct khugepaged_scan - cursor for scanning
103  * @mm_head: the head of the mm list to scan
104  * @mm_slot: the current mm_slot we are scanning
105  * @address: the next address inside that to be scanned
106  *
107  * There is only the one khugepaged_scan instance of this cursor structure.
108  */
109 struct khugepaged_scan {
110         struct list_head mm_head;
111         struct mm_slot *mm_slot;
112         unsigned long address;
113 };
114
115 static struct khugepaged_scan khugepaged_scan = {
116         .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
117 };
118
119 #ifdef CONFIG_SYSFS
120 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
121                                          struct kobj_attribute *attr,
122                                          char *buf)
123 {
124         return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
125 }
126
127 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
128                                           struct kobj_attribute *attr,
129                                           const char *buf, size_t count)
130 {
131         unsigned long msecs;
132         int err;
133
134         err = kstrtoul(buf, 10, &msecs);
135         if (err || msecs > UINT_MAX)
136                 return -EINVAL;
137
138         khugepaged_scan_sleep_millisecs = msecs;
139         khugepaged_sleep_expire = 0;
140         wake_up_interruptible(&khugepaged_wait);
141
142         return count;
143 }
144 static struct kobj_attribute scan_sleep_millisecs_attr =
145         __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
146                scan_sleep_millisecs_store);
147
148 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
149                                           struct kobj_attribute *attr,
150                                           char *buf)
151 {
152         return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
153 }
154
155 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
156                                            struct kobj_attribute *attr,
157                                            const char *buf, size_t count)
158 {
159         unsigned long msecs;
160         int err;
161
162         err = kstrtoul(buf, 10, &msecs);
163         if (err || msecs > UINT_MAX)
164                 return -EINVAL;
165
166         khugepaged_alloc_sleep_millisecs = msecs;
167         khugepaged_sleep_expire = 0;
168         wake_up_interruptible(&khugepaged_wait);
169
170         return count;
171 }
172 static struct kobj_attribute alloc_sleep_millisecs_attr =
173         __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
174                alloc_sleep_millisecs_store);
175
176 static ssize_t pages_to_scan_show(struct kobject *kobj,
177                                   struct kobj_attribute *attr,
178                                   char *buf)
179 {
180         return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
181 }
182 static ssize_t pages_to_scan_store(struct kobject *kobj,
183                                    struct kobj_attribute *attr,
184                                    const char *buf, size_t count)
185 {
186         int err;
187         unsigned long pages;
188
189         err = kstrtoul(buf, 10, &pages);
190         if (err || !pages || pages > UINT_MAX)
191                 return -EINVAL;
192
193         khugepaged_pages_to_scan = pages;
194
195         return count;
196 }
197 static struct kobj_attribute pages_to_scan_attr =
198         __ATTR(pages_to_scan, 0644, pages_to_scan_show,
199                pages_to_scan_store);
200
201 static ssize_t pages_collapsed_show(struct kobject *kobj,
202                                     struct kobj_attribute *attr,
203                                     char *buf)
204 {
205         return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
206 }
207 static struct kobj_attribute pages_collapsed_attr =
208         __ATTR_RO(pages_collapsed);
209
210 static ssize_t full_scans_show(struct kobject *kobj,
211                                struct kobj_attribute *attr,
212                                char *buf)
213 {
214         return sprintf(buf, "%u\n", khugepaged_full_scans);
215 }
216 static struct kobj_attribute full_scans_attr =
217         __ATTR_RO(full_scans);
218
219 static ssize_t khugepaged_defrag_show(struct kobject *kobj,
220                                       struct kobj_attribute *attr, char *buf)
221 {
222         return single_hugepage_flag_show(kobj, attr, buf,
223                                 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
224 }
225 static ssize_t khugepaged_defrag_store(struct kobject *kobj,
226                                        struct kobj_attribute *attr,
227                                        const char *buf, size_t count)
228 {
229         return single_hugepage_flag_store(kobj, attr, buf, count,
230                                  TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
231 }
232 static struct kobj_attribute khugepaged_defrag_attr =
233         __ATTR(defrag, 0644, khugepaged_defrag_show,
234                khugepaged_defrag_store);
235
236 /*
237  * max_ptes_none controls if khugepaged should collapse hugepages over
238  * any unmapped ptes in turn potentially increasing the memory
239  * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
240  * reduce the available free memory in the system as it
241  * runs. Increasing max_ptes_none will instead potentially reduce the
242  * free memory in the system during the khugepaged scan.
243  */
244 static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
245                                              struct kobj_attribute *attr,
246                                              char *buf)
247 {
248         return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
249 }
250 static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
251                                               struct kobj_attribute *attr,
252                                               const char *buf, size_t count)
253 {
254         int err;
255         unsigned long max_ptes_none;
256
257         err = kstrtoul(buf, 10, &max_ptes_none);
258         if (err || max_ptes_none > HPAGE_PMD_NR-1)
259                 return -EINVAL;
260
261         khugepaged_max_ptes_none = max_ptes_none;
262
263         return count;
264 }
265 static struct kobj_attribute khugepaged_max_ptes_none_attr =
266         __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
267                khugepaged_max_ptes_none_store);
268
269 static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj,
270                                              struct kobj_attribute *attr,
271                                              char *buf)
272 {
273         return sprintf(buf, "%u\n", khugepaged_max_ptes_swap);
274 }
275
276 static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj,
277                                               struct kobj_attribute *attr,
278                                               const char *buf, size_t count)
279 {
280         int err;
281         unsigned long max_ptes_swap;
282
283         err  = kstrtoul(buf, 10, &max_ptes_swap);
284         if (err || max_ptes_swap > HPAGE_PMD_NR-1)
285                 return -EINVAL;
286
287         khugepaged_max_ptes_swap = max_ptes_swap;
288
289         return count;
290 }
291
292 static struct kobj_attribute khugepaged_max_ptes_swap_attr =
293         __ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show,
294                khugepaged_max_ptes_swap_store);
295
296 static ssize_t khugepaged_max_ptes_shared_show(struct kobject *kobj,
297                                              struct kobj_attribute *attr,
298                                              char *buf)
299 {
300         return sprintf(buf, "%u\n", khugepaged_max_ptes_shared);
301 }
302
303 static ssize_t khugepaged_max_ptes_shared_store(struct kobject *kobj,
304                                               struct kobj_attribute *attr,
305                                               const char *buf, size_t count)
306 {
307         int err;
308         unsigned long max_ptes_shared;
309
310         err  = kstrtoul(buf, 10, &max_ptes_shared);
311         if (err || max_ptes_shared > HPAGE_PMD_NR-1)
312                 return -EINVAL;
313
314         khugepaged_max_ptes_shared = max_ptes_shared;
315
316         return count;
317 }
318
319 static struct kobj_attribute khugepaged_max_ptes_shared_attr =
320         __ATTR(max_ptes_shared, 0644, khugepaged_max_ptes_shared_show,
321                khugepaged_max_ptes_shared_store);
322
323 static struct attribute *khugepaged_attr[] = {
324         &khugepaged_defrag_attr.attr,
325         &khugepaged_max_ptes_none_attr.attr,
326         &khugepaged_max_ptes_swap_attr.attr,
327         &khugepaged_max_ptes_shared_attr.attr,
328         &pages_to_scan_attr.attr,
329         &pages_collapsed_attr.attr,
330         &full_scans_attr.attr,
331         &scan_sleep_millisecs_attr.attr,
332         &alloc_sleep_millisecs_attr.attr,
333         NULL,
334 };
335
336 struct attribute_group khugepaged_attr_group = {
337         .attrs = khugepaged_attr,
338         .name = "khugepaged",
339 };
340 #endif /* CONFIG_SYSFS */
341
342 int hugepage_madvise(struct vm_area_struct *vma,
343                      unsigned long *vm_flags, int advice)
344 {
345         switch (advice) {
346         case MADV_HUGEPAGE:
347 #ifdef CONFIG_S390
348                 /*
349                  * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
350                  * can't handle this properly after s390_enable_sie, so we simply
351                  * ignore the madvise to prevent qemu from causing a SIGSEGV.
352                  */
353                 if (mm_has_pgste(vma->vm_mm))
354                         return 0;
355 #endif
356                 *vm_flags &= ~VM_NOHUGEPAGE;
357                 *vm_flags |= VM_HUGEPAGE;
358                 /*
359                  * If the vma become good for khugepaged to scan,
360                  * register it here without waiting a page fault that
361                  * may not happen any time soon.
362                  */
363                 if (!(*vm_flags & VM_NO_KHUGEPAGED) &&
364                                 khugepaged_enter_vma_merge(vma, *vm_flags))
365                         return -ENOMEM;
366                 break;
367         case MADV_NOHUGEPAGE:
368                 *vm_flags &= ~VM_HUGEPAGE;
369                 *vm_flags |= VM_NOHUGEPAGE;
370                 /*
371                  * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
372                  * this vma even if we leave the mm registered in khugepaged if
373                  * it got registered before VM_NOHUGEPAGE was set.
374                  */
375                 break;
376         }
377
378         return 0;
379 }
380
381 int __init khugepaged_init(void)
382 {
383         mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
384                                           sizeof(struct mm_slot),
385                                           __alignof__(struct mm_slot), 0, NULL);
386         if (!mm_slot_cache)
387                 return -ENOMEM;
388
389         khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
390         khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
391         khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
392         khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2;
393
394         return 0;
395 }
396
397 void __init khugepaged_destroy(void)
398 {
399         kmem_cache_destroy(mm_slot_cache);
400 }
401
402 static inline struct mm_slot *alloc_mm_slot(void)
403 {
404         if (!mm_slot_cache)     /* initialization failed */
405                 return NULL;
406         return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
407 }
408
409 static inline void free_mm_slot(struct mm_slot *mm_slot)
410 {
411         kmem_cache_free(mm_slot_cache, mm_slot);
412 }
413
414 static struct mm_slot *get_mm_slot(struct mm_struct *mm)
415 {
416         struct mm_slot *mm_slot;
417
418         hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
419                 if (mm == mm_slot->mm)
420                         return mm_slot;
421
422         return NULL;
423 }
424
425 static void insert_to_mm_slots_hash(struct mm_struct *mm,
426                                     struct mm_slot *mm_slot)
427 {
428         mm_slot->mm = mm;
429         hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
430 }
431
432 static inline int khugepaged_test_exit(struct mm_struct *mm)
433 {
434         return atomic_read(&mm->mm_users) == 0;
435 }
436
437 static bool hugepage_vma_check(struct vm_area_struct *vma,
438                                unsigned long vm_flags)
439 {
440         if ((!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
441             (vm_flags & VM_NOHUGEPAGE) ||
442             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
443                 return false;
444
445         if (shmem_file(vma->vm_file) ||
446             (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
447              vma->vm_file &&
448              (vm_flags & VM_DENYWRITE))) {
449                 return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
450                                 HPAGE_PMD_NR);
451         }
452         if (!vma->anon_vma || vma->vm_ops)
453                 return false;
454         if (vma_is_temporary_stack(vma))
455                 return false;
456         return !(vm_flags & VM_NO_KHUGEPAGED);
457 }
458
459 int __khugepaged_enter(struct mm_struct *mm)
460 {
461         struct mm_slot *mm_slot;
462         int wakeup;
463
464         mm_slot = alloc_mm_slot();
465         if (!mm_slot)
466                 return -ENOMEM;
467
468         /* __khugepaged_exit() must not run from under us */
469         VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
470         if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
471                 free_mm_slot(mm_slot);
472                 return 0;
473         }
474
475         spin_lock(&khugepaged_mm_lock);
476         insert_to_mm_slots_hash(mm, mm_slot);
477         /*
478          * Insert just behind the scanning cursor, to let the area settle
479          * down a little.
480          */
481         wakeup = list_empty(&khugepaged_scan.mm_head);
482         list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
483         spin_unlock(&khugepaged_mm_lock);
484
485         mmgrab(mm);
486         if (wakeup)
487                 wake_up_interruptible(&khugepaged_wait);
488
489         return 0;
490 }
491
492 int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
493                                unsigned long vm_flags)
494 {
495         unsigned long hstart, hend;
496
497         /*
498          * khugepaged only supports read-only files for non-shmem files.
499          * khugepaged does not yet work on special mappings. And
500          * file-private shmem THP is not supported.
501          */
502         if (!hugepage_vma_check(vma, vm_flags))
503                 return 0;
504
505         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
506         hend = vma->vm_end & HPAGE_PMD_MASK;
507         if (hstart < hend)
508                 return khugepaged_enter(vma, vm_flags);
509         return 0;
510 }
511
512 void __khugepaged_exit(struct mm_struct *mm)
513 {
514         struct mm_slot *mm_slot;
515         int free = 0;
516
517         spin_lock(&khugepaged_mm_lock);
518         mm_slot = get_mm_slot(mm);
519         if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
520                 hash_del(&mm_slot->hash);
521                 list_del(&mm_slot->mm_node);
522                 free = 1;
523         }
524         spin_unlock(&khugepaged_mm_lock);
525
526         if (free) {
527                 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
528                 free_mm_slot(mm_slot);
529                 mmdrop(mm);
530         } else if (mm_slot) {
531                 /*
532                  * This is required to serialize against
533                  * khugepaged_test_exit() (which is guaranteed to run
534                  * under mmap sem read mode). Stop here (after we
535                  * return all pagetables will be destroyed) until
536                  * khugepaged has finished working on the pagetables
537                  * under the mmap_sem.
538                  */
539                 down_write(&mm->mmap_sem);
540                 up_write(&mm->mmap_sem);
541         }
542 }
543
544 static void release_pte_page(struct page *page)
545 {
546         mod_node_page_state(page_pgdat(page),
547                         NR_ISOLATED_ANON + page_is_file_lru(page),
548                         -compound_nr(page));
549         unlock_page(page);
550         putback_lru_page(page);
551 }
552
553 static void release_pte_pages(pte_t *pte, pte_t *_pte,
554                 struct list_head *compound_pagelist)
555 {
556         struct page *page, *tmp;
557
558         while (--_pte >= pte) {
559                 pte_t pteval = *_pte;
560
561                 page = pte_page(pteval);
562                 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
563                                 !PageCompound(page))
564                         release_pte_page(page);
565         }
566
567         list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
568                 list_del(&page->lru);
569                 release_pte_page(page);
570         }
571 }
572
573 static bool is_refcount_suitable(struct page *page)
574 {
575         int expected_refcount;
576
577         expected_refcount = total_mapcount(page);
578         if (PageSwapCache(page))
579                 expected_refcount += compound_nr(page);
580
581         return page_count(page) == expected_refcount;
582 }
583
584 static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
585                                         unsigned long address,
586                                         pte_t *pte,
587                                         struct list_head *compound_pagelist)
588 {
589         struct page *page = NULL;
590         pte_t *_pte;
591         int none_or_zero = 0, shared = 0, result = 0, referenced = 0;
592         bool writable = false;
593
594         for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
595              _pte++, address += PAGE_SIZE) {
596                 pte_t pteval = *_pte;
597                 if (pte_none(pteval) || (pte_present(pteval) &&
598                                 is_zero_pfn(pte_pfn(pteval)))) {
599                         if (!userfaultfd_armed(vma) &&
600                             ++none_or_zero <= khugepaged_max_ptes_none) {
601                                 continue;
602                         } else {
603                                 result = SCAN_EXCEED_NONE_PTE;
604                                 goto out;
605                         }
606                 }
607                 if (!pte_present(pteval)) {
608                         result = SCAN_PTE_NON_PRESENT;
609                         goto out;
610                 }
611                 page = vm_normal_page(vma, address, pteval);
612                 if (unlikely(!page)) {
613                         result = SCAN_PAGE_NULL;
614                         goto out;
615                 }
616
617                 VM_BUG_ON_PAGE(!PageAnon(page), page);
618
619                 if (page_mapcount(page) > 1 &&
620                                 ++shared > khugepaged_max_ptes_shared) {
621                         result = SCAN_EXCEED_SHARED_PTE;
622                         goto out;
623                 }
624
625                 if (PageCompound(page)) {
626                         struct page *p;
627                         page = compound_head(page);
628
629                         /*
630                          * Check if we have dealt with the compound page
631                          * already
632                          */
633                         list_for_each_entry(p, compound_pagelist, lru) {
634                                 if (page == p)
635                                         goto next;
636                         }
637                 }
638
639                 /*
640                  * We can do it before isolate_lru_page because the
641                  * page can't be freed from under us. NOTE: PG_lock
642                  * is needed to serialize against split_huge_page
643                  * when invoked from the VM.
644                  */
645                 if (!trylock_page(page)) {
646                         result = SCAN_PAGE_LOCK;
647                         goto out;
648                 }
649
650                 /*
651                  * Check if the page has any GUP (or other external) pins.
652                  *
653                  * The page table that maps the page has been already unlinked
654                  * from the page table tree and this process cannot get
655                  * an additinal pin on the page.
656                  *
657                  * New pins can come later if the page is shared across fork,
658                  * but not from this process. The other process cannot write to
659                  * the page, only trigger CoW.
660                  */
661                 if (!is_refcount_suitable(page)) {
662                         unlock_page(page);
663                         result = SCAN_PAGE_COUNT;
664                         goto out;
665                 }
666                 if (!pte_write(pteval) && PageSwapCache(page) &&
667                                 !reuse_swap_page(page, NULL)) {
668                         /*
669                          * Page is in the swap cache and cannot be re-used.
670                          * It cannot be collapsed into a THP.
671                          */
672                         unlock_page(page);
673                         result = SCAN_SWAP_CACHE_PAGE;
674                         goto out;
675                 }
676
677                 /*
678                  * Isolate the page to avoid collapsing an hugepage
679                  * currently in use by the VM.
680                  */
681                 if (isolate_lru_page(page)) {
682                         unlock_page(page);
683                         result = SCAN_DEL_PAGE_LRU;
684                         goto out;
685                 }
686                 mod_node_page_state(page_pgdat(page),
687                                 NR_ISOLATED_ANON + page_is_file_lru(page),
688                                 compound_nr(page));
689                 VM_BUG_ON_PAGE(!PageLocked(page), page);
690                 VM_BUG_ON_PAGE(PageLRU(page), page);
691
692                 if (PageCompound(page))
693                         list_add_tail(&page->lru, compound_pagelist);
694 next:
695                 /* There should be enough young pte to collapse the page */
696                 if (pte_young(pteval) ||
697                     page_is_young(page) || PageReferenced(page) ||
698                     mmu_notifier_test_young(vma->vm_mm, address))
699                         referenced++;
700
701                 if (pte_write(pteval))
702                         writable = true;
703         }
704         if (likely(writable)) {
705                 if (likely(referenced)) {
706                         result = SCAN_SUCCEED;
707                         trace_mm_collapse_huge_page_isolate(page, none_or_zero,
708                                                             referenced, writable, result);
709                         return 1;
710                 }
711         } else {
712                 result = SCAN_PAGE_RO;
713         }
714
715 out:
716         release_pte_pages(pte, _pte, compound_pagelist);
717         trace_mm_collapse_huge_page_isolate(page, none_or_zero,
718                                             referenced, writable, result);
719         return 0;
720 }
721
722 static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
723                                       struct vm_area_struct *vma,
724                                       unsigned long address,
725                                       spinlock_t *ptl,
726                                       struct list_head *compound_pagelist)
727 {
728         struct page *src_page, *tmp;
729         pte_t *_pte;
730         for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
731                                 _pte++, page++, address += PAGE_SIZE) {
732                 pte_t pteval = *_pte;
733
734                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
735                         clear_user_highpage(page, address);
736                         add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
737                         if (is_zero_pfn(pte_pfn(pteval))) {
738                                 /*
739                                  * ptl mostly unnecessary.
740                                  */
741                                 spin_lock(ptl);
742                                 /*
743                                  * paravirt calls inside pte_clear here are
744                                  * superfluous.
745                                  */
746                                 pte_clear(vma->vm_mm, address, _pte);
747                                 spin_unlock(ptl);
748                         }
749                 } else {
750                         src_page = pte_page(pteval);
751                         copy_user_highpage(page, src_page, address, vma);
752                         if (!PageCompound(src_page))
753                                 release_pte_page(src_page);
754                         /*
755                          * ptl mostly unnecessary, but preempt has to
756                          * be disabled to update the per-cpu stats
757                          * inside page_remove_rmap().
758                          */
759                         spin_lock(ptl);
760                         /*
761                          * paravirt calls inside pte_clear here are
762                          * superfluous.
763                          */
764                         pte_clear(vma->vm_mm, address, _pte);
765                         page_remove_rmap(src_page, false);
766                         spin_unlock(ptl);
767                         free_page_and_swap_cache(src_page);
768                 }
769         }
770
771         list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) {
772                 list_del(&src_page->lru);
773                 release_pte_page(src_page);
774         }
775 }
776
777 static void khugepaged_alloc_sleep(void)
778 {
779         DEFINE_WAIT(wait);
780
781         add_wait_queue(&khugepaged_wait, &wait);
782         freezable_schedule_timeout_interruptible(
783                 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
784         remove_wait_queue(&khugepaged_wait, &wait);
785 }
786
787 static int khugepaged_node_load[MAX_NUMNODES];
788
789 static bool khugepaged_scan_abort(int nid)
790 {
791         int i;
792
793         /*
794          * If node_reclaim_mode is disabled, then no extra effort is made to
795          * allocate memory locally.
796          */
797         if (!node_reclaim_mode)
798                 return false;
799
800         /* If there is a count for this node already, it must be acceptable */
801         if (khugepaged_node_load[nid])
802                 return false;
803
804         for (i = 0; i < MAX_NUMNODES; i++) {
805                 if (!khugepaged_node_load[i])
806                         continue;
807                 if (node_distance(nid, i) > node_reclaim_distance)
808                         return true;
809         }
810         return false;
811 }
812
813 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
814 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
815 {
816         return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;
817 }
818
819 #ifdef CONFIG_NUMA
820 static int khugepaged_find_target_node(void)
821 {
822         static int last_khugepaged_target_node = NUMA_NO_NODE;
823         int nid, target_node = 0, max_value = 0;
824
825         /* find first node with max normal pages hit */
826         for (nid = 0; nid < MAX_NUMNODES; nid++)
827                 if (khugepaged_node_load[nid] > max_value) {
828                         max_value = khugepaged_node_load[nid];
829                         target_node = nid;
830                 }
831
832         /* do some balance if several nodes have the same hit record */
833         if (target_node <= last_khugepaged_target_node)
834                 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
835                                 nid++)
836                         if (max_value == khugepaged_node_load[nid]) {
837                                 target_node = nid;
838                                 break;
839                         }
840
841         last_khugepaged_target_node = target_node;
842         return target_node;
843 }
844
845 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
846 {
847         if (IS_ERR(*hpage)) {
848                 if (!*wait)
849                         return false;
850
851                 *wait = false;
852                 *hpage = NULL;
853                 khugepaged_alloc_sleep();
854         } else if (*hpage) {
855                 put_page(*hpage);
856                 *hpage = NULL;
857         }
858
859         return true;
860 }
861
862 static struct page *
863 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
864 {
865         VM_BUG_ON_PAGE(*hpage, *hpage);
866
867         *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
868         if (unlikely(!*hpage)) {
869                 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
870                 *hpage = ERR_PTR(-ENOMEM);
871                 return NULL;
872         }
873
874         prep_transhuge_page(*hpage);
875         count_vm_event(THP_COLLAPSE_ALLOC);
876         return *hpage;
877 }
878 #else
879 static int khugepaged_find_target_node(void)
880 {
881         return 0;
882 }
883
884 static inline struct page *alloc_khugepaged_hugepage(void)
885 {
886         struct page *page;
887
888         page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
889                            HPAGE_PMD_ORDER);
890         if (page)
891                 prep_transhuge_page(page);
892         return page;
893 }
894
895 static struct page *khugepaged_alloc_hugepage(bool *wait)
896 {
897         struct page *hpage;
898
899         do {
900                 hpage = alloc_khugepaged_hugepage();
901                 if (!hpage) {
902                         count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
903                         if (!*wait)
904                                 return NULL;
905
906                         *wait = false;
907                         khugepaged_alloc_sleep();
908                 } else
909                         count_vm_event(THP_COLLAPSE_ALLOC);
910         } while (unlikely(!hpage) && likely(khugepaged_enabled()));
911
912         return hpage;
913 }
914
915 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
916 {
917         if (!*hpage)
918                 *hpage = khugepaged_alloc_hugepage(wait);
919
920         if (unlikely(!*hpage))
921                 return false;
922
923         return true;
924 }
925
926 static struct page *
927 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
928 {
929         VM_BUG_ON(!*hpage);
930
931         return  *hpage;
932 }
933 #endif
934
935 /*
936  * If mmap_sem temporarily dropped, revalidate vma
937  * before taking mmap_sem.
938  * Return 0 if succeeds, otherwise return none-zero
939  * value (scan code).
940  */
941
942 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
943                 struct vm_area_struct **vmap)
944 {
945         struct vm_area_struct *vma;
946         unsigned long hstart, hend;
947
948         if (unlikely(khugepaged_test_exit(mm)))
949                 return SCAN_ANY_PROCESS;
950
951         *vmap = vma = find_vma(mm, address);
952         if (!vma)
953                 return SCAN_VMA_NULL;
954
955         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
956         hend = vma->vm_end & HPAGE_PMD_MASK;
957         if (address < hstart || address + HPAGE_PMD_SIZE > hend)
958                 return SCAN_ADDRESS_RANGE;
959         if (!hugepage_vma_check(vma, vma->vm_flags))
960                 return SCAN_VMA_CHECK;
961         return 0;
962 }
963
964 /*
965  * Bring missing pages in from swap, to complete THP collapse.
966  * Only done if khugepaged_scan_pmd believes it is worthwhile.
967  *
968  * Called and returns without pte mapped or spinlocks held,
969  * but with mmap_sem held to protect against vma changes.
970  */
971
972 static bool __collapse_huge_page_swapin(struct mm_struct *mm,
973                                         struct vm_area_struct *vma,
974                                         unsigned long address, pmd_t *pmd,
975                                         int referenced)
976 {
977         int swapped_in = 0;
978         vm_fault_t ret = 0;
979         struct vm_fault vmf = {
980                 .vma = vma,
981                 .address = address,
982                 .flags = FAULT_FLAG_ALLOW_RETRY,
983                 .pmd = pmd,
984                 .pgoff = linear_page_index(vma, address),
985         };
986
987         vmf.pte = pte_offset_map(pmd, address);
988         for (; vmf.address < address + HPAGE_PMD_NR*PAGE_SIZE;
989                         vmf.pte++, vmf.address += PAGE_SIZE) {
990                 vmf.orig_pte = *vmf.pte;
991                 if (!is_swap_pte(vmf.orig_pte))
992                         continue;
993                 swapped_in++;
994                 ret = do_swap_page(&vmf);
995
996                 /* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
997                 if (ret & VM_FAULT_RETRY) {
998                         down_read(&mm->mmap_sem);
999                         if (hugepage_vma_revalidate(mm, address, &vmf.vma)) {
1000                                 /* vma is no longer available, don't continue to swapin */
1001                                 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
1002                                 return false;
1003                         }
1004                         /* check if the pmd is still valid */
1005                         if (mm_find_pmd(mm, address) != pmd) {
1006                                 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
1007                                 return false;
1008                         }
1009                 }
1010                 if (ret & VM_FAULT_ERROR) {
1011                         trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
1012                         return false;
1013                 }
1014                 /* pte is unmapped now, we need to map it */
1015                 vmf.pte = pte_offset_map(pmd, vmf.address);
1016         }
1017         vmf.pte--;
1018         pte_unmap(vmf.pte);
1019
1020         /* Drain LRU add pagevec to remove extra pin on the swapped in pages */
1021         if (swapped_in)
1022                 lru_add_drain();
1023
1024         trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
1025         return true;
1026 }
1027
1028 static void collapse_huge_page(struct mm_struct *mm,
1029                                    unsigned long address,
1030                                    struct page **hpage,
1031                                    int node, int referenced, int unmapped)
1032 {
1033         LIST_HEAD(compound_pagelist);
1034         pmd_t *pmd, _pmd;
1035         pte_t *pte;
1036         pgtable_t pgtable;
1037         struct page *new_page;
1038         spinlock_t *pmd_ptl, *pte_ptl;
1039         int isolated = 0, result = 0;
1040         struct mem_cgroup *memcg;
1041         struct vm_area_struct *vma;
1042         struct mmu_notifier_range range;
1043         gfp_t gfp;
1044
1045         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1046
1047         /* Only allocate from the target node */
1048         gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
1049
1050         /*
1051          * Before allocating the hugepage, release the mmap_sem read lock.
1052          * The allocation can take potentially a long time if it involves
1053          * sync compaction, and we do not need to hold the mmap_sem during
1054          * that. We will recheck the vma after taking it again in write mode.
1055          */
1056         up_read(&mm->mmap_sem);
1057         new_page = khugepaged_alloc_page(hpage, gfp, node);
1058         if (!new_page) {
1059                 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
1060                 goto out_nolock;
1061         }
1062
1063         if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg))) {
1064                 result = SCAN_CGROUP_CHARGE_FAIL;
1065                 goto out_nolock;
1066         }
1067
1068         down_read(&mm->mmap_sem);
1069         result = hugepage_vma_revalidate(mm, address, &vma);
1070         if (result) {
1071                 mem_cgroup_cancel_charge(new_page, memcg);
1072                 up_read(&mm->mmap_sem);
1073                 goto out_nolock;
1074         }
1075
1076         pmd = mm_find_pmd(mm, address);
1077         if (!pmd) {
1078                 result = SCAN_PMD_NULL;
1079                 mem_cgroup_cancel_charge(new_page, memcg);
1080                 up_read(&mm->mmap_sem);
1081                 goto out_nolock;
1082         }
1083
1084         /*
1085          * __collapse_huge_page_swapin always returns with mmap_sem locked.
1086          * If it fails, we release mmap_sem and jump out_nolock.
1087          * Continuing to collapse causes inconsistency.
1088          */
1089         if (unmapped && !__collapse_huge_page_swapin(mm, vma, address,
1090                                                      pmd, referenced)) {
1091                 mem_cgroup_cancel_charge(new_page, memcg);
1092                 up_read(&mm->mmap_sem);
1093                 goto out_nolock;
1094         }
1095
1096         up_read(&mm->mmap_sem);
1097         /*
1098          * Prevent all access to pagetables with the exception of
1099          * gup_fast later handled by the ptep_clear_flush and the VM
1100          * handled by the anon_vma lock + PG_lock.
1101          */
1102         down_write(&mm->mmap_sem);
1103         result = SCAN_ANY_PROCESS;
1104         if (!mmget_still_valid(mm))
1105                 goto out;
1106         result = hugepage_vma_revalidate(mm, address, &vma);
1107         if (result)
1108                 goto out;
1109         /* check if the pmd is still valid */
1110         if (mm_find_pmd(mm, address) != pmd)
1111                 goto out;
1112
1113         anon_vma_lock_write(vma->anon_vma);
1114
1115         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm,
1116                                 address, address + HPAGE_PMD_SIZE);
1117         mmu_notifier_invalidate_range_start(&range);
1118
1119         pte = pte_offset_map(pmd, address);
1120         pte_ptl = pte_lockptr(mm, pmd);
1121
1122         pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
1123         /*
1124          * After this gup_fast can't run anymore. This also removes
1125          * any huge TLB entry from the CPU so we won't allow
1126          * huge and small TLB entries for the same virtual address
1127          * to avoid the risk of CPU bugs in that area.
1128          */
1129         _pmd = pmdp_collapse_flush(vma, address, pmd);
1130         spin_unlock(pmd_ptl);
1131         mmu_notifier_invalidate_range_end(&range);
1132
1133         spin_lock(pte_ptl);
1134         isolated = __collapse_huge_page_isolate(vma, address, pte,
1135                         &compound_pagelist);
1136         spin_unlock(pte_ptl);
1137
1138         if (unlikely(!isolated)) {
1139                 pte_unmap(pte);
1140                 spin_lock(pmd_ptl);
1141                 BUG_ON(!pmd_none(*pmd));
1142                 /*
1143                  * We can only use set_pmd_at when establishing
1144                  * hugepmds and never for establishing regular pmds that
1145                  * points to regular pagetables. Use pmd_populate for that
1146                  */
1147                 pmd_populate(mm, pmd, pmd_pgtable(_pmd));
1148                 spin_unlock(pmd_ptl);
1149                 anon_vma_unlock_write(vma->anon_vma);
1150                 result = SCAN_FAIL;
1151                 goto out;
1152         }
1153
1154         /*
1155          * All pages are isolated and locked so anon_vma rmap
1156          * can't run anymore.
1157          */
1158         anon_vma_unlock_write(vma->anon_vma);
1159
1160         __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl,
1161                         &compound_pagelist);
1162         pte_unmap(pte);
1163         __SetPageUptodate(new_page);
1164         pgtable = pmd_pgtable(_pmd);
1165
1166         _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
1167         _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
1168
1169         /*
1170          * spin_lock() below is not the equivalent of smp_wmb(), so
1171          * this is needed to avoid the copy_huge_page writes to become
1172          * visible after the set_pmd_at() write.
1173          */
1174         smp_wmb();
1175
1176         spin_lock(pmd_ptl);
1177         BUG_ON(!pmd_none(*pmd));
1178         mem_cgroup_commit_charge(new_page, memcg, false);
1179         page_add_new_anon_rmap(new_page, vma, address, true);
1180         count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
1181         lru_cache_add_active_or_unevictable(new_page, vma);
1182         pgtable_trans_huge_deposit(mm, pmd, pgtable);
1183         set_pmd_at(mm, address, pmd, _pmd);
1184         update_mmu_cache_pmd(vma, address, pmd);
1185         spin_unlock(pmd_ptl);
1186
1187         *hpage = NULL;
1188
1189         khugepaged_pages_collapsed++;
1190         result = SCAN_SUCCEED;
1191 out_up_write:
1192         up_write(&mm->mmap_sem);
1193 out_nolock:
1194         trace_mm_collapse_huge_page(mm, isolated, result);
1195         return;
1196 out:
1197         mem_cgroup_cancel_charge(new_page, memcg);
1198         goto out_up_write;
1199 }
1200
1201 static int khugepaged_scan_pmd(struct mm_struct *mm,
1202                                struct vm_area_struct *vma,
1203                                unsigned long address,
1204                                struct page **hpage)
1205 {
1206         pmd_t *pmd;
1207         pte_t *pte, *_pte;
1208         int ret = 0, result = 0, referenced = 0;
1209         int none_or_zero = 0, shared = 0;
1210         struct page *page = NULL;
1211         unsigned long _address;
1212         spinlock_t *ptl;
1213         int node = NUMA_NO_NODE, unmapped = 0;
1214         bool writable = false;
1215
1216         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1217
1218         pmd = mm_find_pmd(mm, address);
1219         if (!pmd) {
1220                 result = SCAN_PMD_NULL;
1221                 goto out;
1222         }
1223
1224         memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
1225         pte = pte_offset_map_lock(mm, pmd, address, &ptl);
1226         for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
1227              _pte++, _address += PAGE_SIZE) {
1228                 pte_t pteval = *_pte;
1229                 if (is_swap_pte(pteval)) {
1230                         if (++unmapped <= khugepaged_max_ptes_swap) {
1231                                 /*
1232                                  * Always be strict with uffd-wp
1233                                  * enabled swap entries.  Please see
1234                                  * comment below for pte_uffd_wp().
1235                                  */
1236                                 if (pte_swp_uffd_wp(pteval)) {
1237                                         result = SCAN_PTE_UFFD_WP;
1238                                         goto out_unmap;
1239                                 }
1240                                 continue;
1241                         } else {
1242                                 result = SCAN_EXCEED_SWAP_PTE;
1243                                 goto out_unmap;
1244                         }
1245                 }
1246                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
1247                         if (!userfaultfd_armed(vma) &&
1248                             ++none_or_zero <= khugepaged_max_ptes_none) {
1249                                 continue;
1250                         } else {
1251                                 result = SCAN_EXCEED_NONE_PTE;
1252                                 goto out_unmap;
1253                         }
1254                 }
1255                 if (!pte_present(pteval)) {
1256                         result = SCAN_PTE_NON_PRESENT;
1257                         goto out_unmap;
1258                 }
1259                 if (pte_uffd_wp(pteval)) {
1260                         /*
1261                          * Don't collapse the page if any of the small
1262                          * PTEs are armed with uffd write protection.
1263                          * Here we can also mark the new huge pmd as
1264                          * write protected if any of the small ones is
1265                          * marked but that could bring uknown
1266                          * userfault messages that falls outside of
1267                          * the registered range.  So, just be simple.
1268                          */
1269                         result = SCAN_PTE_UFFD_WP;
1270                         goto out_unmap;
1271                 }
1272                 if (pte_write(pteval))
1273                         writable = true;
1274
1275                 page = vm_normal_page(vma, _address, pteval);
1276                 if (unlikely(!page)) {
1277                         result = SCAN_PAGE_NULL;
1278                         goto out_unmap;
1279                 }
1280
1281                 if (page_mapcount(page) > 1 &&
1282                                 ++shared > khugepaged_max_ptes_shared) {
1283                         result = SCAN_EXCEED_SHARED_PTE;
1284                         goto out_unmap;
1285                 }
1286
1287                 page = compound_head(page);
1288
1289                 /*
1290                  * Record which node the original page is from and save this
1291                  * information to khugepaged_node_load[].
1292                  * Khupaged will allocate hugepage from the node has the max
1293                  * hit record.
1294                  */
1295                 node = page_to_nid(page);
1296                 if (khugepaged_scan_abort(node)) {
1297                         result = SCAN_SCAN_ABORT;
1298                         goto out_unmap;
1299                 }
1300                 khugepaged_node_load[node]++;
1301                 if (!PageLRU(page)) {
1302                         result = SCAN_PAGE_LRU;
1303                         goto out_unmap;
1304                 }
1305                 if (PageLocked(page)) {
1306                         result = SCAN_PAGE_LOCK;
1307                         goto out_unmap;
1308                 }
1309                 if (!PageAnon(page)) {
1310                         result = SCAN_PAGE_ANON;
1311                         goto out_unmap;
1312                 }
1313
1314                 /*
1315                  * Check if the page has any GUP (or other external) pins.
1316                  *
1317                  * Here the check is racy it may see totmal_mapcount > refcount
1318                  * in some cases.
1319                  * For example, one process with one forked child process.
1320                  * The parent has the PMD split due to MADV_DONTNEED, then
1321                  * the child is trying unmap the whole PMD, but khugepaged
1322                  * may be scanning the parent between the child has
1323                  * PageDoubleMap flag cleared and dec the mapcount.  So
1324                  * khugepaged may see total_mapcount > refcount.
1325                  *
1326                  * But such case is ephemeral we could always retry collapse
1327                  * later.  However it may report false positive if the page
1328                  * has excessive GUP pins (i.e. 512).  Anyway the same check
1329                  * will be done again later the risk seems low.
1330                  */
1331                 if (!is_refcount_suitable(page)) {
1332                         result = SCAN_PAGE_COUNT;
1333                         goto out_unmap;
1334                 }
1335                 if (pte_young(pteval) ||
1336                     page_is_young(page) || PageReferenced(page) ||
1337                     mmu_notifier_test_young(vma->vm_mm, address))
1338                         referenced++;
1339         }
1340         if (!writable) {
1341                 result = SCAN_PAGE_RO;
1342         } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) {
1343                 result = SCAN_LACK_REFERENCED_PAGE;
1344         } else {
1345                 result = SCAN_SUCCEED;
1346                 ret = 1;
1347         }
1348 out_unmap:
1349         pte_unmap_unlock(pte, ptl);
1350         if (ret) {
1351                 node = khugepaged_find_target_node();
1352                 /* collapse_huge_page will return with the mmap_sem released */
1353                 collapse_huge_page(mm, address, hpage, node,
1354                                 referenced, unmapped);
1355         }
1356 out:
1357         trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
1358                                      none_or_zero, result, unmapped);
1359         return ret;
1360 }
1361
1362 static void collect_mm_slot(struct mm_slot *mm_slot)
1363 {
1364         struct mm_struct *mm = mm_slot->mm;
1365
1366         lockdep_assert_held(&khugepaged_mm_lock);
1367
1368         if (khugepaged_test_exit(mm)) {
1369                 /* free mm_slot */
1370                 hash_del(&mm_slot->hash);
1371                 list_del(&mm_slot->mm_node);
1372
1373                 /*
1374                  * Not strictly needed because the mm exited already.
1375                  *
1376                  * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
1377                  */
1378
1379                 /* khugepaged_mm_lock actually not necessary for the below */
1380                 free_mm_slot(mm_slot);
1381                 mmdrop(mm);
1382         }
1383 }
1384
1385 #ifdef CONFIG_SHMEM
1386 /*
1387  * Notify khugepaged that given addr of the mm is pte-mapped THP. Then
1388  * khugepaged should try to collapse the page table.
1389  */
1390 static int khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
1391                                          unsigned long addr)
1392 {
1393         struct mm_slot *mm_slot;
1394
1395         VM_BUG_ON(addr & ~HPAGE_PMD_MASK);
1396
1397         spin_lock(&khugepaged_mm_lock);
1398         mm_slot = get_mm_slot(mm);
1399         if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP))
1400                 mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr;
1401         spin_unlock(&khugepaged_mm_lock);
1402         return 0;
1403 }
1404
1405 /**
1406  * Try to collapse a pte-mapped THP for mm at address haddr.
1407  *
1408  * This function checks whether all the PTEs in the PMD are pointing to the
1409  * right THP. If so, retract the page table so the THP can refault in with
1410  * as pmd-mapped.
1411  */
1412 void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
1413 {
1414         unsigned long haddr = addr & HPAGE_PMD_MASK;
1415         struct vm_area_struct *vma = find_vma(mm, haddr);
1416         struct page *hpage = NULL;
1417         pte_t *start_pte, *pte;
1418         pmd_t *pmd, _pmd;
1419         spinlock_t *ptl;
1420         int count = 0;
1421         int i;
1422
1423         if (!vma || !vma->vm_file ||
1424             vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE)
1425                 return;
1426
1427         /*
1428          * This vm_flags may not have VM_HUGEPAGE if the page was not
1429          * collapsed by this mm. But we can still collapse if the page is
1430          * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check()
1431          * will not fail the vma for missing VM_HUGEPAGE
1432          */
1433         if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE))
1434                 return;
1435
1436         pmd = mm_find_pmd(mm, haddr);
1437         if (!pmd)
1438                 return;
1439
1440         start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl);
1441
1442         /* step 1: check all mapped PTEs are to the right huge page */
1443         for (i = 0, addr = haddr, pte = start_pte;
1444              i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {
1445                 struct page *page;
1446
1447                 /* empty pte, skip */
1448                 if (pte_none(*pte))
1449                         continue;
1450
1451                 /* page swapped out, abort */
1452                 if (!pte_present(*pte))
1453                         goto abort;
1454
1455                 page = vm_normal_page(vma, addr, *pte);
1456
1457                 if (!page || !PageCompound(page))
1458                         goto abort;
1459
1460                 if (!hpage) {
1461                         hpage = compound_head(page);
1462                         /*
1463                          * The mapping of the THP should not change.
1464                          *
1465                          * Note that uprobe, debugger, or MAP_PRIVATE may
1466                          * change the page table, but the new page will
1467                          * not pass PageCompound() check.
1468                          */
1469                         if (WARN_ON(hpage->mapping != vma->vm_file->f_mapping))
1470                                 goto abort;
1471                 }
1472
1473                 /*
1474                  * Confirm the page maps to the correct subpage.
1475                  *
1476                  * Note that uprobe, debugger, or MAP_PRIVATE may change
1477                  * the page table, but the new page will not pass
1478                  * PageCompound() check.
1479                  */
1480                 if (WARN_ON(hpage + i != page))
1481                         goto abort;
1482                 count++;
1483         }
1484
1485         /* step 2: adjust rmap */
1486         for (i = 0, addr = haddr, pte = start_pte;
1487              i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {
1488                 struct page *page;
1489
1490                 if (pte_none(*pte))
1491                         continue;
1492                 page = vm_normal_page(vma, addr, *pte);
1493                 page_remove_rmap(page, false);
1494         }
1495
1496         pte_unmap_unlock(start_pte, ptl);
1497
1498         /* step 3: set proper refcount and mm_counters. */
1499         if (hpage) {
1500                 page_ref_sub(hpage, count);
1501                 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count);
1502         }
1503
1504         /* step 4: collapse pmd */
1505         ptl = pmd_lock(vma->vm_mm, pmd);
1506         _pmd = pmdp_collapse_flush(vma, addr, pmd);
1507         spin_unlock(ptl);
1508         mm_dec_nr_ptes(mm);
1509         pte_free(mm, pmd_pgtable(_pmd));
1510         return;
1511
1512 abort:
1513         pte_unmap_unlock(start_pte, ptl);
1514 }
1515
1516 static int khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
1517 {
1518         struct mm_struct *mm = mm_slot->mm;
1519         int i;
1520
1521         if (likely(mm_slot->nr_pte_mapped_thp == 0))
1522                 return 0;
1523
1524         if (!down_write_trylock(&mm->mmap_sem))
1525                 return -EBUSY;
1526
1527         if (unlikely(khugepaged_test_exit(mm)))
1528                 goto out;
1529
1530         for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++)
1531                 collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]);
1532
1533 out:
1534         mm_slot->nr_pte_mapped_thp = 0;
1535         up_write(&mm->mmap_sem);
1536         return 0;
1537 }
1538
1539 static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
1540 {
1541         struct vm_area_struct *vma;
1542         unsigned long addr;
1543         pmd_t *pmd, _pmd;
1544
1545         i_mmap_lock_write(mapping);
1546         vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1547                 /*
1548                  * Check vma->anon_vma to exclude MAP_PRIVATE mappings that
1549                  * got written to. These VMAs are likely not worth investing
1550                  * down_write(mmap_sem) as PMD-mapping is likely to be split
1551                  * later.
1552                  *
1553                  * Not that vma->anon_vma check is racy: it can be set up after
1554                  * the check but before we took mmap_sem by the fault path.
1555                  * But page lock would prevent establishing any new ptes of the
1556                  * page, so we are safe.
1557                  *
1558                  * An alternative would be drop the check, but check that page
1559                  * table is clear before calling pmdp_collapse_flush() under
1560                  * ptl. It has higher chance to recover THP for the VMA, but
1561                  * has higher cost too.
1562                  */
1563                 if (vma->anon_vma)
1564                         continue;
1565                 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
1566                 if (addr & ~HPAGE_PMD_MASK)
1567                         continue;
1568                 if (vma->vm_end < addr + HPAGE_PMD_SIZE)
1569                         continue;
1570                 pmd = mm_find_pmd(vma->vm_mm, addr);
1571                 if (!pmd)
1572                         continue;
1573                 /*
1574                  * We need exclusive mmap_sem to retract page table.
1575                  *
1576                  * We use trylock due to lock inversion: we need to acquire
1577                  * mmap_sem while holding page lock. Fault path does it in
1578                  * reverse order. Trylock is a way to avoid deadlock.
1579                  */
1580                 if (down_write_trylock(&vma->vm_mm->mmap_sem)) {
1581                         spinlock_t *ptl = pmd_lock(vma->vm_mm, pmd);
1582                         /* assume page table is clear */
1583                         _pmd = pmdp_collapse_flush(vma, addr, pmd);
1584                         spin_unlock(ptl);
1585                         up_write(&vma->vm_mm->mmap_sem);
1586                         mm_dec_nr_ptes(vma->vm_mm);
1587                         pte_free(vma->vm_mm, pmd_pgtable(_pmd));
1588                 } else {
1589                         /* Try again later */
1590                         khugepaged_add_pte_mapped_thp(vma->vm_mm, addr);
1591                 }
1592         }
1593         i_mmap_unlock_write(mapping);
1594 }
1595
1596 /**
1597  * collapse_file - collapse filemap/tmpfs/shmem pages into huge one.
1598  *
1599  * Basic scheme is simple, details are more complex:
1600  *  - allocate and lock a new huge page;
1601  *  - scan page cache replacing old pages with the new one
1602  *    + swap/gup in pages if necessary;
1603  *    + fill in gaps;
1604  *    + keep old pages around in case rollback is required;
1605  *  - if replacing succeeds:
1606  *    + copy data over;
1607  *    + free old pages;
1608  *    + unlock huge page;
1609  *  - if replacing failed;
1610  *    + put all pages back and unfreeze them;
1611  *    + restore gaps in the page cache;
1612  *    + unlock and free huge page;
1613  */
1614 static void collapse_file(struct mm_struct *mm,
1615                 struct file *file, pgoff_t start,
1616                 struct page **hpage, int node)
1617 {
1618         struct address_space *mapping = file->f_mapping;
1619         gfp_t gfp;
1620         struct page *new_page;
1621         struct mem_cgroup *memcg;
1622         pgoff_t index, end = start + HPAGE_PMD_NR;
1623         LIST_HEAD(pagelist);
1624         XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
1625         int nr_none = 0, result = SCAN_SUCCEED;
1626         bool is_shmem = shmem_file(file);
1627
1628         VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
1629         VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
1630
1631         /* Only allocate from the target node */
1632         gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
1633
1634         new_page = khugepaged_alloc_page(hpage, gfp, node);
1635         if (!new_page) {
1636                 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
1637                 goto out;
1638         }
1639
1640         if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg))) {
1641                 result = SCAN_CGROUP_CHARGE_FAIL;
1642                 goto out;
1643         }
1644
1645         /* This will be less messy when we use multi-index entries */
1646         do {
1647                 xas_lock_irq(&xas);
1648                 xas_create_range(&xas);
1649                 if (!xas_error(&xas))
1650                         break;
1651                 xas_unlock_irq(&xas);
1652                 if (!xas_nomem(&xas, GFP_KERNEL)) {
1653                         mem_cgroup_cancel_charge(new_page, memcg);
1654                         result = SCAN_FAIL;
1655                         goto out;
1656                 }
1657         } while (1);
1658
1659         __SetPageLocked(new_page);
1660         if (is_shmem)
1661                 __SetPageSwapBacked(new_page);
1662         new_page->index = start;
1663         new_page->mapping = mapping;
1664
1665         /*
1666          * At this point the new_page is locked and not up-to-date.
1667          * It's safe to insert it into the page cache, because nobody would
1668          * be able to map it or use it in another way until we unlock it.
1669          */
1670
1671         xas_set(&xas, start);
1672         for (index = start; index < end; index++) {
1673                 struct page *page = xas_next(&xas);
1674
1675                 VM_BUG_ON(index != xas.xa_index);
1676                 if (is_shmem) {
1677                         if (!page) {
1678                                 /*
1679                                  * Stop if extent has been truncated or
1680                                  * hole-punched, and is now completely
1681                                  * empty.
1682                                  */
1683                                 if (index == start) {
1684                                         if (!xas_next_entry(&xas, end - 1)) {
1685                                                 result = SCAN_TRUNCATED;
1686                                                 goto xa_locked;
1687                                         }
1688                                         xas_set(&xas, index);
1689                                 }
1690                                 if (!shmem_charge(mapping->host, 1)) {
1691                                         result = SCAN_FAIL;
1692                                         goto xa_locked;
1693                                 }
1694                                 xas_store(&xas, new_page);
1695                                 nr_none++;
1696                                 continue;
1697                         }
1698
1699                         if (xa_is_value(page) || !PageUptodate(page)) {
1700                                 xas_unlock_irq(&xas);
1701                                 /* swap in or instantiate fallocated page */
1702                                 if (shmem_getpage(mapping->host, index, &page,
1703                                                   SGP_NOHUGE)) {
1704                                         result = SCAN_FAIL;
1705                                         goto xa_unlocked;
1706                                 }
1707                         } else if (trylock_page(page)) {
1708                                 get_page(page);
1709                                 xas_unlock_irq(&xas);
1710                         } else {
1711                                 result = SCAN_PAGE_LOCK;
1712                                 goto xa_locked;
1713                         }
1714                 } else {        /* !is_shmem */
1715                         if (!page || xa_is_value(page)) {
1716                                 xas_unlock_irq(&xas);
1717                                 page_cache_sync_readahead(mapping, &file->f_ra,
1718                                                           file, index,
1719                                                           PAGE_SIZE);
1720                                 /* drain pagevecs to help isolate_lru_page() */
1721                                 lru_add_drain();
1722                                 page = find_lock_page(mapping, index);
1723                                 if (unlikely(page == NULL)) {
1724                                         result = SCAN_FAIL;
1725                                         goto xa_unlocked;
1726                                 }
1727                         } else if (PageDirty(page)) {
1728                                 /*
1729                                  * khugepaged only works on read-only fd,
1730                                  * so this page is dirty because it hasn't
1731                                  * been flushed since first write. There
1732                                  * won't be new dirty pages.
1733                                  *
1734                                  * Trigger async flush here and hope the
1735                                  * writeback is done when khugepaged
1736                                  * revisits this page.
1737                                  *
1738                                  * This is a one-off situation. We are not
1739                                  * forcing writeback in loop.
1740                                  */
1741                                 xas_unlock_irq(&xas);
1742                                 filemap_flush(mapping);
1743                                 result = SCAN_FAIL;
1744                                 goto xa_unlocked;
1745                         } else if (trylock_page(page)) {
1746                                 get_page(page);
1747                                 xas_unlock_irq(&xas);
1748                         } else {
1749                                 result = SCAN_PAGE_LOCK;
1750                                 goto xa_locked;
1751                         }
1752                 }
1753
1754                 /*
1755                  * The page must be locked, so we can drop the i_pages lock
1756                  * without racing with truncate.
1757                  */
1758                 VM_BUG_ON_PAGE(!PageLocked(page), page);
1759
1760                 /* make sure the page is up to date */
1761                 if (unlikely(!PageUptodate(page))) {
1762                         result = SCAN_FAIL;
1763                         goto out_unlock;
1764                 }
1765
1766                 /*
1767                  * If file was truncated then extended, or hole-punched, before
1768                  * we locked the first page, then a THP might be there already.
1769                  */
1770                 if (PageTransCompound(page)) {
1771                         result = SCAN_PAGE_COMPOUND;
1772                         goto out_unlock;
1773                 }
1774
1775                 if (page_mapping(page) != mapping) {
1776                         result = SCAN_TRUNCATED;
1777                         goto out_unlock;
1778                 }
1779
1780                 if (!is_shmem && PageDirty(page)) {
1781                         /*
1782                          * khugepaged only works on read-only fd, so this
1783                          * page is dirty because it hasn't been flushed
1784                          * since first write.
1785                          */
1786                         result = SCAN_FAIL;
1787                         goto out_unlock;
1788                 }
1789
1790                 if (isolate_lru_page(page)) {
1791                         result = SCAN_DEL_PAGE_LRU;
1792                         goto out_unlock;
1793                 }
1794
1795                 if (page_has_private(page) &&
1796                     !try_to_release_page(page, GFP_KERNEL)) {
1797                         result = SCAN_PAGE_HAS_PRIVATE;
1798                         putback_lru_page(page);
1799                         goto out_unlock;
1800                 }
1801
1802                 if (page_mapped(page))
1803                         unmap_mapping_pages(mapping, index, 1, false);
1804
1805                 xas_lock_irq(&xas);
1806                 xas_set(&xas, index);
1807
1808                 VM_BUG_ON_PAGE(page != xas_load(&xas), page);
1809                 VM_BUG_ON_PAGE(page_mapped(page), page);
1810
1811                 /*
1812                  * The page is expected to have page_count() == 3:
1813                  *  - we hold a pin on it;
1814                  *  - one reference from page cache;
1815                  *  - one from isolate_lru_page;
1816                  */
1817                 if (!page_ref_freeze(page, 3)) {
1818                         result = SCAN_PAGE_COUNT;
1819                         xas_unlock_irq(&xas);
1820                         putback_lru_page(page);
1821                         goto out_unlock;
1822                 }
1823
1824                 /*
1825                  * Add the page to the list to be able to undo the collapse if
1826                  * something go wrong.
1827                  */
1828                 list_add_tail(&page->lru, &pagelist);
1829
1830                 /* Finally, replace with the new page. */
1831                 xas_store(&xas, new_page);
1832                 continue;
1833 out_unlock:
1834                 unlock_page(page);
1835                 put_page(page);
1836                 goto xa_unlocked;
1837         }
1838
1839         if (is_shmem)
1840                 __inc_node_page_state(new_page, NR_SHMEM_THPS);
1841         else {
1842                 __inc_node_page_state(new_page, NR_FILE_THPS);
1843                 filemap_nr_thps_inc(mapping);
1844         }
1845
1846         if (nr_none) {
1847                 struct lruvec *lruvec;
1848                 /*
1849                  * XXX: We have started try_charge and pinned the
1850                  * memcg, but the page isn't committed yet so we
1851                  * cannot use mod_lruvec_page_state(). This hackery
1852                  * will be cleaned up when remove the page->mapping
1853                  * dependency from memcg and fully charge above.
1854                  */
1855                 lruvec = mem_cgroup_lruvec(memcg, page_pgdat(new_page));
1856                 __mod_lruvec_state(lruvec, NR_FILE_PAGES, nr_none);
1857                 if (is_shmem)
1858                         __mod_lruvec_state(lruvec, NR_SHMEM, nr_none);
1859         }
1860
1861 xa_locked:
1862         xas_unlock_irq(&xas);
1863 xa_unlocked:
1864
1865         if (result == SCAN_SUCCEED) {
1866                 struct page *page, *tmp;
1867
1868                 /*
1869                  * Replacing old pages with new one has succeeded, now we
1870                  * need to copy the content and free the old pages.
1871                  */
1872                 index = start;
1873                 list_for_each_entry_safe(page, tmp, &pagelist, lru) {
1874                         while (index < page->index) {
1875                                 clear_highpage(new_page + (index % HPAGE_PMD_NR));
1876                                 index++;
1877                         }
1878                         copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
1879                                         page);
1880                         list_del(&page->lru);
1881                         page->mapping = NULL;
1882                         page_ref_unfreeze(page, 1);
1883                         ClearPageActive(page);
1884                         ClearPageUnevictable(page);
1885                         unlock_page(page);
1886                         put_page(page);
1887                         index++;
1888                 }
1889                 while (index < end) {
1890                         clear_highpage(new_page + (index % HPAGE_PMD_NR));
1891                         index++;
1892                 }
1893
1894                 SetPageUptodate(new_page);
1895                 page_ref_add(new_page, HPAGE_PMD_NR - 1);
1896                 mem_cgroup_commit_charge(new_page, memcg, false);
1897
1898                 if (is_shmem) {
1899                         set_page_dirty(new_page);
1900                         lru_cache_add_anon(new_page);
1901                 } else {
1902                         lru_cache_add_file(new_page);
1903                 }
1904                 count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
1905
1906                 /*
1907                  * Remove pte page tables, so we can re-fault the page as huge.
1908                  */
1909                 retract_page_tables(mapping, start);
1910                 *hpage = NULL;
1911
1912                 khugepaged_pages_collapsed++;
1913         } else {
1914                 struct page *page;
1915
1916                 /* Something went wrong: roll back page cache changes */
1917                 xas_lock_irq(&xas);
1918                 mapping->nrpages -= nr_none;
1919
1920                 if (is_shmem)
1921                         shmem_uncharge(mapping->host, nr_none);
1922
1923                 xas_set(&xas, start);
1924                 xas_for_each(&xas, page, end - 1) {
1925                         page = list_first_entry_or_null(&pagelist,
1926                                         struct page, lru);
1927                         if (!page || xas.xa_index < page->index) {
1928                                 if (!nr_none)
1929                                         break;
1930                                 nr_none--;
1931                                 /* Put holes back where they were */
1932                                 xas_store(&xas, NULL);
1933                                 continue;
1934                         }
1935
1936                         VM_BUG_ON_PAGE(page->index != xas.xa_index, page);
1937
1938                         /* Unfreeze the page. */
1939                         list_del(&page->lru);
1940                         page_ref_unfreeze(page, 2);
1941                         xas_store(&xas, page);
1942                         xas_pause(&xas);
1943                         xas_unlock_irq(&xas);
1944                         unlock_page(page);
1945                         putback_lru_page(page);
1946                         xas_lock_irq(&xas);
1947                 }
1948                 VM_BUG_ON(nr_none);
1949                 xas_unlock_irq(&xas);
1950
1951                 mem_cgroup_cancel_charge(new_page, memcg);
1952                 new_page->mapping = NULL;
1953         }
1954
1955         unlock_page(new_page);
1956 out:
1957         VM_BUG_ON(!list_empty(&pagelist));
1958         /* TODO: tracepoints */
1959 }
1960
1961 static void khugepaged_scan_file(struct mm_struct *mm,
1962                 struct file *file, pgoff_t start, struct page **hpage)
1963 {
1964         struct page *page = NULL;
1965         struct address_space *mapping = file->f_mapping;
1966         XA_STATE(xas, &mapping->i_pages, start);
1967         int present, swap;
1968         int node = NUMA_NO_NODE;
1969         int result = SCAN_SUCCEED;
1970
1971         present = 0;
1972         swap = 0;
1973         memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
1974         rcu_read_lock();
1975         xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) {
1976                 if (xas_retry(&xas, page))
1977                         continue;
1978
1979                 if (xa_is_value(page)) {
1980                         if (++swap > khugepaged_max_ptes_swap) {
1981                                 result = SCAN_EXCEED_SWAP_PTE;
1982                                 break;
1983                         }
1984                         continue;
1985                 }
1986
1987                 if (PageTransCompound(page)) {
1988                         result = SCAN_PAGE_COMPOUND;
1989                         break;
1990                 }
1991
1992                 node = page_to_nid(page);
1993                 if (khugepaged_scan_abort(node)) {
1994                         result = SCAN_SCAN_ABORT;
1995                         break;
1996                 }
1997                 khugepaged_node_load[node]++;
1998
1999                 if (!PageLRU(page)) {
2000                         result = SCAN_PAGE_LRU;
2001                         break;
2002                 }
2003
2004                 if (page_count(page) !=
2005                     1 + page_mapcount(page) + page_has_private(page)) {
2006                         result = SCAN_PAGE_COUNT;
2007                         break;
2008                 }
2009
2010                 /*
2011                  * We probably should check if the page is referenced here, but
2012                  * nobody would transfer pte_young() to PageReferenced() for us.
2013                  * And rmap walk here is just too costly...
2014                  */
2015
2016                 present++;
2017
2018                 if (need_resched()) {
2019                         xas_pause(&xas);
2020                         cond_resched_rcu();
2021                 }
2022         }
2023         rcu_read_unlock();
2024
2025         if (result == SCAN_SUCCEED) {
2026                 if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
2027                         result = SCAN_EXCEED_NONE_PTE;
2028                 } else {
2029                         node = khugepaged_find_target_node();
2030                         collapse_file(mm, file, start, hpage, node);
2031                 }
2032         }
2033
2034         /* TODO: tracepoints */
2035 }
2036 #else
2037 static void khugepaged_scan_file(struct mm_struct *mm,
2038                 struct file *file, pgoff_t start, struct page **hpage)
2039 {
2040         BUILD_BUG();
2041 }
2042
2043 static int khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
2044 {
2045         return 0;
2046 }
2047 #endif
2048
2049 static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
2050                                             struct page **hpage)
2051         __releases(&khugepaged_mm_lock)
2052         __acquires(&khugepaged_mm_lock)
2053 {
2054         struct mm_slot *mm_slot;
2055         struct mm_struct *mm;
2056         struct vm_area_struct *vma;
2057         int progress = 0;
2058
2059         VM_BUG_ON(!pages);
2060         lockdep_assert_held(&khugepaged_mm_lock);
2061
2062         if (khugepaged_scan.mm_slot)
2063                 mm_slot = khugepaged_scan.mm_slot;
2064         else {
2065                 mm_slot = list_entry(khugepaged_scan.mm_head.next,
2066                                      struct mm_slot, mm_node);
2067                 khugepaged_scan.address = 0;
2068                 khugepaged_scan.mm_slot = mm_slot;
2069         }
2070         spin_unlock(&khugepaged_mm_lock);
2071         khugepaged_collapse_pte_mapped_thps(mm_slot);
2072
2073         mm = mm_slot->mm;
2074         /*
2075          * Don't wait for semaphore (to avoid long wait times).  Just move to
2076          * the next mm on the list.
2077          */
2078         vma = NULL;
2079         if (unlikely(!down_read_trylock(&mm->mmap_sem)))
2080                 goto breakouterloop_mmap_sem;
2081         if (likely(!khugepaged_test_exit(mm)))
2082                 vma = find_vma(mm, khugepaged_scan.address);
2083
2084         progress++;
2085         for (; vma; vma = vma->vm_next) {
2086                 unsigned long hstart, hend;
2087
2088                 cond_resched();
2089                 if (unlikely(khugepaged_test_exit(mm))) {
2090                         progress++;
2091                         break;
2092                 }
2093                 if (!hugepage_vma_check(vma, vma->vm_flags)) {
2094 skip:
2095                         progress++;
2096                         continue;
2097                 }
2098                 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2099                 hend = vma->vm_end & HPAGE_PMD_MASK;
2100                 if (hstart >= hend)
2101                         goto skip;
2102                 if (khugepaged_scan.address > hend)
2103                         goto skip;
2104                 if (khugepaged_scan.address < hstart)
2105                         khugepaged_scan.address = hstart;
2106                 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
2107                 if (shmem_file(vma->vm_file) && !shmem_huge_enabled(vma))
2108                         goto skip;
2109
2110                 while (khugepaged_scan.address < hend) {
2111                         int ret;
2112                         cond_resched();
2113                         if (unlikely(khugepaged_test_exit(mm)))
2114                                 goto breakouterloop;
2115
2116                         VM_BUG_ON(khugepaged_scan.address < hstart ||
2117                                   khugepaged_scan.address + HPAGE_PMD_SIZE >
2118                                   hend);
2119                         if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {
2120                                 struct file *file = get_file(vma->vm_file);
2121                                 pgoff_t pgoff = linear_page_index(vma,
2122                                                 khugepaged_scan.address);
2123
2124                                 up_read(&mm->mmap_sem);
2125                                 ret = 1;
2126                                 khugepaged_scan_file(mm, file, pgoff, hpage);
2127                                 fput(file);
2128                         } else {
2129                                 ret = khugepaged_scan_pmd(mm, vma,
2130                                                 khugepaged_scan.address,
2131                                                 hpage);
2132                         }
2133                         /* move to next address */
2134                         khugepaged_scan.address += HPAGE_PMD_SIZE;
2135                         progress += HPAGE_PMD_NR;
2136                         if (ret)
2137                                 /* we released mmap_sem so break loop */
2138                                 goto breakouterloop_mmap_sem;
2139                         if (progress >= pages)
2140                                 goto breakouterloop;
2141                 }
2142         }
2143 breakouterloop:
2144         up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
2145 breakouterloop_mmap_sem:
2146
2147         spin_lock(&khugepaged_mm_lock);
2148         VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
2149         /*
2150          * Release the current mm_slot if this mm is about to die, or
2151          * if we scanned all vmas of this mm.
2152          */
2153         if (khugepaged_test_exit(mm) || !vma) {
2154                 /*
2155                  * Make sure that if mm_users is reaching zero while
2156                  * khugepaged runs here, khugepaged_exit will find
2157                  * mm_slot not pointing to the exiting mm.
2158                  */
2159                 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
2160                         khugepaged_scan.mm_slot = list_entry(
2161                                 mm_slot->mm_node.next,
2162                                 struct mm_slot, mm_node);
2163                         khugepaged_scan.address = 0;
2164                 } else {
2165                         khugepaged_scan.mm_slot = NULL;
2166                         khugepaged_full_scans++;
2167                 }
2168
2169                 collect_mm_slot(mm_slot);
2170         }
2171
2172         return progress;
2173 }
2174
2175 static int khugepaged_has_work(void)
2176 {
2177         return !list_empty(&khugepaged_scan.mm_head) &&
2178                 khugepaged_enabled();
2179 }
2180
2181 static int khugepaged_wait_event(void)
2182 {
2183         return !list_empty(&khugepaged_scan.mm_head) ||
2184                 kthread_should_stop();
2185 }
2186
2187 static void khugepaged_do_scan(void)
2188 {
2189         struct page *hpage = NULL;
2190         unsigned int progress = 0, pass_through_head = 0;
2191         unsigned int pages = khugepaged_pages_to_scan;
2192         bool wait = true;
2193
2194         barrier(); /* write khugepaged_pages_to_scan to local stack */
2195
2196         lru_add_drain_all();
2197
2198         while (progress < pages) {
2199                 if (!khugepaged_prealloc_page(&hpage, &wait))
2200                         break;
2201
2202                 cond_resched();
2203
2204                 if (unlikely(kthread_should_stop() || try_to_freeze()))
2205                         break;
2206
2207                 spin_lock(&khugepaged_mm_lock);
2208                 if (!khugepaged_scan.mm_slot)
2209                         pass_through_head++;
2210                 if (khugepaged_has_work() &&
2211                     pass_through_head < 2)
2212                         progress += khugepaged_scan_mm_slot(pages - progress,
2213                                                             &hpage);
2214                 else
2215                         progress = pages;
2216                 spin_unlock(&khugepaged_mm_lock);
2217         }
2218
2219         if (!IS_ERR_OR_NULL(hpage))
2220                 put_page(hpage);
2221 }
2222
2223 static bool khugepaged_should_wakeup(void)
2224 {
2225         return kthread_should_stop() ||
2226                time_after_eq(jiffies, khugepaged_sleep_expire);
2227 }
2228
2229 static void khugepaged_wait_work(void)
2230 {
2231         if (khugepaged_has_work()) {
2232                 const unsigned long scan_sleep_jiffies =
2233                         msecs_to_jiffies(khugepaged_scan_sleep_millisecs);
2234
2235                 if (!scan_sleep_jiffies)
2236                         return;
2237
2238                 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies;
2239                 wait_event_freezable_timeout(khugepaged_wait,
2240                                              khugepaged_should_wakeup(),
2241                                              scan_sleep_jiffies);
2242                 return;
2243         }
2244
2245         if (khugepaged_enabled())
2246                 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
2247 }
2248
2249 static int khugepaged(void *none)
2250 {
2251         struct mm_slot *mm_slot;
2252
2253         set_freezable();
2254         set_user_nice(current, MAX_NICE);
2255
2256         while (!kthread_should_stop()) {
2257                 khugepaged_do_scan();
2258                 khugepaged_wait_work();
2259         }
2260
2261         spin_lock(&khugepaged_mm_lock);
2262         mm_slot = khugepaged_scan.mm_slot;
2263         khugepaged_scan.mm_slot = NULL;
2264         if (mm_slot)
2265                 collect_mm_slot(mm_slot);
2266         spin_unlock(&khugepaged_mm_lock);
2267         return 0;
2268 }
2269
2270 static void set_recommended_min_free_kbytes(void)
2271 {
2272         struct zone *zone;
2273         int nr_zones = 0;
2274         unsigned long recommended_min;
2275
2276         for_each_populated_zone(zone) {
2277                 /*
2278                  * We don't need to worry about fragmentation of
2279                  * ZONE_MOVABLE since it only has movable pages.
2280                  */
2281                 if (zone_idx(zone) > gfp_zone(GFP_USER))
2282                         continue;
2283
2284                 nr_zones++;
2285         }
2286
2287         /* Ensure 2 pageblocks are free to assist fragmentation avoidance */
2288         recommended_min = pageblock_nr_pages * nr_zones * 2;
2289
2290         /*
2291          * Make sure that on average at least two pageblocks are almost free
2292          * of another type, one for a migratetype to fall back to and a
2293          * second to avoid subsequent fallbacks of other types There are 3
2294          * MIGRATE_TYPES we care about.
2295          */
2296         recommended_min += pageblock_nr_pages * nr_zones *
2297                            MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
2298
2299         /* don't ever allow to reserve more than 5% of the lowmem */
2300         recommended_min = min(recommended_min,
2301                               (unsigned long) nr_free_buffer_pages() / 20);
2302         recommended_min <<= (PAGE_SHIFT-10);
2303
2304         if (recommended_min > min_free_kbytes) {
2305                 if (user_min_free_kbytes >= 0)
2306                         pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
2307                                 min_free_kbytes, recommended_min);
2308
2309                 min_free_kbytes = recommended_min;
2310         }
2311         setup_per_zone_wmarks();
2312 }
2313
2314 int start_stop_khugepaged(void)
2315 {
2316         static struct task_struct *khugepaged_thread __read_mostly;
2317         static DEFINE_MUTEX(khugepaged_mutex);
2318         int err = 0;
2319
2320         mutex_lock(&khugepaged_mutex);
2321         if (khugepaged_enabled()) {
2322                 if (!khugepaged_thread)
2323                         khugepaged_thread = kthread_run(khugepaged, NULL,
2324                                                         "khugepaged");
2325                 if (IS_ERR(khugepaged_thread)) {
2326                         pr_err("khugepaged: kthread_run(khugepaged) failed\n");
2327                         err = PTR_ERR(khugepaged_thread);
2328                         khugepaged_thread = NULL;
2329                         goto fail;
2330                 }
2331
2332                 if (!list_empty(&khugepaged_scan.mm_head))
2333                         wake_up_interruptible(&khugepaged_wait);
2334
2335                 set_recommended_min_free_kbytes();
2336         } else if (khugepaged_thread) {
2337                 kthread_stop(khugepaged_thread);
2338                 khugepaged_thread = NULL;
2339         }
2340 fail:
2341         mutex_unlock(&khugepaged_mutex);
2342         return err;
2343 }