Merge branch 'drm-intel-fixes' of git://people.freedesktop.org/~danvet/drm-intel...
[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24
25 #include <asm/page.h>
26 #include <asm/pgtable.h>
27 #include <linux/io.h>
28
29 #include <linux/hugetlb.h>
30 #include <linux/node.h>
31 #include "internal.h"
32
33 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
34 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
35 unsigned long hugepages_treat_as_movable;
36
37 static int max_hstate;
38 unsigned int default_hstate_idx;
39 struct hstate hstates[HUGE_MAX_HSTATE];
40
41 __initdata LIST_HEAD(huge_boot_pages);
42
43 /* for command line parsing */
44 static struct hstate * __initdata parsed_hstate;
45 static unsigned long __initdata default_hstate_max_huge_pages;
46 static unsigned long __initdata default_hstate_size;
47
48 #define for_each_hstate(h) \
49         for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
50
51 /*
52  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
53  */
54 static DEFINE_SPINLOCK(hugetlb_lock);
55
56 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
57 {
58         bool free = (spool->count == 0) && (spool->used_hpages == 0);
59
60         spin_unlock(&spool->lock);
61
62         /* If no pages are used, and no other handles to the subpool
63          * remain, free the subpool the subpool remain */
64         if (free)
65                 kfree(spool);
66 }
67
68 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
69 {
70         struct hugepage_subpool *spool;
71
72         spool = kmalloc(sizeof(*spool), GFP_KERNEL);
73         if (!spool)
74                 return NULL;
75
76         spin_lock_init(&spool->lock);
77         spool->count = 1;
78         spool->max_hpages = nr_blocks;
79         spool->used_hpages = 0;
80
81         return spool;
82 }
83
84 void hugepage_put_subpool(struct hugepage_subpool *spool)
85 {
86         spin_lock(&spool->lock);
87         BUG_ON(!spool->count);
88         spool->count--;
89         unlock_or_release_subpool(spool);
90 }
91
92 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
93                                       long delta)
94 {
95         int ret = 0;
96
97         if (!spool)
98                 return 0;
99
100         spin_lock(&spool->lock);
101         if ((spool->used_hpages + delta) <= spool->max_hpages) {
102                 spool->used_hpages += delta;
103         } else {
104                 ret = -ENOMEM;
105         }
106         spin_unlock(&spool->lock);
107
108         return ret;
109 }
110
111 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
112                                        long delta)
113 {
114         if (!spool)
115                 return;
116
117         spin_lock(&spool->lock);
118         spool->used_hpages -= delta;
119         /* If hugetlbfs_put_super couldn't free spool due to
120         * an outstanding quota reference, free it now. */
121         unlock_or_release_subpool(spool);
122 }
123
124 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
125 {
126         return HUGETLBFS_SB(inode->i_sb)->spool;
127 }
128
129 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
130 {
131         return subpool_inode(vma->vm_file->f_dentry->d_inode);
132 }
133
134 /*
135  * Region tracking -- allows tracking of reservations and instantiated pages
136  *                    across the pages in a mapping.
137  *
138  * The region data structures are protected by a combination of the mmap_sem
139  * and the hugetlb_instantion_mutex.  To access or modify a region the caller
140  * must either hold the mmap_sem for write, or the mmap_sem for read and
141  * the hugetlb_instantiation mutex:
142  *
143  *      down_write(&mm->mmap_sem);
144  * or
145  *      down_read(&mm->mmap_sem);
146  *      mutex_lock(&hugetlb_instantiation_mutex);
147  */
148 struct file_region {
149         struct list_head link;
150         long from;
151         long to;
152 };
153
154 static long region_add(struct list_head *head, long f, long t)
155 {
156         struct file_region *rg, *nrg, *trg;
157
158         /* Locate the region we are either in or before. */
159         list_for_each_entry(rg, head, link)
160                 if (f <= rg->to)
161                         break;
162
163         /* Round our left edge to the current segment if it encloses us. */
164         if (f > rg->from)
165                 f = rg->from;
166
167         /* Check for and consume any regions we now overlap with. */
168         nrg = rg;
169         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
170                 if (&rg->link == head)
171                         break;
172                 if (rg->from > t)
173                         break;
174
175                 /* If this area reaches higher then extend our area to
176                  * include it completely.  If this is not the first area
177                  * which we intend to reuse, free it. */
178                 if (rg->to > t)
179                         t = rg->to;
180                 if (rg != nrg) {
181                         list_del(&rg->link);
182                         kfree(rg);
183                 }
184         }
185         nrg->from = f;
186         nrg->to = t;
187         return 0;
188 }
189
190 static long region_chg(struct list_head *head, long f, long t)
191 {
192         struct file_region *rg, *nrg;
193         long chg = 0;
194
195         /* Locate the region we are before or in. */
196         list_for_each_entry(rg, head, link)
197                 if (f <= rg->to)
198                         break;
199
200         /* If we are below the current region then a new region is required.
201          * Subtle, allocate a new region at the position but make it zero
202          * size such that we can guarantee to record the reservation. */
203         if (&rg->link == head || t < rg->from) {
204                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
205                 if (!nrg)
206                         return -ENOMEM;
207                 nrg->from = f;
208                 nrg->to   = f;
209                 INIT_LIST_HEAD(&nrg->link);
210                 list_add(&nrg->link, rg->link.prev);
211
212                 return t - f;
213         }
214
215         /* Round our left edge to the current segment if it encloses us. */
216         if (f > rg->from)
217                 f = rg->from;
218         chg = t - f;
219
220         /* Check for and consume any regions we now overlap with. */
221         list_for_each_entry(rg, rg->link.prev, link) {
222                 if (&rg->link == head)
223                         break;
224                 if (rg->from > t)
225                         return chg;
226
227                 /* We overlap with this area, if it extends further than
228                  * us then we must extend ourselves.  Account for its
229                  * existing reservation. */
230                 if (rg->to > t) {
231                         chg += rg->to - t;
232                         t = rg->to;
233                 }
234                 chg -= rg->to - rg->from;
235         }
236         return chg;
237 }
238
239 static long region_truncate(struct list_head *head, long end)
240 {
241         struct file_region *rg, *trg;
242         long chg = 0;
243
244         /* Locate the region we are either in or before. */
245         list_for_each_entry(rg, head, link)
246                 if (end <= rg->to)
247                         break;
248         if (&rg->link == head)
249                 return 0;
250
251         /* If we are in the middle of a region then adjust it. */
252         if (end > rg->from) {
253                 chg = rg->to - end;
254                 rg->to = end;
255                 rg = list_entry(rg->link.next, typeof(*rg), link);
256         }
257
258         /* Drop any remaining regions. */
259         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
260                 if (&rg->link == head)
261                         break;
262                 chg += rg->to - rg->from;
263                 list_del(&rg->link);
264                 kfree(rg);
265         }
266         return chg;
267 }
268
269 static long region_count(struct list_head *head, long f, long t)
270 {
271         struct file_region *rg;
272         long chg = 0;
273
274         /* Locate each segment we overlap with, and count that overlap. */
275         list_for_each_entry(rg, head, link) {
276                 long seg_from;
277                 long seg_to;
278
279                 if (rg->to <= f)
280                         continue;
281                 if (rg->from >= t)
282                         break;
283
284                 seg_from = max(rg->from, f);
285                 seg_to = min(rg->to, t);
286
287                 chg += seg_to - seg_from;
288         }
289
290         return chg;
291 }
292
293 /*
294  * Convert the address within this vma to the page offset within
295  * the mapping, in pagecache page units; huge pages here.
296  */
297 static pgoff_t vma_hugecache_offset(struct hstate *h,
298                         struct vm_area_struct *vma, unsigned long address)
299 {
300         return ((address - vma->vm_start) >> huge_page_shift(h)) +
301                         (vma->vm_pgoff >> huge_page_order(h));
302 }
303
304 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
305                                      unsigned long address)
306 {
307         return vma_hugecache_offset(hstate_vma(vma), vma, address);
308 }
309
310 /*
311  * Return the size of the pages allocated when backing a VMA. In the majority
312  * cases this will be same size as used by the page table entries.
313  */
314 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
315 {
316         struct hstate *hstate;
317
318         if (!is_vm_hugetlb_page(vma))
319                 return PAGE_SIZE;
320
321         hstate = hstate_vma(vma);
322
323         return 1UL << (hstate->order + PAGE_SHIFT);
324 }
325 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
326
327 /*
328  * Return the page size being used by the MMU to back a VMA. In the majority
329  * of cases, the page size used by the kernel matches the MMU size. On
330  * architectures where it differs, an architecture-specific version of this
331  * function is required.
332  */
333 #ifndef vma_mmu_pagesize
334 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
335 {
336         return vma_kernel_pagesize(vma);
337 }
338 #endif
339
340 /*
341  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
342  * bits of the reservation map pointer, which are always clear due to
343  * alignment.
344  */
345 #define HPAGE_RESV_OWNER    (1UL << 0)
346 #define HPAGE_RESV_UNMAPPED (1UL << 1)
347 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
348
349 /*
350  * These helpers are used to track how many pages are reserved for
351  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
352  * is guaranteed to have their future faults succeed.
353  *
354  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
355  * the reserve counters are updated with the hugetlb_lock held. It is safe
356  * to reset the VMA at fork() time as it is not in use yet and there is no
357  * chance of the global counters getting corrupted as a result of the values.
358  *
359  * The private mapping reservation is represented in a subtly different
360  * manner to a shared mapping.  A shared mapping has a region map associated
361  * with the underlying file, this region map represents the backing file
362  * pages which have ever had a reservation assigned which this persists even
363  * after the page is instantiated.  A private mapping has a region map
364  * associated with the original mmap which is attached to all VMAs which
365  * reference it, this region map represents those offsets which have consumed
366  * reservation ie. where pages have been instantiated.
367  */
368 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
369 {
370         return (unsigned long)vma->vm_private_data;
371 }
372
373 static void set_vma_private_data(struct vm_area_struct *vma,
374                                                         unsigned long value)
375 {
376         vma->vm_private_data = (void *)value;
377 }
378
379 struct resv_map {
380         struct kref refs;
381         struct list_head regions;
382 };
383
384 static struct resv_map *resv_map_alloc(void)
385 {
386         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
387         if (!resv_map)
388                 return NULL;
389
390         kref_init(&resv_map->refs);
391         INIT_LIST_HEAD(&resv_map->regions);
392
393         return resv_map;
394 }
395
396 static void resv_map_release(struct kref *ref)
397 {
398         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
399
400         /* Clear out any active regions before we release the map. */
401         region_truncate(&resv_map->regions, 0);
402         kfree(resv_map);
403 }
404
405 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
406 {
407         VM_BUG_ON(!is_vm_hugetlb_page(vma));
408         if (!(vma->vm_flags & VM_MAYSHARE))
409                 return (struct resv_map *)(get_vma_private_data(vma) &
410                                                         ~HPAGE_RESV_MASK);
411         return NULL;
412 }
413
414 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
415 {
416         VM_BUG_ON(!is_vm_hugetlb_page(vma));
417         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
418
419         set_vma_private_data(vma, (get_vma_private_data(vma) &
420                                 HPAGE_RESV_MASK) | (unsigned long)map);
421 }
422
423 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
424 {
425         VM_BUG_ON(!is_vm_hugetlb_page(vma));
426         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
427
428         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
429 }
430
431 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
432 {
433         VM_BUG_ON(!is_vm_hugetlb_page(vma));
434
435         return (get_vma_private_data(vma) & flag) != 0;
436 }
437
438 /* Decrement the reserved pages in the hugepage pool by one */
439 static void decrement_hugepage_resv_vma(struct hstate *h,
440                         struct vm_area_struct *vma)
441 {
442         if (vma->vm_flags & VM_NORESERVE)
443                 return;
444
445         if (vma->vm_flags & VM_MAYSHARE) {
446                 /* Shared mappings always use reserves */
447                 h->resv_huge_pages--;
448         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
449                 /*
450                  * Only the process that called mmap() has reserves for
451                  * private mappings.
452                  */
453                 h->resv_huge_pages--;
454         }
455 }
456
457 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
458 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
459 {
460         VM_BUG_ON(!is_vm_hugetlb_page(vma));
461         if (!(vma->vm_flags & VM_MAYSHARE))
462                 vma->vm_private_data = (void *)0;
463 }
464
465 /* Returns true if the VMA has associated reserve pages */
466 static int vma_has_reserves(struct vm_area_struct *vma)
467 {
468         if (vma->vm_flags & VM_MAYSHARE)
469                 return 1;
470         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
471                 return 1;
472         return 0;
473 }
474
475 static void copy_gigantic_page(struct page *dst, struct page *src)
476 {
477         int i;
478         struct hstate *h = page_hstate(src);
479         struct page *dst_base = dst;
480         struct page *src_base = src;
481
482         for (i = 0; i < pages_per_huge_page(h); ) {
483                 cond_resched();
484                 copy_highpage(dst, src);
485
486                 i++;
487                 dst = mem_map_next(dst, dst_base, i);
488                 src = mem_map_next(src, src_base, i);
489         }
490 }
491
492 void copy_huge_page(struct page *dst, struct page *src)
493 {
494         int i;
495         struct hstate *h = page_hstate(src);
496
497         if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
498                 copy_gigantic_page(dst, src);
499                 return;
500         }
501
502         might_sleep();
503         for (i = 0; i < pages_per_huge_page(h); i++) {
504                 cond_resched();
505                 copy_highpage(dst + i, src + i);
506         }
507 }
508
509 static void enqueue_huge_page(struct hstate *h, struct page *page)
510 {
511         int nid = page_to_nid(page);
512         list_add(&page->lru, &h->hugepage_freelists[nid]);
513         h->free_huge_pages++;
514         h->free_huge_pages_node[nid]++;
515 }
516
517 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
518 {
519         struct page *page;
520
521         if (list_empty(&h->hugepage_freelists[nid]))
522                 return NULL;
523         page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
524         list_del(&page->lru);
525         set_page_refcounted(page);
526         h->free_huge_pages--;
527         h->free_huge_pages_node[nid]--;
528         return page;
529 }
530
531 static struct page *dequeue_huge_page_vma(struct hstate *h,
532                                 struct vm_area_struct *vma,
533                                 unsigned long address, int avoid_reserve)
534 {
535         struct page *page = NULL;
536         struct mempolicy *mpol;
537         nodemask_t *nodemask;
538         struct zonelist *zonelist;
539         struct zone *zone;
540         struct zoneref *z;
541         unsigned int cpuset_mems_cookie;
542
543 retry_cpuset:
544         cpuset_mems_cookie = get_mems_allowed();
545         zonelist = huge_zonelist(vma, address,
546                                         htlb_alloc_mask, &mpol, &nodemask);
547         /*
548          * A child process with MAP_PRIVATE mappings created by their parent
549          * have no page reserves. This check ensures that reservations are
550          * not "stolen". The child may still get SIGKILLed
551          */
552         if (!vma_has_reserves(vma) &&
553                         h->free_huge_pages - h->resv_huge_pages == 0)
554                 goto err;
555
556         /* If reserves cannot be used, ensure enough pages are in the pool */
557         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
558                 goto err;
559
560         for_each_zone_zonelist_nodemask(zone, z, zonelist,
561                                                 MAX_NR_ZONES - 1, nodemask) {
562                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask)) {
563                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
564                         if (page) {
565                                 if (!avoid_reserve)
566                                         decrement_hugepage_resv_vma(h, vma);
567                                 break;
568                         }
569                 }
570         }
571
572         mpol_cond_put(mpol);
573         if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
574                 goto retry_cpuset;
575         return page;
576
577 err:
578         mpol_cond_put(mpol);
579         return NULL;
580 }
581
582 static void update_and_free_page(struct hstate *h, struct page *page)
583 {
584         int i;
585
586         VM_BUG_ON(h->order >= MAX_ORDER);
587
588         h->nr_huge_pages--;
589         h->nr_huge_pages_node[page_to_nid(page)]--;
590         for (i = 0; i < pages_per_huge_page(h); i++) {
591                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
592                                 1 << PG_referenced | 1 << PG_dirty |
593                                 1 << PG_active | 1 << PG_reserved |
594                                 1 << PG_private | 1 << PG_writeback);
595         }
596         set_compound_page_dtor(page, NULL);
597         set_page_refcounted(page);
598         arch_release_hugepage(page);
599         __free_pages(page, huge_page_order(h));
600 }
601
602 struct hstate *size_to_hstate(unsigned long size)
603 {
604         struct hstate *h;
605
606         for_each_hstate(h) {
607                 if (huge_page_size(h) == size)
608                         return h;
609         }
610         return NULL;
611 }
612
613 static void free_huge_page(struct page *page)
614 {
615         /*
616          * Can't pass hstate in here because it is called from the
617          * compound page destructor.
618          */
619         struct hstate *h = page_hstate(page);
620         int nid = page_to_nid(page);
621         struct hugepage_subpool *spool =
622                 (struct hugepage_subpool *)page_private(page);
623
624         set_page_private(page, 0);
625         page->mapping = NULL;
626         BUG_ON(page_count(page));
627         BUG_ON(page_mapcount(page));
628         INIT_LIST_HEAD(&page->lru);
629
630         spin_lock(&hugetlb_lock);
631         if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
632                 update_and_free_page(h, page);
633                 h->surplus_huge_pages--;
634                 h->surplus_huge_pages_node[nid]--;
635         } else {
636                 enqueue_huge_page(h, page);
637         }
638         spin_unlock(&hugetlb_lock);
639         hugepage_subpool_put_pages(spool, 1);
640 }
641
642 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
643 {
644         set_compound_page_dtor(page, free_huge_page);
645         spin_lock(&hugetlb_lock);
646         h->nr_huge_pages++;
647         h->nr_huge_pages_node[nid]++;
648         spin_unlock(&hugetlb_lock);
649         put_page(page); /* free it into the hugepage allocator */
650 }
651
652 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
653 {
654         int i;
655         int nr_pages = 1 << order;
656         struct page *p = page + 1;
657
658         /* we rely on prep_new_huge_page to set the destructor */
659         set_compound_order(page, order);
660         __SetPageHead(page);
661         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
662                 __SetPageTail(p);
663                 set_page_count(p, 0);
664                 p->first_page = page;
665         }
666 }
667
668 int PageHuge(struct page *page)
669 {
670         compound_page_dtor *dtor;
671
672         if (!PageCompound(page))
673                 return 0;
674
675         page = compound_head(page);
676         dtor = get_compound_page_dtor(page);
677
678         return dtor == free_huge_page;
679 }
680 EXPORT_SYMBOL_GPL(PageHuge);
681
682 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
683 {
684         struct page *page;
685
686         if (h->order >= MAX_ORDER)
687                 return NULL;
688
689         page = alloc_pages_exact_node(nid,
690                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
691                                                 __GFP_REPEAT|__GFP_NOWARN,
692                 huge_page_order(h));
693         if (page) {
694                 if (arch_prepare_hugepage(page)) {
695                         __free_pages(page, huge_page_order(h));
696                         return NULL;
697                 }
698                 prep_new_huge_page(h, page, nid);
699         }
700
701         return page;
702 }
703
704 /*
705  * common helper functions for hstate_next_node_to_{alloc|free}.
706  * We may have allocated or freed a huge page based on a different
707  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
708  * be outside of *nodes_allowed.  Ensure that we use an allowed
709  * node for alloc or free.
710  */
711 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
712 {
713         nid = next_node(nid, *nodes_allowed);
714         if (nid == MAX_NUMNODES)
715                 nid = first_node(*nodes_allowed);
716         VM_BUG_ON(nid >= MAX_NUMNODES);
717
718         return nid;
719 }
720
721 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
722 {
723         if (!node_isset(nid, *nodes_allowed))
724                 nid = next_node_allowed(nid, nodes_allowed);
725         return nid;
726 }
727
728 /*
729  * returns the previously saved node ["this node"] from which to
730  * allocate a persistent huge page for the pool and advance the
731  * next node from which to allocate, handling wrap at end of node
732  * mask.
733  */
734 static int hstate_next_node_to_alloc(struct hstate *h,
735                                         nodemask_t *nodes_allowed)
736 {
737         int nid;
738
739         VM_BUG_ON(!nodes_allowed);
740
741         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
742         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
743
744         return nid;
745 }
746
747 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
748 {
749         struct page *page;
750         int start_nid;
751         int next_nid;
752         int ret = 0;
753
754         start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
755         next_nid = start_nid;
756
757         do {
758                 page = alloc_fresh_huge_page_node(h, next_nid);
759                 if (page) {
760                         ret = 1;
761                         break;
762                 }
763                 next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
764         } while (next_nid != start_nid);
765
766         if (ret)
767                 count_vm_event(HTLB_BUDDY_PGALLOC);
768         else
769                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
770
771         return ret;
772 }
773
774 /*
775  * helper for free_pool_huge_page() - return the previously saved
776  * node ["this node"] from which to free a huge page.  Advance the
777  * next node id whether or not we find a free huge page to free so
778  * that the next attempt to free addresses the next node.
779  */
780 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
781 {
782         int nid;
783
784         VM_BUG_ON(!nodes_allowed);
785
786         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
787         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
788
789         return nid;
790 }
791
792 /*
793  * Free huge page from pool from next node to free.
794  * Attempt to keep persistent huge pages more or less
795  * balanced over allowed nodes.
796  * Called with hugetlb_lock locked.
797  */
798 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
799                                                          bool acct_surplus)
800 {
801         int start_nid;
802         int next_nid;
803         int ret = 0;
804
805         start_nid = hstate_next_node_to_free(h, nodes_allowed);
806         next_nid = start_nid;
807
808         do {
809                 /*
810                  * If we're returning unused surplus pages, only examine
811                  * nodes with surplus pages.
812                  */
813                 if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
814                     !list_empty(&h->hugepage_freelists[next_nid])) {
815                         struct page *page =
816                                 list_entry(h->hugepage_freelists[next_nid].next,
817                                           struct page, lru);
818                         list_del(&page->lru);
819                         h->free_huge_pages--;
820                         h->free_huge_pages_node[next_nid]--;
821                         if (acct_surplus) {
822                                 h->surplus_huge_pages--;
823                                 h->surplus_huge_pages_node[next_nid]--;
824                         }
825                         update_and_free_page(h, page);
826                         ret = 1;
827                         break;
828                 }
829                 next_nid = hstate_next_node_to_free(h, nodes_allowed);
830         } while (next_nid != start_nid);
831
832         return ret;
833 }
834
835 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
836 {
837         struct page *page;
838         unsigned int r_nid;
839
840         if (h->order >= MAX_ORDER)
841                 return NULL;
842
843         /*
844          * Assume we will successfully allocate the surplus page to
845          * prevent racing processes from causing the surplus to exceed
846          * overcommit
847          *
848          * This however introduces a different race, where a process B
849          * tries to grow the static hugepage pool while alloc_pages() is
850          * called by process A. B will only examine the per-node
851          * counters in determining if surplus huge pages can be
852          * converted to normal huge pages in adjust_pool_surplus(). A
853          * won't be able to increment the per-node counter, until the
854          * lock is dropped by B, but B doesn't drop hugetlb_lock until
855          * no more huge pages can be converted from surplus to normal
856          * state (and doesn't try to convert again). Thus, we have a
857          * case where a surplus huge page exists, the pool is grown, and
858          * the surplus huge page still exists after, even though it
859          * should just have been converted to a normal huge page. This
860          * does not leak memory, though, as the hugepage will be freed
861          * once it is out of use. It also does not allow the counters to
862          * go out of whack in adjust_pool_surplus() as we don't modify
863          * the node values until we've gotten the hugepage and only the
864          * per-node value is checked there.
865          */
866         spin_lock(&hugetlb_lock);
867         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
868                 spin_unlock(&hugetlb_lock);
869                 return NULL;
870         } else {
871                 h->nr_huge_pages++;
872                 h->surplus_huge_pages++;
873         }
874         spin_unlock(&hugetlb_lock);
875
876         if (nid == NUMA_NO_NODE)
877                 page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
878                                    __GFP_REPEAT|__GFP_NOWARN,
879                                    huge_page_order(h));
880         else
881                 page = alloc_pages_exact_node(nid,
882                         htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
883                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
884
885         if (page && arch_prepare_hugepage(page)) {
886                 __free_pages(page, huge_page_order(h));
887                 page = NULL;
888         }
889
890         spin_lock(&hugetlb_lock);
891         if (page) {
892                 r_nid = page_to_nid(page);
893                 set_compound_page_dtor(page, free_huge_page);
894                 /*
895                  * We incremented the global counters already
896                  */
897                 h->nr_huge_pages_node[r_nid]++;
898                 h->surplus_huge_pages_node[r_nid]++;
899                 __count_vm_event(HTLB_BUDDY_PGALLOC);
900         } else {
901                 h->nr_huge_pages--;
902                 h->surplus_huge_pages--;
903                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
904         }
905         spin_unlock(&hugetlb_lock);
906
907         return page;
908 }
909
910 /*
911  * This allocation function is useful in the context where vma is irrelevant.
912  * E.g. soft-offlining uses this function because it only cares physical
913  * address of error page.
914  */
915 struct page *alloc_huge_page_node(struct hstate *h, int nid)
916 {
917         struct page *page;
918
919         spin_lock(&hugetlb_lock);
920         page = dequeue_huge_page_node(h, nid);
921         spin_unlock(&hugetlb_lock);
922
923         if (!page)
924                 page = alloc_buddy_huge_page(h, nid);
925
926         return page;
927 }
928
929 /*
930  * Increase the hugetlb pool such that it can accommodate a reservation
931  * of size 'delta'.
932  */
933 static int gather_surplus_pages(struct hstate *h, int delta)
934 {
935         struct list_head surplus_list;
936         struct page *page, *tmp;
937         int ret, i;
938         int needed, allocated;
939         bool alloc_ok = true;
940
941         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
942         if (needed <= 0) {
943                 h->resv_huge_pages += delta;
944                 return 0;
945         }
946
947         allocated = 0;
948         INIT_LIST_HEAD(&surplus_list);
949
950         ret = -ENOMEM;
951 retry:
952         spin_unlock(&hugetlb_lock);
953         for (i = 0; i < needed; i++) {
954                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
955                 if (!page) {
956                         alloc_ok = false;
957                         break;
958                 }
959                 list_add(&page->lru, &surplus_list);
960         }
961         allocated += i;
962
963         /*
964          * After retaking hugetlb_lock, we need to recalculate 'needed'
965          * because either resv_huge_pages or free_huge_pages may have changed.
966          */
967         spin_lock(&hugetlb_lock);
968         needed = (h->resv_huge_pages + delta) -
969                         (h->free_huge_pages + allocated);
970         if (needed > 0) {
971                 if (alloc_ok)
972                         goto retry;
973                 /*
974                  * We were not able to allocate enough pages to
975                  * satisfy the entire reservation so we free what
976                  * we've allocated so far.
977                  */
978                 goto free;
979         }
980         /*
981          * The surplus_list now contains _at_least_ the number of extra pages
982          * needed to accommodate the reservation.  Add the appropriate number
983          * of pages to the hugetlb pool and free the extras back to the buddy
984          * allocator.  Commit the entire reservation here to prevent another
985          * process from stealing the pages as they are added to the pool but
986          * before they are reserved.
987          */
988         needed += allocated;
989         h->resv_huge_pages += delta;
990         ret = 0;
991
992         /* Free the needed pages to the hugetlb pool */
993         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
994                 if ((--needed) < 0)
995                         break;
996                 list_del(&page->lru);
997                 /*
998                  * This page is now managed by the hugetlb allocator and has
999                  * no users -- drop the buddy allocator's reference.
1000                  */
1001                 put_page_testzero(page);
1002                 VM_BUG_ON(page_count(page));
1003                 enqueue_huge_page(h, page);
1004         }
1005 free:
1006         spin_unlock(&hugetlb_lock);
1007
1008         /* Free unnecessary surplus pages to the buddy allocator */
1009         if (!list_empty(&surplus_list)) {
1010                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1011                         list_del(&page->lru);
1012                         put_page(page);
1013                 }
1014         }
1015         spin_lock(&hugetlb_lock);
1016
1017         return ret;
1018 }
1019
1020 /*
1021  * When releasing a hugetlb pool reservation, any surplus pages that were
1022  * allocated to satisfy the reservation must be explicitly freed if they were
1023  * never used.
1024  * Called with hugetlb_lock held.
1025  */
1026 static void return_unused_surplus_pages(struct hstate *h,
1027                                         unsigned long unused_resv_pages)
1028 {
1029         unsigned long nr_pages;
1030
1031         /* Uncommit the reservation */
1032         h->resv_huge_pages -= unused_resv_pages;
1033
1034         /* Cannot return gigantic pages currently */
1035         if (h->order >= MAX_ORDER)
1036                 return;
1037
1038         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1039
1040         /*
1041          * We want to release as many surplus pages as possible, spread
1042          * evenly across all nodes with memory. Iterate across these nodes
1043          * until we can no longer free unreserved surplus pages. This occurs
1044          * when the nodes with surplus pages have no free pages.
1045          * free_pool_huge_page() will balance the the freed pages across the
1046          * on-line nodes with memory and will handle the hstate accounting.
1047          */
1048         while (nr_pages--) {
1049                 if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
1050                         break;
1051         }
1052 }
1053
1054 /*
1055  * Determine if the huge page at addr within the vma has an associated
1056  * reservation.  Where it does not we will need to logically increase
1057  * reservation and actually increase subpool usage before an allocation
1058  * can occur.  Where any new reservation would be required the
1059  * reservation change is prepared, but not committed.  Once the page
1060  * has been allocated from the subpool and instantiated the change should
1061  * be committed via vma_commit_reservation.  No action is required on
1062  * failure.
1063  */
1064 static long vma_needs_reservation(struct hstate *h,
1065                         struct vm_area_struct *vma, unsigned long addr)
1066 {
1067         struct address_space *mapping = vma->vm_file->f_mapping;
1068         struct inode *inode = mapping->host;
1069
1070         if (vma->vm_flags & VM_MAYSHARE) {
1071                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1072                 return region_chg(&inode->i_mapping->private_list,
1073                                                         idx, idx + 1);
1074
1075         } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1076                 return 1;
1077
1078         } else  {
1079                 long err;
1080                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1081                 struct resv_map *reservations = vma_resv_map(vma);
1082
1083                 err = region_chg(&reservations->regions, idx, idx + 1);
1084                 if (err < 0)
1085                         return err;
1086                 return 0;
1087         }
1088 }
1089 static void vma_commit_reservation(struct hstate *h,
1090                         struct vm_area_struct *vma, unsigned long addr)
1091 {
1092         struct address_space *mapping = vma->vm_file->f_mapping;
1093         struct inode *inode = mapping->host;
1094
1095         if (vma->vm_flags & VM_MAYSHARE) {
1096                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1097                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
1098
1099         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1100                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1101                 struct resv_map *reservations = vma_resv_map(vma);
1102
1103                 /* Mark this page used in the map. */
1104                 region_add(&reservations->regions, idx, idx + 1);
1105         }
1106 }
1107
1108 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1109                                     unsigned long addr, int avoid_reserve)
1110 {
1111         struct hugepage_subpool *spool = subpool_vma(vma);
1112         struct hstate *h = hstate_vma(vma);
1113         struct page *page;
1114         long chg;
1115
1116         /*
1117          * Processes that did not create the mapping will have no
1118          * reserves and will not have accounted against subpool
1119          * limit. Check that the subpool limit can be made before
1120          * satisfying the allocation MAP_NORESERVE mappings may also
1121          * need pages and subpool limit allocated allocated if no reserve
1122          * mapping overlaps.
1123          */
1124         chg = vma_needs_reservation(h, vma, addr);
1125         if (chg < 0)
1126                 return ERR_PTR(-VM_FAULT_OOM);
1127         if (chg)
1128                 if (hugepage_subpool_get_pages(spool, chg))
1129                         return ERR_PTR(-VM_FAULT_SIGBUS);
1130
1131         spin_lock(&hugetlb_lock);
1132         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
1133         spin_unlock(&hugetlb_lock);
1134
1135         if (!page) {
1136                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1137                 if (!page) {
1138                         hugepage_subpool_put_pages(spool, chg);
1139                         return ERR_PTR(-VM_FAULT_SIGBUS);
1140                 }
1141         }
1142
1143         set_page_private(page, (unsigned long)spool);
1144
1145         vma_commit_reservation(h, vma, addr);
1146
1147         return page;
1148 }
1149
1150 int __weak alloc_bootmem_huge_page(struct hstate *h)
1151 {
1152         struct huge_bootmem_page *m;
1153         int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
1154
1155         while (nr_nodes) {
1156                 void *addr;
1157
1158                 addr = __alloc_bootmem_node_nopanic(
1159                                 NODE_DATA(hstate_next_node_to_alloc(h,
1160                                                 &node_states[N_HIGH_MEMORY])),
1161                                 huge_page_size(h), huge_page_size(h), 0);
1162
1163                 if (addr) {
1164                         /*
1165                          * Use the beginning of the huge page to store the
1166                          * huge_bootmem_page struct (until gather_bootmem
1167                          * puts them into the mem_map).
1168                          */
1169                         m = addr;
1170                         goto found;
1171                 }
1172                 nr_nodes--;
1173         }
1174         return 0;
1175
1176 found:
1177         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1178         /* Put them into a private list first because mem_map is not up yet */
1179         list_add(&m->list, &huge_boot_pages);
1180         m->hstate = h;
1181         return 1;
1182 }
1183
1184 static void prep_compound_huge_page(struct page *page, int order)
1185 {
1186         if (unlikely(order > (MAX_ORDER - 1)))
1187                 prep_compound_gigantic_page(page, order);
1188         else
1189                 prep_compound_page(page, order);
1190 }
1191
1192 /* Put bootmem huge pages into the standard lists after mem_map is up */
1193 static void __init gather_bootmem_prealloc(void)
1194 {
1195         struct huge_bootmem_page *m;
1196
1197         list_for_each_entry(m, &huge_boot_pages, list) {
1198                 struct hstate *h = m->hstate;
1199                 struct page *page;
1200
1201 #ifdef CONFIG_HIGHMEM
1202                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1203                 free_bootmem_late((unsigned long)m,
1204                                   sizeof(struct huge_bootmem_page));
1205 #else
1206                 page = virt_to_page(m);
1207 #endif
1208                 __ClearPageReserved(page);
1209                 WARN_ON(page_count(page) != 1);
1210                 prep_compound_huge_page(page, h->order);
1211                 prep_new_huge_page(h, page, page_to_nid(page));
1212                 /*
1213                  * If we had gigantic hugepages allocated at boot time, we need
1214                  * to restore the 'stolen' pages to totalram_pages in order to
1215                  * fix confusing memory reports from free(1) and another
1216                  * side-effects, like CommitLimit going negative.
1217                  */
1218                 if (h->order > (MAX_ORDER - 1))
1219                         totalram_pages += 1 << h->order;
1220         }
1221 }
1222
1223 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1224 {
1225         unsigned long i;
1226
1227         for (i = 0; i < h->max_huge_pages; ++i) {
1228                 if (h->order >= MAX_ORDER) {
1229                         if (!alloc_bootmem_huge_page(h))
1230                                 break;
1231                 } else if (!alloc_fresh_huge_page(h,
1232                                          &node_states[N_HIGH_MEMORY]))
1233                         break;
1234         }
1235         h->max_huge_pages = i;
1236 }
1237
1238 static void __init hugetlb_init_hstates(void)
1239 {
1240         struct hstate *h;
1241
1242         for_each_hstate(h) {
1243                 /* oversize hugepages were init'ed in early boot */
1244                 if (h->order < MAX_ORDER)
1245                         hugetlb_hstate_alloc_pages(h);
1246         }
1247 }
1248
1249 static char * __init memfmt(char *buf, unsigned long n)
1250 {
1251         if (n >= (1UL << 30))
1252                 sprintf(buf, "%lu GB", n >> 30);
1253         else if (n >= (1UL << 20))
1254                 sprintf(buf, "%lu MB", n >> 20);
1255         else
1256                 sprintf(buf, "%lu KB", n >> 10);
1257         return buf;
1258 }
1259
1260 static void __init report_hugepages(void)
1261 {
1262         struct hstate *h;
1263
1264         for_each_hstate(h) {
1265                 char buf[32];
1266                 printk(KERN_INFO "HugeTLB registered %s page size, "
1267                                  "pre-allocated %ld pages\n",
1268                         memfmt(buf, huge_page_size(h)),
1269                         h->free_huge_pages);
1270         }
1271 }
1272
1273 #ifdef CONFIG_HIGHMEM
1274 static void try_to_free_low(struct hstate *h, unsigned long count,
1275                                                 nodemask_t *nodes_allowed)
1276 {
1277         int i;
1278
1279         if (h->order >= MAX_ORDER)
1280                 return;
1281
1282         for_each_node_mask(i, *nodes_allowed) {
1283                 struct page *page, *next;
1284                 struct list_head *freel = &h->hugepage_freelists[i];
1285                 list_for_each_entry_safe(page, next, freel, lru) {
1286                         if (count >= h->nr_huge_pages)
1287                                 return;
1288                         if (PageHighMem(page))
1289                                 continue;
1290                         list_del(&page->lru);
1291                         update_and_free_page(h, page);
1292                         h->free_huge_pages--;
1293                         h->free_huge_pages_node[page_to_nid(page)]--;
1294                 }
1295         }
1296 }
1297 #else
1298 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1299                                                 nodemask_t *nodes_allowed)
1300 {
1301 }
1302 #endif
1303
1304 /*
1305  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1306  * balanced by operating on them in a round-robin fashion.
1307  * Returns 1 if an adjustment was made.
1308  */
1309 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1310                                 int delta)
1311 {
1312         int start_nid, next_nid;
1313         int ret = 0;
1314
1315         VM_BUG_ON(delta != -1 && delta != 1);
1316
1317         if (delta < 0)
1318                 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
1319         else
1320                 start_nid = hstate_next_node_to_free(h, nodes_allowed);
1321         next_nid = start_nid;
1322
1323         do {
1324                 int nid = next_nid;
1325                 if (delta < 0)  {
1326                         /*
1327                          * To shrink on this node, there must be a surplus page
1328                          */
1329                         if (!h->surplus_huge_pages_node[nid]) {
1330                                 next_nid = hstate_next_node_to_alloc(h,
1331                                                                 nodes_allowed);
1332                                 continue;
1333                         }
1334                 }
1335                 if (delta > 0) {
1336                         /*
1337                          * Surplus cannot exceed the total number of pages
1338                          */
1339                         if (h->surplus_huge_pages_node[nid] >=
1340                                                 h->nr_huge_pages_node[nid]) {
1341                                 next_nid = hstate_next_node_to_free(h,
1342                                                                 nodes_allowed);
1343                                 continue;
1344                         }
1345                 }
1346
1347                 h->surplus_huge_pages += delta;
1348                 h->surplus_huge_pages_node[nid] += delta;
1349                 ret = 1;
1350                 break;
1351         } while (next_nid != start_nid);
1352
1353         return ret;
1354 }
1355
1356 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1357 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1358                                                 nodemask_t *nodes_allowed)
1359 {
1360         unsigned long min_count, ret;
1361
1362         if (h->order >= MAX_ORDER)
1363                 return h->max_huge_pages;
1364
1365         /*
1366          * Increase the pool size
1367          * First take pages out of surplus state.  Then make up the
1368          * remaining difference by allocating fresh huge pages.
1369          *
1370          * We might race with alloc_buddy_huge_page() here and be unable
1371          * to convert a surplus huge page to a normal huge page. That is
1372          * not critical, though, it just means the overall size of the
1373          * pool might be one hugepage larger than it needs to be, but
1374          * within all the constraints specified by the sysctls.
1375          */
1376         spin_lock(&hugetlb_lock);
1377         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1378                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1379                         break;
1380         }
1381
1382         while (count > persistent_huge_pages(h)) {
1383                 /*
1384                  * If this allocation races such that we no longer need the
1385                  * page, free_huge_page will handle it by freeing the page
1386                  * and reducing the surplus.
1387                  */
1388                 spin_unlock(&hugetlb_lock);
1389                 ret = alloc_fresh_huge_page(h, nodes_allowed);
1390                 spin_lock(&hugetlb_lock);
1391                 if (!ret)
1392                         goto out;
1393
1394                 /* Bail for signals. Probably ctrl-c from user */
1395                 if (signal_pending(current))
1396                         goto out;
1397         }
1398
1399         /*
1400          * Decrease the pool size
1401          * First return free pages to the buddy allocator (being careful
1402          * to keep enough around to satisfy reservations).  Then place
1403          * pages into surplus state as needed so the pool will shrink
1404          * to the desired size as pages become free.
1405          *
1406          * By placing pages into the surplus state independent of the
1407          * overcommit value, we are allowing the surplus pool size to
1408          * exceed overcommit. There are few sane options here. Since
1409          * alloc_buddy_huge_page() is checking the global counter,
1410          * though, we'll note that we're not allowed to exceed surplus
1411          * and won't grow the pool anywhere else. Not until one of the
1412          * sysctls are changed, or the surplus pages go out of use.
1413          */
1414         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1415         min_count = max(count, min_count);
1416         try_to_free_low(h, min_count, nodes_allowed);
1417         while (min_count < persistent_huge_pages(h)) {
1418                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1419                         break;
1420         }
1421         while (count < persistent_huge_pages(h)) {
1422                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1423                         break;
1424         }
1425 out:
1426         ret = persistent_huge_pages(h);
1427         spin_unlock(&hugetlb_lock);
1428         return ret;
1429 }
1430
1431 #define HSTATE_ATTR_RO(_name) \
1432         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1433
1434 #define HSTATE_ATTR(_name) \
1435         static struct kobj_attribute _name##_attr = \
1436                 __ATTR(_name, 0644, _name##_show, _name##_store)
1437
1438 static struct kobject *hugepages_kobj;
1439 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1440
1441 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1442
1443 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1444 {
1445         int i;
1446
1447         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1448                 if (hstate_kobjs[i] == kobj) {
1449                         if (nidp)
1450                                 *nidp = NUMA_NO_NODE;
1451                         return &hstates[i];
1452                 }
1453
1454         return kobj_to_node_hstate(kobj, nidp);
1455 }
1456
1457 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1458                                         struct kobj_attribute *attr, char *buf)
1459 {
1460         struct hstate *h;
1461         unsigned long nr_huge_pages;
1462         int nid;
1463
1464         h = kobj_to_hstate(kobj, &nid);
1465         if (nid == NUMA_NO_NODE)
1466                 nr_huge_pages = h->nr_huge_pages;
1467         else
1468                 nr_huge_pages = h->nr_huge_pages_node[nid];
1469
1470         return sprintf(buf, "%lu\n", nr_huge_pages);
1471 }
1472
1473 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1474                         struct kobject *kobj, struct kobj_attribute *attr,
1475                         const char *buf, size_t len)
1476 {
1477         int err;
1478         int nid;
1479         unsigned long count;
1480         struct hstate *h;
1481         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1482
1483         err = strict_strtoul(buf, 10, &count);
1484         if (err)
1485                 goto out;
1486
1487         h = kobj_to_hstate(kobj, &nid);
1488         if (h->order >= MAX_ORDER) {
1489                 err = -EINVAL;
1490                 goto out;
1491         }
1492
1493         if (nid == NUMA_NO_NODE) {
1494                 /*
1495                  * global hstate attribute
1496                  */
1497                 if (!(obey_mempolicy &&
1498                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1499                         NODEMASK_FREE(nodes_allowed);
1500                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1501                 }
1502         } else if (nodes_allowed) {
1503                 /*
1504                  * per node hstate attribute: adjust count to global,
1505                  * but restrict alloc/free to the specified node.
1506                  */
1507                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1508                 init_nodemask_of_node(nodes_allowed, nid);
1509         } else
1510                 nodes_allowed = &node_states[N_HIGH_MEMORY];
1511
1512         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1513
1514         if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1515                 NODEMASK_FREE(nodes_allowed);
1516
1517         return len;
1518 out:
1519         NODEMASK_FREE(nodes_allowed);
1520         return err;
1521 }
1522
1523 static ssize_t nr_hugepages_show(struct kobject *kobj,
1524                                        struct kobj_attribute *attr, char *buf)
1525 {
1526         return nr_hugepages_show_common(kobj, attr, buf);
1527 }
1528
1529 static ssize_t nr_hugepages_store(struct kobject *kobj,
1530                struct kobj_attribute *attr, const char *buf, size_t len)
1531 {
1532         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1533 }
1534 HSTATE_ATTR(nr_hugepages);
1535
1536 #ifdef CONFIG_NUMA
1537
1538 /*
1539  * hstate attribute for optionally mempolicy-based constraint on persistent
1540  * huge page alloc/free.
1541  */
1542 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1543                                        struct kobj_attribute *attr, char *buf)
1544 {
1545         return nr_hugepages_show_common(kobj, attr, buf);
1546 }
1547
1548 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1549                struct kobj_attribute *attr, const char *buf, size_t len)
1550 {
1551         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1552 }
1553 HSTATE_ATTR(nr_hugepages_mempolicy);
1554 #endif
1555
1556
1557 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1558                                         struct kobj_attribute *attr, char *buf)
1559 {
1560         struct hstate *h = kobj_to_hstate(kobj, NULL);
1561         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1562 }
1563
1564 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1565                 struct kobj_attribute *attr, const char *buf, size_t count)
1566 {
1567         int err;
1568         unsigned long input;
1569         struct hstate *h = kobj_to_hstate(kobj, NULL);
1570
1571         if (h->order >= MAX_ORDER)
1572                 return -EINVAL;
1573
1574         err = strict_strtoul(buf, 10, &input);
1575         if (err)
1576                 return err;
1577
1578         spin_lock(&hugetlb_lock);
1579         h->nr_overcommit_huge_pages = input;
1580         spin_unlock(&hugetlb_lock);
1581
1582         return count;
1583 }
1584 HSTATE_ATTR(nr_overcommit_hugepages);
1585
1586 static ssize_t free_hugepages_show(struct kobject *kobj,
1587                                         struct kobj_attribute *attr, char *buf)
1588 {
1589         struct hstate *h;
1590         unsigned long free_huge_pages;
1591         int nid;
1592
1593         h = kobj_to_hstate(kobj, &nid);
1594         if (nid == NUMA_NO_NODE)
1595                 free_huge_pages = h->free_huge_pages;
1596         else
1597                 free_huge_pages = h->free_huge_pages_node[nid];
1598
1599         return sprintf(buf, "%lu\n", free_huge_pages);
1600 }
1601 HSTATE_ATTR_RO(free_hugepages);
1602
1603 static ssize_t resv_hugepages_show(struct kobject *kobj,
1604                                         struct kobj_attribute *attr, char *buf)
1605 {
1606         struct hstate *h = kobj_to_hstate(kobj, NULL);
1607         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1608 }
1609 HSTATE_ATTR_RO(resv_hugepages);
1610
1611 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1612                                         struct kobj_attribute *attr, char *buf)
1613 {
1614         struct hstate *h;
1615         unsigned long surplus_huge_pages;
1616         int nid;
1617
1618         h = kobj_to_hstate(kobj, &nid);
1619         if (nid == NUMA_NO_NODE)
1620                 surplus_huge_pages = h->surplus_huge_pages;
1621         else
1622                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1623
1624         return sprintf(buf, "%lu\n", surplus_huge_pages);
1625 }
1626 HSTATE_ATTR_RO(surplus_hugepages);
1627
1628 static struct attribute *hstate_attrs[] = {
1629         &nr_hugepages_attr.attr,
1630         &nr_overcommit_hugepages_attr.attr,
1631         &free_hugepages_attr.attr,
1632         &resv_hugepages_attr.attr,
1633         &surplus_hugepages_attr.attr,
1634 #ifdef CONFIG_NUMA
1635         &nr_hugepages_mempolicy_attr.attr,
1636 #endif
1637         NULL,
1638 };
1639
1640 static struct attribute_group hstate_attr_group = {
1641         .attrs = hstate_attrs,
1642 };
1643
1644 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1645                                     struct kobject **hstate_kobjs,
1646                                     struct attribute_group *hstate_attr_group)
1647 {
1648         int retval;
1649         int hi = h - hstates;
1650
1651         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1652         if (!hstate_kobjs[hi])
1653                 return -ENOMEM;
1654
1655         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1656         if (retval)
1657                 kobject_put(hstate_kobjs[hi]);
1658
1659         return retval;
1660 }
1661
1662 static void __init hugetlb_sysfs_init(void)
1663 {
1664         struct hstate *h;
1665         int err;
1666
1667         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1668         if (!hugepages_kobj)
1669                 return;
1670
1671         for_each_hstate(h) {
1672                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1673                                          hstate_kobjs, &hstate_attr_group);
1674                 if (err)
1675                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1676                                                                 h->name);
1677         }
1678 }
1679
1680 #ifdef CONFIG_NUMA
1681
1682 /*
1683  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1684  * with node devices in node_devices[] using a parallel array.  The array
1685  * index of a node device or _hstate == node id.
1686  * This is here to avoid any static dependency of the node device driver, in
1687  * the base kernel, on the hugetlb module.
1688  */
1689 struct node_hstate {
1690         struct kobject          *hugepages_kobj;
1691         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1692 };
1693 struct node_hstate node_hstates[MAX_NUMNODES];
1694
1695 /*
1696  * A subset of global hstate attributes for node devices
1697  */
1698 static struct attribute *per_node_hstate_attrs[] = {
1699         &nr_hugepages_attr.attr,
1700         &free_hugepages_attr.attr,
1701         &surplus_hugepages_attr.attr,
1702         NULL,
1703 };
1704
1705 static struct attribute_group per_node_hstate_attr_group = {
1706         .attrs = per_node_hstate_attrs,
1707 };
1708
1709 /*
1710  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1711  * Returns node id via non-NULL nidp.
1712  */
1713 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1714 {
1715         int nid;
1716
1717         for (nid = 0; nid < nr_node_ids; nid++) {
1718                 struct node_hstate *nhs = &node_hstates[nid];
1719                 int i;
1720                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1721                         if (nhs->hstate_kobjs[i] == kobj) {
1722                                 if (nidp)
1723                                         *nidp = nid;
1724                                 return &hstates[i];
1725                         }
1726         }
1727
1728         BUG();
1729         return NULL;
1730 }
1731
1732 /*
1733  * Unregister hstate attributes from a single node device.
1734  * No-op if no hstate attributes attached.
1735  */
1736 void hugetlb_unregister_node(struct node *node)
1737 {
1738         struct hstate *h;
1739         struct node_hstate *nhs = &node_hstates[node->dev.id];
1740
1741         if (!nhs->hugepages_kobj)
1742                 return;         /* no hstate attributes */
1743
1744         for_each_hstate(h)
1745                 if (nhs->hstate_kobjs[h - hstates]) {
1746                         kobject_put(nhs->hstate_kobjs[h - hstates]);
1747                         nhs->hstate_kobjs[h - hstates] = NULL;
1748                 }
1749
1750         kobject_put(nhs->hugepages_kobj);
1751         nhs->hugepages_kobj = NULL;
1752 }
1753
1754 /*
1755  * hugetlb module exit:  unregister hstate attributes from node devices
1756  * that have them.
1757  */
1758 static void hugetlb_unregister_all_nodes(void)
1759 {
1760         int nid;
1761
1762         /*
1763          * disable node device registrations.
1764          */
1765         register_hugetlbfs_with_node(NULL, NULL);
1766
1767         /*
1768          * remove hstate attributes from any nodes that have them.
1769          */
1770         for (nid = 0; nid < nr_node_ids; nid++)
1771                 hugetlb_unregister_node(&node_devices[nid]);
1772 }
1773
1774 /*
1775  * Register hstate attributes for a single node device.
1776  * No-op if attributes already registered.
1777  */
1778 void hugetlb_register_node(struct node *node)
1779 {
1780         struct hstate *h;
1781         struct node_hstate *nhs = &node_hstates[node->dev.id];
1782         int err;
1783
1784         if (nhs->hugepages_kobj)
1785                 return;         /* already allocated */
1786
1787         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1788                                                         &node->dev.kobj);
1789         if (!nhs->hugepages_kobj)
1790                 return;
1791
1792         for_each_hstate(h) {
1793                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1794                                                 nhs->hstate_kobjs,
1795                                                 &per_node_hstate_attr_group);
1796                 if (err) {
1797                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1798                                         " for node %d\n",
1799                                                 h->name, node->dev.id);
1800                         hugetlb_unregister_node(node);
1801                         break;
1802                 }
1803         }
1804 }
1805
1806 /*
1807  * hugetlb init time:  register hstate attributes for all registered node
1808  * devices of nodes that have memory.  All on-line nodes should have
1809  * registered their associated device by this time.
1810  */
1811 static void hugetlb_register_all_nodes(void)
1812 {
1813         int nid;
1814
1815         for_each_node_state(nid, N_HIGH_MEMORY) {
1816                 struct node *node = &node_devices[nid];
1817                 if (node->dev.id == nid)
1818                         hugetlb_register_node(node);
1819         }
1820
1821         /*
1822          * Let the node device driver know we're here so it can
1823          * [un]register hstate attributes on node hotplug.
1824          */
1825         register_hugetlbfs_with_node(hugetlb_register_node,
1826                                      hugetlb_unregister_node);
1827 }
1828 #else   /* !CONFIG_NUMA */
1829
1830 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1831 {
1832         BUG();
1833         if (nidp)
1834                 *nidp = -1;
1835         return NULL;
1836 }
1837
1838 static void hugetlb_unregister_all_nodes(void) { }
1839
1840 static void hugetlb_register_all_nodes(void) { }
1841
1842 #endif
1843
1844 static void __exit hugetlb_exit(void)
1845 {
1846         struct hstate *h;
1847
1848         hugetlb_unregister_all_nodes();
1849
1850         for_each_hstate(h) {
1851                 kobject_put(hstate_kobjs[h - hstates]);
1852         }
1853
1854         kobject_put(hugepages_kobj);
1855 }
1856 module_exit(hugetlb_exit);
1857
1858 static int __init hugetlb_init(void)
1859 {
1860         /* Some platform decide whether they support huge pages at boot
1861          * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1862          * there is no such support
1863          */
1864         if (HPAGE_SHIFT == 0)
1865                 return 0;
1866
1867         if (!size_to_hstate(default_hstate_size)) {
1868                 default_hstate_size = HPAGE_SIZE;
1869                 if (!size_to_hstate(default_hstate_size))
1870                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1871         }
1872         default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1873         if (default_hstate_max_huge_pages)
1874                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1875
1876         hugetlb_init_hstates();
1877
1878         gather_bootmem_prealloc();
1879
1880         report_hugepages();
1881
1882         hugetlb_sysfs_init();
1883
1884         hugetlb_register_all_nodes();
1885
1886         return 0;
1887 }
1888 module_init(hugetlb_init);
1889
1890 /* Should be called on processing a hugepagesz=... option */
1891 void __init hugetlb_add_hstate(unsigned order)
1892 {
1893         struct hstate *h;
1894         unsigned long i;
1895
1896         if (size_to_hstate(PAGE_SIZE << order)) {
1897                 printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1898                 return;
1899         }
1900         BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1901         BUG_ON(order == 0);
1902         h = &hstates[max_hstate++];
1903         h->order = order;
1904         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
1905         h->nr_huge_pages = 0;
1906         h->free_huge_pages = 0;
1907         for (i = 0; i < MAX_NUMNODES; ++i)
1908                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
1909         h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1910         h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
1911         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1912                                         huge_page_size(h)/1024);
1913
1914         parsed_hstate = h;
1915 }
1916
1917 static int __init hugetlb_nrpages_setup(char *s)
1918 {
1919         unsigned long *mhp;
1920         static unsigned long *last_mhp;
1921
1922         /*
1923          * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1924          * so this hugepages= parameter goes to the "default hstate".
1925          */
1926         if (!max_hstate)
1927                 mhp = &default_hstate_max_huge_pages;
1928         else
1929                 mhp = &parsed_hstate->max_huge_pages;
1930
1931         if (mhp == last_mhp) {
1932                 printk(KERN_WARNING "hugepages= specified twice without "
1933                         "interleaving hugepagesz=, ignoring\n");
1934                 return 1;
1935         }
1936
1937         if (sscanf(s, "%lu", mhp) <= 0)
1938                 *mhp = 0;
1939
1940         /*
1941          * Global state is always initialized later in hugetlb_init.
1942          * But we need to allocate >= MAX_ORDER hstates here early to still
1943          * use the bootmem allocator.
1944          */
1945         if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1946                 hugetlb_hstate_alloc_pages(parsed_hstate);
1947
1948         last_mhp = mhp;
1949
1950         return 1;
1951 }
1952 __setup("hugepages=", hugetlb_nrpages_setup);
1953
1954 static int __init hugetlb_default_setup(char *s)
1955 {
1956         default_hstate_size = memparse(s, &s);
1957         return 1;
1958 }
1959 __setup("default_hugepagesz=", hugetlb_default_setup);
1960
1961 static unsigned int cpuset_mems_nr(unsigned int *array)
1962 {
1963         int node;
1964         unsigned int nr = 0;
1965
1966         for_each_node_mask(node, cpuset_current_mems_allowed)
1967                 nr += array[node];
1968
1969         return nr;
1970 }
1971
1972 #ifdef CONFIG_SYSCTL
1973 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1974                          struct ctl_table *table, int write,
1975                          void __user *buffer, size_t *length, loff_t *ppos)
1976 {
1977         struct hstate *h = &default_hstate;
1978         unsigned long tmp;
1979         int ret;
1980
1981         tmp = h->max_huge_pages;
1982
1983         if (write && h->order >= MAX_ORDER)
1984                 return -EINVAL;
1985
1986         table->data = &tmp;
1987         table->maxlen = sizeof(unsigned long);
1988         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
1989         if (ret)
1990                 goto out;
1991
1992         if (write) {
1993                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1994                                                 GFP_KERNEL | __GFP_NORETRY);
1995                 if (!(obey_mempolicy &&
1996                                init_nodemask_of_mempolicy(nodes_allowed))) {
1997                         NODEMASK_FREE(nodes_allowed);
1998                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1999                 }
2000                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2001
2002                 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
2003                         NODEMASK_FREE(nodes_allowed);
2004         }
2005 out:
2006         return ret;
2007 }
2008
2009 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2010                           void __user *buffer, size_t *length, loff_t *ppos)
2011 {
2012
2013         return hugetlb_sysctl_handler_common(false, table, write,
2014                                                         buffer, length, ppos);
2015 }
2016
2017 #ifdef CONFIG_NUMA
2018 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2019                           void __user *buffer, size_t *length, loff_t *ppos)
2020 {
2021         return hugetlb_sysctl_handler_common(true, table, write,
2022                                                         buffer, length, ppos);
2023 }
2024 #endif /* CONFIG_NUMA */
2025
2026 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
2027                         void __user *buffer,
2028                         size_t *length, loff_t *ppos)
2029 {
2030         proc_dointvec(table, write, buffer, length, ppos);
2031         if (hugepages_treat_as_movable)
2032                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
2033         else
2034                 htlb_alloc_mask = GFP_HIGHUSER;
2035         return 0;
2036 }
2037
2038 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2039                         void __user *buffer,
2040                         size_t *length, loff_t *ppos)
2041 {
2042         struct hstate *h = &default_hstate;
2043         unsigned long tmp;
2044         int ret;
2045
2046         tmp = h->nr_overcommit_huge_pages;
2047
2048         if (write && h->order >= MAX_ORDER)
2049                 return -EINVAL;
2050
2051         table->data = &tmp;
2052         table->maxlen = sizeof(unsigned long);
2053         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2054         if (ret)
2055                 goto out;
2056
2057         if (write) {
2058                 spin_lock(&hugetlb_lock);
2059                 h->nr_overcommit_huge_pages = tmp;
2060                 spin_unlock(&hugetlb_lock);
2061         }
2062 out:
2063         return ret;
2064 }
2065
2066 #endif /* CONFIG_SYSCTL */
2067
2068 void hugetlb_report_meminfo(struct seq_file *m)
2069 {
2070         struct hstate *h = &default_hstate;
2071         seq_printf(m,
2072                         "HugePages_Total:   %5lu\n"
2073                         "HugePages_Free:    %5lu\n"
2074                         "HugePages_Rsvd:    %5lu\n"
2075                         "HugePages_Surp:    %5lu\n"
2076                         "Hugepagesize:   %8lu kB\n",
2077                         h->nr_huge_pages,
2078                         h->free_huge_pages,
2079                         h->resv_huge_pages,
2080                         h->surplus_huge_pages,
2081                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2082 }
2083
2084 int hugetlb_report_node_meminfo(int nid, char *buf)
2085 {
2086         struct hstate *h = &default_hstate;
2087         return sprintf(buf,
2088                 "Node %d HugePages_Total: %5u\n"
2089                 "Node %d HugePages_Free:  %5u\n"
2090                 "Node %d HugePages_Surp:  %5u\n",
2091                 nid, h->nr_huge_pages_node[nid],
2092                 nid, h->free_huge_pages_node[nid],
2093                 nid, h->surplus_huge_pages_node[nid]);
2094 }
2095
2096 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2097 unsigned long hugetlb_total_pages(void)
2098 {
2099         struct hstate *h = &default_hstate;
2100         return h->nr_huge_pages * pages_per_huge_page(h);
2101 }
2102
2103 static int hugetlb_acct_memory(struct hstate *h, long delta)
2104 {
2105         int ret = -ENOMEM;
2106
2107         spin_lock(&hugetlb_lock);
2108         /*
2109          * When cpuset is configured, it breaks the strict hugetlb page
2110          * reservation as the accounting is done on a global variable. Such
2111          * reservation is completely rubbish in the presence of cpuset because
2112          * the reservation is not checked against page availability for the
2113          * current cpuset. Application can still potentially OOM'ed by kernel
2114          * with lack of free htlb page in cpuset that the task is in.
2115          * Attempt to enforce strict accounting with cpuset is almost
2116          * impossible (or too ugly) because cpuset is too fluid that
2117          * task or memory node can be dynamically moved between cpusets.
2118          *
2119          * The change of semantics for shared hugetlb mapping with cpuset is
2120          * undesirable. However, in order to preserve some of the semantics,
2121          * we fall back to check against current free page availability as
2122          * a best attempt and hopefully to minimize the impact of changing
2123          * semantics that cpuset has.
2124          */
2125         if (delta > 0) {
2126                 if (gather_surplus_pages(h, delta) < 0)
2127                         goto out;
2128
2129                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2130                         return_unused_surplus_pages(h, delta);
2131                         goto out;
2132                 }
2133         }
2134
2135         ret = 0;
2136         if (delta < 0)
2137                 return_unused_surplus_pages(h, (unsigned long) -delta);
2138
2139 out:
2140         spin_unlock(&hugetlb_lock);
2141         return ret;
2142 }
2143
2144 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2145 {
2146         struct resv_map *reservations = vma_resv_map(vma);
2147
2148         /*
2149          * This new VMA should share its siblings reservation map if present.
2150          * The VMA will only ever have a valid reservation map pointer where
2151          * it is being copied for another still existing VMA.  As that VMA
2152          * has a reference to the reservation map it cannot disappear until
2153          * after this open call completes.  It is therefore safe to take a
2154          * new reference here without additional locking.
2155          */
2156         if (reservations)
2157                 kref_get(&reservations->refs);
2158 }
2159
2160 static void resv_map_put(struct vm_area_struct *vma)
2161 {
2162         struct resv_map *reservations = vma_resv_map(vma);
2163
2164         if (!reservations)
2165                 return;
2166         kref_put(&reservations->refs, resv_map_release);
2167 }
2168
2169 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2170 {
2171         struct hstate *h = hstate_vma(vma);
2172         struct resv_map *reservations = vma_resv_map(vma);
2173         struct hugepage_subpool *spool = subpool_vma(vma);
2174         unsigned long reserve;
2175         unsigned long start;
2176         unsigned long end;
2177
2178         if (reservations) {
2179                 start = vma_hugecache_offset(h, vma, vma->vm_start);
2180                 end = vma_hugecache_offset(h, vma, vma->vm_end);
2181
2182                 reserve = (end - start) -
2183                         region_count(&reservations->regions, start, end);
2184
2185                 resv_map_put(vma);
2186
2187                 if (reserve) {
2188                         hugetlb_acct_memory(h, -reserve);
2189                         hugepage_subpool_put_pages(spool, reserve);
2190                 }
2191         }
2192 }
2193
2194 /*
2195  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2196  * handle_mm_fault() to try to instantiate regular-sized pages in the
2197  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2198  * this far.
2199  */
2200 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2201 {
2202         BUG();
2203         return 0;
2204 }
2205
2206 const struct vm_operations_struct hugetlb_vm_ops = {
2207         .fault = hugetlb_vm_op_fault,
2208         .open = hugetlb_vm_op_open,
2209         .close = hugetlb_vm_op_close,
2210 };
2211
2212 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2213                                 int writable)
2214 {
2215         pte_t entry;
2216
2217         if (writable) {
2218                 entry =
2219                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2220         } else {
2221                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2222         }
2223         entry = pte_mkyoung(entry);
2224         entry = pte_mkhuge(entry);
2225         entry = arch_make_huge_pte(entry, vma, page, writable);
2226
2227         return entry;
2228 }
2229
2230 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2231                                    unsigned long address, pte_t *ptep)
2232 {
2233         pte_t entry;
2234
2235         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2236         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2237                 update_mmu_cache(vma, address, ptep);
2238 }
2239
2240
2241 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2242                             struct vm_area_struct *vma)
2243 {
2244         pte_t *src_pte, *dst_pte, entry;
2245         struct page *ptepage;
2246         unsigned long addr;
2247         int cow;
2248         struct hstate *h = hstate_vma(vma);
2249         unsigned long sz = huge_page_size(h);
2250
2251         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2252
2253         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2254                 src_pte = huge_pte_offset(src, addr);
2255                 if (!src_pte)
2256                         continue;
2257                 dst_pte = huge_pte_alloc(dst, addr, sz);
2258                 if (!dst_pte)
2259                         goto nomem;
2260
2261                 /* If the pagetables are shared don't copy or take references */
2262                 if (dst_pte == src_pte)
2263                         continue;
2264
2265                 spin_lock(&dst->page_table_lock);
2266                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2267                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
2268                         if (cow)
2269                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2270                         entry = huge_ptep_get(src_pte);
2271                         ptepage = pte_page(entry);
2272                         get_page(ptepage);
2273                         page_dup_rmap(ptepage);
2274                         set_huge_pte_at(dst, addr, dst_pte, entry);
2275                 }
2276                 spin_unlock(&src->page_table_lock);
2277                 spin_unlock(&dst->page_table_lock);
2278         }
2279         return 0;
2280
2281 nomem:
2282         return -ENOMEM;
2283 }
2284
2285 static int is_hugetlb_entry_migration(pte_t pte)
2286 {
2287         swp_entry_t swp;
2288
2289         if (huge_pte_none(pte) || pte_present(pte))
2290                 return 0;
2291         swp = pte_to_swp_entry(pte);
2292         if (non_swap_entry(swp) && is_migration_entry(swp))
2293                 return 1;
2294         else
2295                 return 0;
2296 }
2297
2298 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2299 {
2300         swp_entry_t swp;
2301
2302         if (huge_pte_none(pte) || pte_present(pte))
2303                 return 0;
2304         swp = pte_to_swp_entry(pte);
2305         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2306                 return 1;
2307         else
2308                 return 0;
2309 }
2310
2311 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2312                             unsigned long end, struct page *ref_page)
2313 {
2314         struct mm_struct *mm = vma->vm_mm;
2315         unsigned long address;
2316         pte_t *ptep;
2317         pte_t pte;
2318         struct page *page;
2319         struct page *tmp;
2320         struct hstate *h = hstate_vma(vma);
2321         unsigned long sz = huge_page_size(h);
2322
2323         /*
2324          * A page gathering list, protected by per file i_mmap_mutex. The
2325          * lock is used to avoid list corruption from multiple unmapping
2326          * of the same page since we are using page->lru.
2327          */
2328         LIST_HEAD(page_list);
2329
2330         WARN_ON(!is_vm_hugetlb_page(vma));
2331         BUG_ON(start & ~huge_page_mask(h));
2332         BUG_ON(end & ~huge_page_mask(h));
2333
2334         mmu_notifier_invalidate_range_start(mm, start, end);
2335         spin_lock(&mm->page_table_lock);
2336         for (address = start; address < end; address += sz) {
2337                 ptep = huge_pte_offset(mm, address);
2338                 if (!ptep)
2339                         continue;
2340
2341                 if (huge_pmd_unshare(mm, &address, ptep))
2342                         continue;
2343
2344                 pte = huge_ptep_get(ptep);
2345                 if (huge_pte_none(pte))
2346                         continue;
2347
2348                 /*
2349                  * HWPoisoned hugepage is already unmapped and dropped reference
2350                  */
2351                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte)))
2352                         continue;
2353
2354                 page = pte_page(pte);
2355                 /*
2356                  * If a reference page is supplied, it is because a specific
2357                  * page is being unmapped, not a range. Ensure the page we
2358                  * are about to unmap is the actual page of interest.
2359                  */
2360                 if (ref_page) {
2361                         if (page != ref_page)
2362                                 continue;
2363
2364                         /*
2365                          * Mark the VMA as having unmapped its page so that
2366                          * future faults in this VMA will fail rather than
2367                          * looking like data was lost
2368                          */
2369                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2370                 }
2371
2372                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2373                 if (pte_dirty(pte))
2374                         set_page_dirty(page);
2375                 list_add(&page->lru, &page_list);
2376
2377                 /* Bail out after unmapping reference page if supplied */
2378                 if (ref_page)
2379                         break;
2380         }
2381         flush_tlb_range(vma, start, end);
2382         spin_unlock(&mm->page_table_lock);
2383         mmu_notifier_invalidate_range_end(mm, start, end);
2384         list_for_each_entry_safe(page, tmp, &page_list, lru) {
2385                 page_remove_rmap(page);
2386                 list_del(&page->lru);
2387                 put_page(page);
2388         }
2389 }
2390
2391 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2392                           unsigned long end, struct page *ref_page)
2393 {
2394         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2395         __unmap_hugepage_range(vma, start, end, ref_page);
2396         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2397 }
2398
2399 /*
2400  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2401  * mappping it owns the reserve page for. The intention is to unmap the page
2402  * from other VMAs and let the children be SIGKILLed if they are faulting the
2403  * same region.
2404  */
2405 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2406                                 struct page *page, unsigned long address)
2407 {
2408         struct hstate *h = hstate_vma(vma);
2409         struct vm_area_struct *iter_vma;
2410         struct address_space *mapping;
2411         struct prio_tree_iter iter;
2412         pgoff_t pgoff;
2413
2414         /*
2415          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2416          * from page cache lookup which is in HPAGE_SIZE units.
2417          */
2418         address = address & huge_page_mask(h);
2419         pgoff = vma_hugecache_offset(h, vma, address);
2420         mapping = vma->vm_file->f_dentry->d_inode->i_mapping;
2421
2422         /*
2423          * Take the mapping lock for the duration of the table walk. As
2424          * this mapping should be shared between all the VMAs,
2425          * __unmap_hugepage_range() is called as the lock is already held
2426          */
2427         mutex_lock(&mapping->i_mmap_mutex);
2428         vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2429                 /* Do not unmap the current VMA */
2430                 if (iter_vma == vma)
2431                         continue;
2432
2433                 /*
2434                  * Unmap the page from other VMAs without their own reserves.
2435                  * They get marked to be SIGKILLed if they fault in these
2436                  * areas. This is because a future no-page fault on this VMA
2437                  * could insert a zeroed page instead of the data existing
2438                  * from the time of fork. This would look like data corruption
2439                  */
2440                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2441                         __unmap_hugepage_range(iter_vma,
2442                                 address, address + huge_page_size(h),
2443                                 page);
2444         }
2445         mutex_unlock(&mapping->i_mmap_mutex);
2446
2447         return 1;
2448 }
2449
2450 /*
2451  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2452  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2453  * cannot race with other handlers or page migration.
2454  * Keep the pte_same checks anyway to make transition from the mutex easier.
2455  */
2456 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2457                         unsigned long address, pte_t *ptep, pte_t pte,
2458                         struct page *pagecache_page)
2459 {
2460         struct hstate *h = hstate_vma(vma);
2461         struct page *old_page, *new_page;
2462         int avoidcopy;
2463         int outside_reserve = 0;
2464
2465         old_page = pte_page(pte);
2466
2467 retry_avoidcopy:
2468         /* If no-one else is actually using this page, avoid the copy
2469          * and just make the page writable */
2470         avoidcopy = (page_mapcount(old_page) == 1);
2471         if (avoidcopy) {
2472                 if (PageAnon(old_page))
2473                         page_move_anon_rmap(old_page, vma, address);
2474                 set_huge_ptep_writable(vma, address, ptep);
2475                 return 0;
2476         }
2477
2478         /*
2479          * If the process that created a MAP_PRIVATE mapping is about to
2480          * perform a COW due to a shared page count, attempt to satisfy
2481          * the allocation without using the existing reserves. The pagecache
2482          * page is used to determine if the reserve at this address was
2483          * consumed or not. If reserves were used, a partial faulted mapping
2484          * at the time of fork() could consume its reserves on COW instead
2485          * of the full address range.
2486          */
2487         if (!(vma->vm_flags & VM_MAYSHARE) &&
2488                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2489                         old_page != pagecache_page)
2490                 outside_reserve = 1;
2491
2492         page_cache_get(old_page);
2493
2494         /* Drop page_table_lock as buddy allocator may be called */
2495         spin_unlock(&mm->page_table_lock);
2496         new_page = alloc_huge_page(vma, address, outside_reserve);
2497
2498         if (IS_ERR(new_page)) {
2499                 page_cache_release(old_page);
2500
2501                 /*
2502                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2503                  * it is due to references held by a child and an insufficient
2504                  * huge page pool. To guarantee the original mappers
2505                  * reliability, unmap the page from child processes. The child
2506                  * may get SIGKILLed if it later faults.
2507                  */
2508                 if (outside_reserve) {
2509                         BUG_ON(huge_pte_none(pte));
2510                         if (unmap_ref_private(mm, vma, old_page, address)) {
2511                                 BUG_ON(huge_pte_none(pte));
2512                                 spin_lock(&mm->page_table_lock);
2513                                 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2514                                 if (likely(pte_same(huge_ptep_get(ptep), pte)))
2515                                         goto retry_avoidcopy;
2516                                 /*
2517                                  * race occurs while re-acquiring page_table_lock, and
2518                                  * our job is done.
2519                                  */
2520                                 return 0;
2521                         }
2522                         WARN_ON_ONCE(1);
2523                 }
2524
2525                 /* Caller expects lock to be held */
2526                 spin_lock(&mm->page_table_lock);
2527                 return -PTR_ERR(new_page);
2528         }
2529
2530         /*
2531          * When the original hugepage is shared one, it does not have
2532          * anon_vma prepared.
2533          */
2534         if (unlikely(anon_vma_prepare(vma))) {
2535                 page_cache_release(new_page);
2536                 page_cache_release(old_page);
2537                 /* Caller expects lock to be held */
2538                 spin_lock(&mm->page_table_lock);
2539                 return VM_FAULT_OOM;
2540         }
2541
2542         copy_user_huge_page(new_page, old_page, address, vma,
2543                             pages_per_huge_page(h));
2544         __SetPageUptodate(new_page);
2545
2546         /*
2547          * Retake the page_table_lock to check for racing updates
2548          * before the page tables are altered
2549          */
2550         spin_lock(&mm->page_table_lock);
2551         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2552         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2553                 /* Break COW */
2554                 mmu_notifier_invalidate_range_start(mm,
2555                         address & huge_page_mask(h),
2556                         (address & huge_page_mask(h)) + huge_page_size(h));
2557                 huge_ptep_clear_flush(vma, address, ptep);
2558                 set_huge_pte_at(mm, address, ptep,
2559                                 make_huge_pte(vma, new_page, 1));
2560                 page_remove_rmap(old_page);
2561                 hugepage_add_new_anon_rmap(new_page, vma, address);
2562                 /* Make the old page be freed below */
2563                 new_page = old_page;
2564                 mmu_notifier_invalidate_range_end(mm,
2565                         address & huge_page_mask(h),
2566                         (address & huge_page_mask(h)) + huge_page_size(h));
2567         }
2568         page_cache_release(new_page);
2569         page_cache_release(old_page);
2570         return 0;
2571 }
2572
2573 /* Return the pagecache page at a given address within a VMA */
2574 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2575                         struct vm_area_struct *vma, unsigned long address)
2576 {
2577         struct address_space *mapping;
2578         pgoff_t idx;
2579
2580         mapping = vma->vm_file->f_mapping;
2581         idx = vma_hugecache_offset(h, vma, address);
2582
2583         return find_lock_page(mapping, idx);
2584 }
2585
2586 /*
2587  * Return whether there is a pagecache page to back given address within VMA.
2588  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2589  */
2590 static bool hugetlbfs_pagecache_present(struct hstate *h,
2591                         struct vm_area_struct *vma, unsigned long address)
2592 {
2593         struct address_space *mapping;
2594         pgoff_t idx;
2595         struct page *page;
2596
2597         mapping = vma->vm_file->f_mapping;
2598         idx = vma_hugecache_offset(h, vma, address);
2599
2600         page = find_get_page(mapping, idx);
2601         if (page)
2602                 put_page(page);
2603         return page != NULL;
2604 }
2605
2606 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2607                         unsigned long address, pte_t *ptep, unsigned int flags)
2608 {
2609         struct hstate *h = hstate_vma(vma);
2610         int ret = VM_FAULT_SIGBUS;
2611         int anon_rmap = 0;
2612         pgoff_t idx;
2613         unsigned long size;
2614         struct page *page;
2615         struct address_space *mapping;
2616         pte_t new_pte;
2617
2618         /*
2619          * Currently, we are forced to kill the process in the event the
2620          * original mapper has unmapped pages from the child due to a failed
2621          * COW. Warn that such a situation has occurred as it may not be obvious
2622          */
2623         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2624                 printk(KERN_WARNING
2625                         "PID %d killed due to inadequate hugepage pool\n",
2626                         current->pid);
2627                 return ret;
2628         }
2629
2630         mapping = vma->vm_file->f_mapping;
2631         idx = vma_hugecache_offset(h, vma, address);
2632
2633         /*
2634          * Use page lock to guard against racing truncation
2635          * before we get page_table_lock.
2636          */
2637 retry:
2638         page = find_lock_page(mapping, idx);
2639         if (!page) {
2640                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2641                 if (idx >= size)
2642                         goto out;
2643                 page = alloc_huge_page(vma, address, 0);
2644                 if (IS_ERR(page)) {
2645                         ret = -PTR_ERR(page);
2646                         goto out;
2647                 }
2648                 clear_huge_page(page, address, pages_per_huge_page(h));
2649                 __SetPageUptodate(page);
2650
2651                 if (vma->vm_flags & VM_MAYSHARE) {
2652                         int err;
2653                         struct inode *inode = mapping->host;
2654
2655                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2656                         if (err) {
2657                                 put_page(page);
2658                                 if (err == -EEXIST)
2659                                         goto retry;
2660                                 goto out;
2661                         }
2662
2663                         spin_lock(&inode->i_lock);
2664                         inode->i_blocks += blocks_per_huge_page(h);
2665                         spin_unlock(&inode->i_lock);
2666                 } else {
2667                         lock_page(page);
2668                         if (unlikely(anon_vma_prepare(vma))) {
2669                                 ret = VM_FAULT_OOM;
2670                                 goto backout_unlocked;
2671                         }
2672                         anon_rmap = 1;
2673                 }
2674         } else {
2675                 /*
2676                  * If memory error occurs between mmap() and fault, some process
2677                  * don't have hwpoisoned swap entry for errored virtual address.
2678                  * So we need to block hugepage fault by PG_hwpoison bit check.
2679                  */
2680                 if (unlikely(PageHWPoison(page))) {
2681                         ret = VM_FAULT_HWPOISON |
2682                               VM_FAULT_SET_HINDEX(h - hstates);
2683                         goto backout_unlocked;
2684                 }
2685         }
2686
2687         /*
2688          * If we are going to COW a private mapping later, we examine the
2689          * pending reservations for this page now. This will ensure that
2690          * any allocations necessary to record that reservation occur outside
2691          * the spinlock.
2692          */
2693         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2694                 if (vma_needs_reservation(h, vma, address) < 0) {
2695                         ret = VM_FAULT_OOM;
2696                         goto backout_unlocked;
2697                 }
2698
2699         spin_lock(&mm->page_table_lock);
2700         size = i_size_read(mapping->host) >> huge_page_shift(h);
2701         if (idx >= size)
2702                 goto backout;
2703
2704         ret = 0;
2705         if (!huge_pte_none(huge_ptep_get(ptep)))
2706                 goto backout;
2707
2708         if (anon_rmap)
2709                 hugepage_add_new_anon_rmap(page, vma, address);
2710         else
2711                 page_dup_rmap(page);
2712         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2713                                 && (vma->vm_flags & VM_SHARED)));
2714         set_huge_pte_at(mm, address, ptep, new_pte);
2715
2716         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2717                 /* Optimization, do the COW without a second fault */
2718                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2719         }
2720
2721         spin_unlock(&mm->page_table_lock);
2722         unlock_page(page);
2723 out:
2724         return ret;
2725
2726 backout:
2727         spin_unlock(&mm->page_table_lock);
2728 backout_unlocked:
2729         unlock_page(page);
2730         put_page(page);
2731         goto out;
2732 }
2733
2734 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2735                         unsigned long address, unsigned int flags)
2736 {
2737         pte_t *ptep;
2738         pte_t entry;
2739         int ret;
2740         struct page *page = NULL;
2741         struct page *pagecache_page = NULL;
2742         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2743         struct hstate *h = hstate_vma(vma);
2744
2745         address &= huge_page_mask(h);
2746
2747         ptep = huge_pte_offset(mm, address);
2748         if (ptep) {
2749                 entry = huge_ptep_get(ptep);
2750                 if (unlikely(is_hugetlb_entry_migration(entry))) {
2751                         migration_entry_wait(mm, (pmd_t *)ptep, address);
2752                         return 0;
2753                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2754                         return VM_FAULT_HWPOISON_LARGE |
2755                                VM_FAULT_SET_HINDEX(h - hstates);
2756         }
2757
2758         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2759         if (!ptep)
2760                 return VM_FAULT_OOM;
2761
2762         /*
2763          * Serialize hugepage allocation and instantiation, so that we don't
2764          * get spurious allocation failures if two CPUs race to instantiate
2765          * the same page in the page cache.
2766          */
2767         mutex_lock(&hugetlb_instantiation_mutex);
2768         entry = huge_ptep_get(ptep);
2769         if (huge_pte_none(entry)) {
2770                 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2771                 goto out_mutex;
2772         }
2773
2774         ret = 0;
2775
2776         /*
2777          * If we are going to COW the mapping later, we examine the pending
2778          * reservations for this page now. This will ensure that any
2779          * allocations necessary to record that reservation occur outside the
2780          * spinlock. For private mappings, we also lookup the pagecache
2781          * page now as it is used to determine if a reservation has been
2782          * consumed.
2783          */
2784         if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2785                 if (vma_needs_reservation(h, vma, address) < 0) {
2786                         ret = VM_FAULT_OOM;
2787                         goto out_mutex;
2788                 }
2789
2790                 if (!(vma->vm_flags & VM_MAYSHARE))
2791                         pagecache_page = hugetlbfs_pagecache_page(h,
2792                                                                 vma, address);
2793         }
2794
2795         /*
2796          * hugetlb_cow() requires page locks of pte_page(entry) and
2797          * pagecache_page, so here we need take the former one
2798          * when page != pagecache_page or !pagecache_page.
2799          * Note that locking order is always pagecache_page -> page,
2800          * so no worry about deadlock.
2801          */
2802         page = pte_page(entry);
2803         get_page(page);
2804         if (page != pagecache_page)
2805                 lock_page(page);
2806
2807         spin_lock(&mm->page_table_lock);
2808         /* Check for a racing update before calling hugetlb_cow */
2809         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2810                 goto out_page_table_lock;
2811
2812
2813         if (flags & FAULT_FLAG_WRITE) {
2814                 if (!pte_write(entry)) {
2815                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
2816                                                         pagecache_page);
2817                         goto out_page_table_lock;
2818                 }
2819                 entry = pte_mkdirty(entry);
2820         }
2821         entry = pte_mkyoung(entry);
2822         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2823                                                 flags & FAULT_FLAG_WRITE))
2824                 update_mmu_cache(vma, address, ptep);
2825
2826 out_page_table_lock:
2827         spin_unlock(&mm->page_table_lock);
2828
2829         if (pagecache_page) {
2830                 unlock_page(pagecache_page);
2831                 put_page(pagecache_page);
2832         }
2833         if (page != pagecache_page)
2834                 unlock_page(page);
2835         put_page(page);
2836
2837 out_mutex:
2838         mutex_unlock(&hugetlb_instantiation_mutex);
2839
2840         return ret;
2841 }
2842
2843 /* Can be overriden by architectures */
2844 __attribute__((weak)) struct page *
2845 follow_huge_pud(struct mm_struct *mm, unsigned long address,
2846                pud_t *pud, int write)
2847 {
2848         BUG();
2849         return NULL;
2850 }
2851
2852 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2853                         struct page **pages, struct vm_area_struct **vmas,
2854                         unsigned long *position, int *length, int i,
2855                         unsigned int flags)
2856 {
2857         unsigned long pfn_offset;
2858         unsigned long vaddr = *position;
2859         int remainder = *length;
2860         struct hstate *h = hstate_vma(vma);
2861
2862         spin_lock(&mm->page_table_lock);
2863         while (vaddr < vma->vm_end && remainder) {
2864                 pte_t *pte;
2865                 int absent;
2866                 struct page *page;
2867
2868                 /*
2869                  * Some archs (sparc64, sh*) have multiple pte_ts to
2870                  * each hugepage.  We have to make sure we get the
2871                  * first, for the page indexing below to work.
2872                  */
2873                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
2874                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
2875
2876                 /*
2877                  * When coredumping, it suits get_dump_page if we just return
2878                  * an error where there's an empty slot with no huge pagecache
2879                  * to back it.  This way, we avoid allocating a hugepage, and
2880                  * the sparse dumpfile avoids allocating disk blocks, but its
2881                  * huge holes still show up with zeroes where they need to be.
2882                  */
2883                 if (absent && (flags & FOLL_DUMP) &&
2884                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
2885                         remainder = 0;
2886                         break;
2887                 }
2888
2889                 if (absent ||
2890                     ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2891                         int ret;
2892
2893                         spin_unlock(&mm->page_table_lock);
2894                         ret = hugetlb_fault(mm, vma, vaddr,
2895                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2896                         spin_lock(&mm->page_table_lock);
2897                         if (!(ret & VM_FAULT_ERROR))
2898                                 continue;
2899
2900                         remainder = 0;
2901                         break;
2902                 }
2903
2904                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
2905                 page = pte_page(huge_ptep_get(pte));
2906 same_page:
2907                 if (pages) {
2908                         pages[i] = mem_map_offset(page, pfn_offset);
2909                         get_page(pages[i]);
2910                 }
2911
2912                 if (vmas)
2913                         vmas[i] = vma;
2914
2915                 vaddr += PAGE_SIZE;
2916                 ++pfn_offset;
2917                 --remainder;
2918                 ++i;
2919                 if (vaddr < vma->vm_end && remainder &&
2920                                 pfn_offset < pages_per_huge_page(h)) {
2921                         /*
2922                          * We use pfn_offset to avoid touching the pageframes
2923                          * of this compound page.
2924                          */
2925                         goto same_page;
2926                 }
2927         }
2928         spin_unlock(&mm->page_table_lock);
2929         *length = remainder;
2930         *position = vaddr;
2931
2932         return i ? i : -EFAULT;
2933 }
2934
2935 void hugetlb_change_protection(struct vm_area_struct *vma,
2936                 unsigned long address, unsigned long end, pgprot_t newprot)
2937 {
2938         struct mm_struct *mm = vma->vm_mm;
2939         unsigned long start = address;
2940         pte_t *ptep;
2941         pte_t pte;
2942         struct hstate *h = hstate_vma(vma);
2943
2944         BUG_ON(address >= end);
2945         flush_cache_range(vma, address, end);
2946
2947         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2948         spin_lock(&mm->page_table_lock);
2949         for (; address < end; address += huge_page_size(h)) {
2950                 ptep = huge_pte_offset(mm, address);
2951                 if (!ptep)
2952                         continue;
2953                 if (huge_pmd_unshare(mm, &address, ptep))
2954                         continue;
2955                 if (!huge_pte_none(huge_ptep_get(ptep))) {
2956                         pte = huge_ptep_get_and_clear(mm, address, ptep);
2957                         pte = pte_mkhuge(pte_modify(pte, newprot));
2958                         set_huge_pte_at(mm, address, ptep, pte);
2959                 }
2960         }
2961         spin_unlock(&mm->page_table_lock);
2962         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2963
2964         flush_tlb_range(vma, start, end);
2965 }
2966
2967 int hugetlb_reserve_pages(struct inode *inode,
2968                                         long from, long to,
2969                                         struct vm_area_struct *vma,
2970                                         vm_flags_t vm_flags)
2971 {
2972         long ret, chg;
2973         struct hstate *h = hstate_inode(inode);
2974         struct hugepage_subpool *spool = subpool_inode(inode);
2975
2976         /*
2977          * Only apply hugepage reservation if asked. At fault time, an
2978          * attempt will be made for VM_NORESERVE to allocate a page
2979          * without using reserves
2980          */
2981         if (vm_flags & VM_NORESERVE)
2982                 return 0;
2983
2984         /*
2985          * Shared mappings base their reservation on the number of pages that
2986          * are already allocated on behalf of the file. Private mappings need
2987          * to reserve the full area even if read-only as mprotect() may be
2988          * called to make the mapping read-write. Assume !vma is a shm mapping
2989          */
2990         if (!vma || vma->vm_flags & VM_MAYSHARE)
2991                 chg = region_chg(&inode->i_mapping->private_list, from, to);
2992         else {
2993                 struct resv_map *resv_map = resv_map_alloc();
2994                 if (!resv_map)
2995                         return -ENOMEM;
2996
2997                 chg = to - from;
2998
2999                 set_vma_resv_map(vma, resv_map);
3000                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3001         }
3002
3003         if (chg < 0) {
3004                 ret = chg;
3005                 goto out_err;
3006         }
3007
3008         /* There must be enough pages in the subpool for the mapping */
3009         if (hugepage_subpool_get_pages(spool, chg)) {
3010                 ret = -ENOSPC;
3011                 goto out_err;
3012         }
3013
3014         /*
3015          * Check enough hugepages are available for the reservation.
3016          * Hand the pages back to the subpool if there are not
3017          */
3018         ret = hugetlb_acct_memory(h, chg);
3019         if (ret < 0) {
3020                 hugepage_subpool_put_pages(spool, chg);
3021                 goto out_err;
3022         }
3023
3024         /*
3025          * Account for the reservations made. Shared mappings record regions
3026          * that have reservations as they are shared by multiple VMAs.
3027          * When the last VMA disappears, the region map says how much
3028          * the reservation was and the page cache tells how much of
3029          * the reservation was consumed. Private mappings are per-VMA and
3030          * only the consumed reservations are tracked. When the VMA
3031          * disappears, the original reservation is the VMA size and the
3032          * consumed reservations are stored in the map. Hence, nothing
3033          * else has to be done for private mappings here
3034          */
3035         if (!vma || vma->vm_flags & VM_MAYSHARE)
3036                 region_add(&inode->i_mapping->private_list, from, to);
3037         return 0;
3038 out_err:
3039         if (vma)
3040                 resv_map_put(vma);
3041         return ret;
3042 }
3043
3044 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3045 {
3046         struct hstate *h = hstate_inode(inode);
3047         long chg = region_truncate(&inode->i_mapping->private_list, offset);
3048         struct hugepage_subpool *spool = subpool_inode(inode);
3049
3050         spin_lock(&inode->i_lock);
3051         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3052         spin_unlock(&inode->i_lock);
3053
3054         hugepage_subpool_put_pages(spool, (chg - freed));
3055         hugetlb_acct_memory(h, -(chg - freed));
3056 }
3057
3058 #ifdef CONFIG_MEMORY_FAILURE
3059
3060 /* Should be called in hugetlb_lock */
3061 static int is_hugepage_on_freelist(struct page *hpage)
3062 {
3063         struct page *page;
3064         struct page *tmp;
3065         struct hstate *h = page_hstate(hpage);
3066         int nid = page_to_nid(hpage);
3067
3068         list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3069                 if (page == hpage)
3070                         return 1;
3071         return 0;
3072 }
3073
3074 /*
3075  * This function is called from memory failure code.
3076  * Assume the caller holds page lock of the head page.
3077  */
3078 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3079 {
3080         struct hstate *h = page_hstate(hpage);
3081         int nid = page_to_nid(hpage);
3082         int ret = -EBUSY;
3083
3084         spin_lock(&hugetlb_lock);
3085         if (is_hugepage_on_freelist(hpage)) {
3086                 list_del(&hpage->lru);
3087                 set_page_refcounted(hpage);
3088                 h->free_huge_pages--;
3089                 h->free_huge_pages_node[nid]--;
3090                 ret = 0;
3091         }
3092         spin_unlock(&hugetlb_lock);
3093         return ret;
3094 }
3095 #endif