mm/hugetlb.c

   1 /*
   2  * Generic hugetlb support.
   3  * (C) Nadia Yvette Chambers, April 2004
   4  */
   5 #include <linux/list.h>
   6 #include <linux/init.h>
   7 #include <linux/module.h>
   8 #include <linux/mm.h>
   9 #include <linux/seq_file.h>
  10 #include <linux/sysctl.h>
  11 #include <linux/highmem.h>
  12 #include <linux/mmu_notifier.h>
  13 #include <linux/nodemask.h>
  14 #include <linux/pagemap.h>
  15 #include <linux/mempolicy.h>
  16 #include <linux/cpuset.h>
  17 #include <linux/mutex.h>
  18 #include <linux/bootmem.h>
  19 #include <linux/sysfs.h>
  20 #include <linux/slab.h>
  21 #include <linux/rmap.h>
  22 #include <linux/swap.h>
  23 #include <linux/swapops.h>
  24 #include <linux/page-isolation.h>
  25
  26 #include <asm/page.h>
  27 #include <asm/pgtable.h>
  28 #include <asm/tlb.h>
  29
  30 #include <linux/io.h>
  31 #include <linux/hugetlb.h>
  32 #include <linux/hugetlb_cgroup.h>
  33 #include <linux/node.h>
  34 #include "internal.h"
  35
  36 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  37 unsigned long hugepages_treat_as_movable;
  38
  39 int hugetlb_max_hstate __read_mostly;
  40 unsigned int default_hstate_idx;
  41 struct hstate hstates[HUGE_MAX_HSTATE];
  42
  43 __initdata LIST_HEAD(huge_boot_pages);
  44
  45 /* for command line parsing */
  46 static struct hstate * __initdata parsed_hstate;
  47 static unsigned long __initdata default_hstate_max_huge_pages;
  48 static unsigned long __initdata default_hstate_size;
  49
  50 /*
  51  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  52  * free_huge_pages, and surplus_huge_pages.
  53  */
  54 DEFINE_SPINLOCK(hugetlb_lock);
  55
  56 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  57 {
  58         bool free = (spool->count == 0) && (spool->used_hpages == 0);
  59
  60         spin_unlock(&spool->lock);
  61
  62         /* If no pages are used, and no other handles to the subpool
  63          * remain, free the subpool the subpool remain */
  64         if (free)
  65                 kfree(spool);
  66 }
  67
  68 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
  69 {
  70         struct hugepage_subpool *spool;
  71
  72         spool = kmalloc(sizeof(*spool), GFP_KERNEL);
  73         if (!spool)
  74                 return NULL;
  75
  76         spin_lock_init(&spool->lock);
  77         spool->count = 1;
  78         spool->max_hpages = nr_blocks;
  79         spool->used_hpages = 0;
  80
  81         return spool;
  82 }
  83
  84 void hugepage_put_subpool(struct hugepage_subpool *spool)
  85 {
  86         spin_lock(&spool->lock);
  87         BUG_ON(!spool->count);
  88         spool->count--;
  89         unlock_or_release_subpool(spool);
  90 }
  91
  92 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
  93                                       long delta)
  94 {
  95         int ret = 0;
  96
  97         if (!spool)
  98                 return 0;
  99
 100         spin_lock(&spool->lock);
 101         if ((spool->used_hpages + delta) <= spool->max_hpages) {
 102                 spool->used_hpages += delta;
 103         } else {
 104                 ret = -ENOMEM;
 105         }
 106         spin_unlock(&spool->lock);
 107
 108         return ret;
 109 }
 110
 111 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 112                                        long delta)
 113 {
 114         if (!spool)
 115                 return;
 116
 117         spin_lock(&spool->lock);
 118         spool->used_hpages -= delta;
 119         /* If hugetlbfs_put_super couldn't free spool due to
 120         * an outstanding quota reference, free it now. */
 121         unlock_or_release_subpool(spool);
 122 }
 123
 124 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 125 {
 126         return HUGETLBFS_SB(inode->i_sb)->spool;
 127 }
 128
 129 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 130 {
 131         return subpool_inode(file_inode(vma->vm_file));
 132 }
 133
 134 /*
 135  * Region tracking -- allows tracking of reservations and instantiated pages
 136  *                    across the pages in a mapping.
 137  *
 138  * The region data structures are protected by a combination of the mmap_sem
 139  * and the hugetlb_instantiation_mutex.  To access or modify a region the caller
 140  * must either hold the mmap_sem for write, or the mmap_sem for read and
 141  * the hugetlb_instantiation_mutex:
 142  *
 143  *      down_write(&mm->mmap_sem);
 144  * or
 145  *      down_read(&mm->mmap_sem);
 146  *      mutex_lock(&hugetlb_instantiation_mutex);
 147  */
 148 struct file_region {
 149         struct list_head link;
 150         long from;
 151         long to;
 152 };
 153
 154 static long region_add(struct list_head *head, long f, long t)
 155 {
 156         struct file_region *rg, *nrg, *trg;
 157
 158         /* Locate the region we are either in or before. */
 159         list_for_each_entry(rg, head, link)
 160                 if (f <= rg->to)
 161                         break;
 162
 163         /* Round our left edge to the current segment if it encloses us. */
 164         if (f > rg->from)
 165                 f = rg->from;
 166
 167         /* Check for and consume any regions we now overlap with. */
 168         nrg = rg;
 169         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 170                 if (&rg->link == head)
 171                         break;
 172                 if (rg->from > t)
 173                         break;
 174
 175                 /* If this area reaches higher then extend our area to
 176                  * include it completely.  If this is not the first area
 177                  * which we intend to reuse, free it. */
 178                 if (rg->to > t)
 179                         t = rg->to;
 180                 if (rg != nrg) {
 181                         list_del(&rg->link);
 182                         kfree(rg);
 183                 }
 184         }
 185         nrg->from = f;
 186         nrg->to = t;
 187         return 0;
 188 }
 189
 190 static long region_chg(struct list_head *head, long f, long t)
 191 {
 192         struct file_region *rg, *nrg;
 193         long chg = 0;
 194
 195         /* Locate the region we are before or in. */
 196         list_for_each_entry(rg, head, link)
 197                 if (f <= rg->to)
 198                         break;
 199
 200         /* If we are below the current region then a new region is required.
 201          * Subtle, allocate a new region at the position but make it zero
 202          * size such that we can guarantee to record the reservation. */
 203         if (&rg->link == head || t < rg->from) {
 204                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 205                 if (!nrg)
 206                         return -ENOMEM;
 207                 nrg->from = f;
 208                 nrg->to   = f;
 209                 INIT_LIST_HEAD(&nrg->link);
 210                 list_add(&nrg->link, rg->link.prev);
 211
 212                 return t - f;
 213         }
 214
 215         /* Round our left edge to the current segment if it encloses us. */
 216         if (f > rg->from)
 217                 f = rg->from;
 218         chg = t - f;
 219
 220         /* Check for and consume any regions we now overlap with. */
 221         list_for_each_entry(rg, rg->link.prev, link) {
 222                 if (&rg->link == head)
 223                         break;
 224                 if (rg->from > t)
 225                         return chg;
 226
 227                 /* We overlap with this area, if it extends further than
 228                  * us then we must extend ourselves.  Account for its
 229                  * existing reservation. */
 230                 if (rg->to > t) {
 231                         chg += rg->to - t;
 232                         t = rg->to;
 233                 }
 234                 chg -= rg->to - rg->from;
 235         }
 236         return chg;
 237 }
 238
 239 static long region_truncate(struct list_head *head, long end)
 240 {
 241         struct file_region *rg, *trg;
 242         long chg = 0;
 243
 244         /* Locate the region we are either in or before. */
 245         list_for_each_entry(rg, head, link)
 246                 if (end <= rg->to)
 247                         break;
 248         if (&rg->link == head)
 249                 return 0;
 250
 251         /* If we are in the middle of a region then adjust it. */
 252         if (end > rg->from) {
 253                 chg = rg->to - end;
 254                 rg->to = end;
 255                 rg = list_entry(rg->link.next, typeof(*rg), link);
 256         }
 257
 258         /* Drop any remaining regions. */
 259         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 260                 if (&rg->link == head)
 261                         break;
 262                 chg += rg->to - rg->from;
 263                 list_del(&rg->link);
 264                 kfree(rg);
 265         }
 266         return chg;
 267 }
 268
 269 static long region_count(struct list_head *head, long f, long t)
 270 {
 271         struct file_region *rg;
 272         long chg = 0;
 273
 274         /* Locate each segment we overlap with, and count that overlap. */
 275         list_for_each_entry(rg, head, link) {
 276                 long seg_from;
 277                 long seg_to;
 278
 279                 if (rg->to <= f)
 280                         continue;
 281                 if (rg->from >= t)
 282                         break;
 283
 284                 seg_from = max(rg->from, f);
 285                 seg_to = min(rg->to, t);
 286
 287                 chg += seg_to - seg_from;
 288         }
 289
 290         return chg;
 291 }
 292
 293 /*
 294  * Convert the address within this vma to the page offset within
 295  * the mapping, in pagecache page units; huge pages here.
 296  */
 297 static pgoff_t vma_hugecache_offset(struct hstate *h,
 298                         struct vm_area_struct *vma, unsigned long address)
 299 {
 300         return ((address - vma->vm_start) >> huge_page_shift(h)) +
 301                         (vma->vm_pgoff >> huge_page_order(h));
 302 }
 303
 304 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
 305                                      unsigned long address)
 306 {
 307         return vma_hugecache_offset(hstate_vma(vma), vma, address);
 308 }
 309
 310 /*
 311  * Return the size of the pages allocated when backing a VMA. In the majority
 312  * cases this will be same size as used by the page table entries.
 313  */
 314 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
 315 {
 316         struct hstate *hstate;
 317
 318         if (!is_vm_hugetlb_page(vma))
 319                 return PAGE_SIZE;
 320
 321         hstate = hstate_vma(vma);
 322
 323         return 1UL << huge_page_shift(hstate);
 324 }
 325 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
 326
 327 /*
 328  * Return the page size being used by the MMU to back a VMA. In the majority
 329  * of cases, the page size used by the kernel matches the MMU size. On
 330  * architectures where it differs, an architecture-specific version of this
 331  * function is required.
 332  */
 333 #ifndef vma_mmu_pagesize
 334 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 335 {
 336         return vma_kernel_pagesize(vma);
 337 }
 338 #endif
 339
 340 /*
 341  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
 342  * bits of the reservation map pointer, which are always clear due to
 343  * alignment.
 344  */
 345 #define HPAGE_RESV_OWNER    (1UL << 0)
 346 #define HPAGE_RESV_UNMAPPED (1UL << 1)
 347 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
 348
 349 /*
 350  * These helpers are used to track how many pages are reserved for
 351  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
 352  * is guaranteed to have their future faults succeed.
 353  *
 354  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
 355  * the reserve counters are updated with the hugetlb_lock held. It is safe
 356  * to reset the VMA at fork() time as it is not in use yet and there is no
 357  * chance of the global counters getting corrupted as a result of the values.
 358  *
 359  * The private mapping reservation is represented in a subtly different
 360  * manner to a shared mapping.  A shared mapping has a region map associated
 361  * with the underlying file, this region map represents the backing file
 362  * pages which have ever had a reservation assigned which this persists even
 363  * after the page is instantiated.  A private mapping has a region map
 364  * associated with the original mmap which is attached to all VMAs which
 365  * reference it, this region map represents those offsets which have consumed
 366  * reservation ie. where pages have been instantiated.
 367  */
 368 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
 369 {
 370         return (unsigned long)vma->vm_private_data;
 371 }
 372
 373 static void set_vma_private_data(struct vm_area_struct *vma,
 374                                                         unsigned long value)
 375 {
 376         vma->vm_private_data = (void *)value;
 377 }
 378
 379 struct resv_map {
 380         struct kref refs;
 381         struct list_head regions;
 382 };
 383
 384 static struct resv_map *resv_map_alloc(void)
 385 {
 386         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
 387         if (!resv_map)
 388                 return NULL;
 389
 390         kref_init(&resv_map->refs);
 391         INIT_LIST_HEAD(&resv_map->regions);
 392
 393         return resv_map;
 394 }
 395
 396 static void resv_map_release(struct kref *ref)
 397 {
 398         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
 399
 400         /* Clear out any active regions before we release the map. */
 401         region_truncate(&resv_map->regions, 0);
 402         kfree(resv_map);
 403 }
 404
 405 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 406 {
 407         VM_BUG_ON(!is_vm_hugetlb_page(vma));
 408         if (!(vma->vm_flags & VM_MAYSHARE))
 409                 return (struct resv_map *)(get_vma_private_data(vma) &
 410                                                         ~HPAGE_RESV_MASK);
 411         return NULL;
 412 }
 413
 414 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 415 {
 416         VM_BUG_ON(!is_vm_hugetlb_page(vma));
 417         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
 418
 419         set_vma_private_data(vma, (get_vma_private_data(vma) &
 420                                 HPAGE_RESV_MASK) | (unsigned long)map);
 421 }
 422
 423 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 424 {
 425         VM_BUG_ON(!is_vm_hugetlb_page(vma));
 426         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
 427
 428         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
 429 }
 430
 431 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 432 {
 433         VM_BUG_ON(!is_vm_hugetlb_page(vma));
 434
 435         return (get_vma_private_data(vma) & flag) != 0;
 436 }
 437
 438 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
 439 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
 440 {
 441         VM_BUG_ON(!is_vm_hugetlb_page(vma));
 442         if (!(vma->vm_flags & VM_MAYSHARE))
 443                 vma->vm_private_data = (void *)0;
 444 }
 445
 446 /* Returns true if the VMA has associated reserve pages */
 447 static int vma_has_reserves(struct vm_area_struct *vma, long chg)
 448 {
 449         if (vma->vm_flags & VM_NORESERVE) {
 450                 /*
 451                  * This address is already reserved by other process(chg == 0),
 452                  * so, we should decrement reserved count. Without decrementing,
 453                  * reserve count remains after releasing inode, because this
 454                  * allocated page will go into page cache and is regarded as
 455                  * coming from reserved pool in releasing step.  Currently, we
 456                  * don't have any other solution to deal with this situation
 457                  * properly, so add work-around here.
 458                  */
 459                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
 460                         return 1;
 461                 else
 462                         return 0;
 463         }
 464
 465         /* Shared mappings always use reserves */
 466         if (vma->vm_flags & VM_MAYSHARE)
 467                 return 1;
 468
 469         /*
 470          * Only the process that called mmap() has reserves for
 471          * private mappings.
 472          */
 473         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
 474                 return 1;
 475
 476         return 0;
 477 }
 478
 479 static void copy_gigantic_page(struct page *dst, struct page *src)
 480 {
 481         int i;
 482         struct hstate *h = page_hstate(src);
 483         struct page *dst_base = dst;
 484         struct page *src_base = src;
 485
 486         for (i = 0; i < pages_per_huge_page(h); ) {
 487                 cond_resched();
 488                 copy_highpage(dst, src);
 489
 490                 i++;
 491                 dst = mem_map_next(dst, dst_base, i);
 492                 src = mem_map_next(src, src_base, i);
 493         }
 494 }
 495
 496 void copy_huge_page(struct page *dst, struct page *src)
 497 {
 498         int i;
 499         struct hstate *h = page_hstate(src);
 500
 501         if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
 502                 copy_gigantic_page(dst, src);
 503                 return;
 504         }
 505
 506         might_sleep();
 507         for (i = 0; i < pages_per_huge_page(h); i++) {
 508                 cond_resched();
 509                 copy_highpage(dst + i, src + i);
 510         }
 511 }
 512
 513 static void enqueue_huge_page(struct hstate *h, struct page *page)
 514 {
 515         int nid = page_to_nid(page);
 516         list_move(&page->lru, &h->hugepage_freelists[nid]);
 517         h->free_huge_pages++;
 518         h->free_huge_pages_node[nid]++;
 519 }
 520
 521 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
 522 {
 523         struct page *page;
 524
 525         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
 526                 if (!is_migrate_isolate_page(page))
 527                         break;
 528         /*
 529          * if 'non-isolated free hugepage' not found on the list,
 530          * the allocation fails.
 531          */
 532         if (&h->hugepage_freelists[nid] == &page->lru)
 533                 return NULL;
 534         list_move(&page->lru, &h->hugepage_activelist);
 535         set_page_refcounted(page);
 536         h->free_huge_pages--;
 537         h->free_huge_pages_node[nid]--;
 538         return page;
 539 }
 540
 541 /* Movability of hugepages depends on migration support. */
 542 static inline gfp_t htlb_alloc_mask(struct hstate *h)
 543 {
 544         if (hugepages_treat_as_movable || hugepage_migration_support(h))
 545                 return GFP_HIGHUSER_MOVABLE;
 546         else
 547                 return GFP_HIGHUSER;
 548 }
 549
 550 static struct page *dequeue_huge_page_vma(struct hstate *h,
 551                                 struct vm_area_struct *vma,
 552                                 unsigned long address, int avoid_reserve,
 553                                 long chg)
 554 {
 555         struct page *page = NULL;
 556         struct mempolicy *mpol;
 557         nodemask_t *nodemask;
 558         struct zonelist *zonelist;
 559         struct zone *zone;
 560         struct zoneref *z;
 561         unsigned int cpuset_mems_cookie;
 562
 563         /*
 564          * A child process with MAP_PRIVATE mappings created by their parent
 565          * have no page reserves. This check ensures that reservations are
 566          * not "stolen". The child may still get SIGKILLed
 567          */
 568         if (!vma_has_reserves(vma, chg) &&
 569                         h->free_huge_pages - h->resv_huge_pages == 0)
 570                 goto err;
 571
 572         /* If reserves cannot be used, ensure enough pages are in the pool */
 573         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
 574                 goto err;
 575
 576 retry_cpuset:
 577         cpuset_mems_cookie = get_mems_allowed();
 578         zonelist = huge_zonelist(vma, address,
 579                                         htlb_alloc_mask(h), &mpol, &nodemask);
 580
 581         for_each_zone_zonelist_nodemask(zone, z, zonelist,
 582                                                 MAX_NR_ZONES - 1, nodemask) {
 583                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
 584                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
 585                         if (page) {
 586                                 if (avoid_reserve)
 587                                         break;
 588                                 if (!vma_has_reserves(vma, chg))
 589                                         break;
 590
 591                                 SetPagePrivate(page);
 592                                 h->resv_huge_pages--;
 593                                 break;
 594                         }
 595                 }
 596         }
 597
 598         mpol_cond_put(mpol);
 599         if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
 600                 goto retry_cpuset;
 601         return page;
 602
 603 err:
 604         return NULL;
 605 }
 606
 607 static void update_and_free_page(struct hstate *h, struct page *page)
 608 {
 609         int i;
 610
 611         VM_BUG_ON(h->order >= MAX_ORDER);
 612
 613         h->nr_huge_pages--;
 614         h->nr_huge_pages_node[page_to_nid(page)]--;
 615         for (i = 0; i < pages_per_huge_page(h); i++) {
 616                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
 617                                 1 << PG_referenced | 1 << PG_dirty |
 618                                 1 << PG_active | 1 << PG_reserved |
 619                                 1 << PG_private | 1 << PG_writeback);
 620         }
 621         VM_BUG_ON(hugetlb_cgroup_from_page(page));
 622         set_compound_page_dtor(page, NULL);
 623         set_page_refcounted(page);
 624         arch_release_hugepage(page);
 625         __free_pages(page, huge_page_order(h));
 626 }
 627
 628 struct hstate *size_to_hstate(unsigned long size)
 629 {
 630         struct hstate *h;
 631
 632         for_each_hstate(h) {
 633                 if (huge_page_size(h) == size)
 634                         return h;
 635         }
 636         return NULL;
 637 }
 638
 639 static void free_huge_page(struct page *page)
 640 {
 641         /*
 642          * Can't pass hstate in here because it is called from the
 643          * compound page destructor.
 644          */
 645         struct hstate *h = page_hstate(page);
 646         int nid = page_to_nid(page);
 647         struct hugepage_subpool *spool =
 648                 (struct hugepage_subpool *)page_private(page);
 649         bool restore_reserve;
 650
 651         set_page_private(page, 0);
 652         page->mapping = NULL;
 653         BUG_ON(page_count(page));
 654         BUG_ON(page_mapcount(page));
 655         restore_reserve = PagePrivate(page);
 656         ClearPagePrivate(page);
 657
 658         spin_lock(&hugetlb_lock);
 659         hugetlb_cgroup_uncharge_page(hstate_index(h),
 660                                      pages_per_huge_page(h), page);
 661         if (restore_reserve)
 662                 h->resv_huge_pages++;
 663
 664         if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
 665                 /* remove the page from active list */
 666                 list_del(&page->lru);
 667                 update_and_free_page(h, page);
 668                 h->surplus_huge_pages--;
 669                 h->surplus_huge_pages_node[nid]--;
 670         } else {
 671                 arch_clear_hugepage_flags(page);
 672                 enqueue_huge_page(h, page);
 673         }
 674         spin_unlock(&hugetlb_lock);
 675         hugepage_subpool_put_pages(spool, 1);
 676 }
 677
 678 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
 679 {
 680         INIT_LIST_HEAD(&page->lru);
 681         set_compound_page_dtor(page, free_huge_page);
 682         spin_lock(&hugetlb_lock);
 683         set_hugetlb_cgroup(page, NULL);
 684         h->nr_huge_pages++;
 685         h->nr_huge_pages_node[nid]++;
 686         spin_unlock(&hugetlb_lock);
 687         put_page(page); /* free it into the hugepage allocator */
 688 }
 689
 690 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
 691 {
 692         int i;
 693         int nr_pages = 1 << order;
 694         struct page *p = page + 1;
 695
 696         /* we rely on prep_new_huge_page to set the destructor */
 697         set_compound_order(page, order);
 698         __SetPageHead(page);
 699         __ClearPageReserved(page);
 700         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
 701                 __SetPageTail(p);
 702                 /*
 703                  * For gigantic hugepages allocated through bootmem at
 704                  * boot, it's safer to be consistent with the not-gigantic
 705                  * hugepages and clear the PG_reserved bit from all tail pages
 706                  * too.  Otherwse drivers using get_user_pages() to access tail
 707                  * pages may get the reference counting wrong if they see
 708                  * PG_reserved set on a tail page (despite the head page not
 709                  * having PG_reserved set).  Enforcing this consistency between
 710                  * head and tail pages allows drivers to optimize away a check
 711                  * on the head page when they need know if put_page() is needed
 712                  * after get_user_pages().
 713                  */
 714                 __ClearPageReserved(p);
 715                 set_page_count(p, 0);
 716                 p->first_page = page;
 717         }
 718 }
 719
 720 /*
 721  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
 722  * transparent huge pages.  See the PageTransHuge() documentation for more
 723  * details.
 724  */
 725 int PageHuge(struct page *page)
 726 {
 727         compound_page_dtor *dtor;
 728
 729         if (!PageCompound(page))
 730                 return 0;
 731
 732         page = compound_head(page);
 733         dtor = get_compound_page_dtor(page);
 734
 735         return dtor == free_huge_page;
 736 }
 737 EXPORT_SYMBOL_GPL(PageHuge);
 738
 739 pgoff_t __basepage_index(struct page *page)
 740 {
 741         struct page *page_head = compound_head(page);
 742         pgoff_t index = page_index(page_head);
 743         unsigned long compound_idx;
 744
 745         if (!PageHuge(page_head))
 746                 return page_index(page);
 747
 748         if (compound_order(page_head) >= MAX_ORDER)
 749                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
 750         else
 751                 compound_idx = page - page_head;
 752
 753         return (index << compound_order(page_head)) + compound_idx;
 754 }
 755
 756 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
 757 {
 758         struct page *page;
 759
 760         if (h->order >= MAX_ORDER)
 761                 return NULL;
 762
 763         page = alloc_pages_exact_node(nid,
 764                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
 765                                                 __GFP_REPEAT|__GFP_NOWARN,
 766                 huge_page_order(h));
 767         if (page) {
 768                 if (arch_prepare_hugepage(page)) {
 769                         __free_pages(page, huge_page_order(h));
 770                         return NULL;
 771                 }
 772                 prep_new_huge_page(h, page, nid);
 773         }
 774
 775         return page;
 776 }
 777
 778 /*
 779  * common helper functions for hstate_next_node_to_{alloc|free}.
 780  * We may have allocated or freed a huge page based on a different
 781  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
 782  * be outside of *nodes_allowed.  Ensure that we use an allowed
 783  * node for alloc or free.
 784  */
 785 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
 786 {
 787         nid = next_node(nid, *nodes_allowed);
 788         if (nid == MAX_NUMNODES)
 789                 nid = first_node(*nodes_allowed);
 790         VM_BUG_ON(nid >= MAX_NUMNODES);
 791
 792         return nid;
 793 }
 794
 795 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
 796 {
 797         if (!node_isset(nid, *nodes_allowed))
 798                 nid = next_node_allowed(nid, nodes_allowed);
 799         return nid;
 800 }
 801
 802 /*
 803  * returns the previously saved node ["this node"] from which to
 804  * allocate a persistent huge page for the pool and advance the
 805  * next node from which to allocate, handling wrap at end of node
 806  * mask.
 807  */
 808 static int hstate_next_node_to_alloc(struct hstate *h,
 809                                         nodemask_t *nodes_allowed)
 810 {
 811         int nid;
 812
 813         VM_BUG_ON(!nodes_allowed);
 814
 815         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
 816         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
 817
 818         return nid;
 819 }
 820
 821 /*
 822  * helper for free_pool_huge_page() - return the previously saved
 823  * node ["this node"] from which to free a huge page.  Advance the
 824  * next node id whether or not we find a free huge page to free so
 825  * that the next attempt to free addresses the next node.
 826  */
 827 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
 828 {
 829         int nid;
 830
 831         VM_BUG_ON(!nodes_allowed);
 832
 833         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
 834         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
 835
 836         return nid;
 837 }
 838
 839 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
 840         for (nr_nodes = nodes_weight(*mask);                            \
 841                 nr_nodes > 0 &&                                         \
 842                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
 843                 nr_nodes--)
 844
 845 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
 846         for (nr_nodes = nodes_weight(*mask);                            \
 847                 nr_nodes > 0 &&                                         \
 848                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
 849                 nr_nodes--)
 850
 851 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
 852 {
 853         struct page *page;
 854         int nr_nodes, node;
 855         int ret = 0;
 856
 857         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
 858                 page = alloc_fresh_huge_page_node(h, node);
 859                 if (page) {
 860                         ret = 1;
 861                         break;
 862                 }
 863         }
 864
 865         if (ret)
 866                 count_vm_event(HTLB_BUDDY_PGALLOC);
 867         else
 868                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
 869
 870         return ret;
 871 }
 872
 873 /*
 874  * Free huge page from pool from next node to free.
 875  * Attempt to keep persistent huge pages more or less
 876  * balanced over allowed nodes.
 877  * Called with hugetlb_lock locked.
 878  */
 879 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
 880                                                          bool acct_surplus)
 881 {
 882         int nr_nodes, node;
 883         int ret = 0;
 884
 885         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
 886                 /*
 887                  * If we're returning unused surplus pages, only examine
 888                  * nodes with surplus pages.
 889                  */
 890                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
 891                     !list_empty(&h->hugepage_freelists[node])) {
 892                         struct page *page =
 893                                 list_entry(h->hugepage_freelists[node].next,
 894                                           struct page, lru);
 895                         list_del(&page->lru);
 896                         h->free_huge_pages--;
 897                         h->free_huge_pages_node[node]--;
 898                         if (acct_surplus) {
 899                                 h->surplus_huge_pages--;
 900                                 h->surplus_huge_pages_node[node]--;
 901                         }
 902                         update_and_free_page(h, page);
 903                         ret = 1;
 904                         break;
 905                 }
 906         }
 907
 908         return ret;
 909 }
 910
 911 /*
 912  * Dissolve a given free hugepage into free buddy pages. This function does
 913  * nothing for in-use (including surplus) hugepages.
 914  */
 915 static void dissolve_free_huge_page(struct page *page)
 916 {
 917         spin_lock(&hugetlb_lock);
 918         if (PageHuge(page) && !page_count(page)) {
 919                 struct hstate *h = page_hstate(page);
 920                 int nid = page_to_nid(page);
 921                 list_del(&page->lru);
 922                 h->free_huge_pages--;
 923                 h->free_huge_pages_node[nid]--;
 924                 update_and_free_page(h, page);
 925         }
 926         spin_unlock(&hugetlb_lock);
 927 }
 928
 929 /*
 930  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
 931  * make specified memory blocks removable from the system.
 932  * Note that start_pfn should aligned with (minimum) hugepage size.
 933  */
 934 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
 935 {
 936         unsigned int order = 8 * sizeof(void *);
 937         unsigned long pfn;
 938         struct hstate *h;
 939
 940         /* Set scan step to minimum hugepage size */
 941         for_each_hstate(h)
 942                 if (order > huge_page_order(h))
 943                         order = huge_page_order(h);
 944         VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
 945         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
 946                 dissolve_free_huge_page(pfn_to_page(pfn));
 947 }
 948
 949 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
 950 {
 951         struct page *page;
 952         unsigned int r_nid;
 953
 954         if (h->order >= MAX_ORDER)
 955                 return NULL;
 956
 957         /*
 958          * Assume we will successfully allocate the surplus page to
 959          * prevent racing processes from causing the surplus to exceed
 960          * overcommit
 961          *
 962          * This however introduces a different race, where a process B
 963          * tries to grow the static hugepage pool while alloc_pages() is
 964          * called by process A. B will only examine the per-node
 965          * counters in determining if surplus huge pages can be
 966          * converted to normal huge pages in adjust_pool_surplus(). A
 967          * won't be able to increment the per-node counter, until the
 968          * lock is dropped by B, but B doesn't drop hugetlb_lock until
 969          * no more huge pages can be converted from surplus to normal
 970          * state (and doesn't try to convert again). Thus, we have a
 971          * case where a surplus huge page exists, the pool is grown, and
 972          * the surplus huge page still exists after, even though it
 973          * should just have been converted to a normal huge page. This
 974          * does not leak memory, though, as the hugepage will be freed
 975          * once it is out of use. It also does not allow the counters to
 976          * go out of whack in adjust_pool_surplus() as we don't modify
 977          * the node values until we've gotten the hugepage and only the
 978          * per-node value is checked there.
 979          */
 980         spin_lock(&hugetlb_lock);
 981         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
 982                 spin_unlock(&hugetlb_lock);
 983                 return NULL;
 984         } else {
 985                 h->nr_huge_pages++;
 986                 h->surplus_huge_pages++;
 987         }
 988         spin_unlock(&hugetlb_lock);
 989
 990         if (nid == NUMA_NO_NODE)
 991                 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
 992                                    __GFP_REPEAT|__GFP_NOWARN,
 993                                    huge_page_order(h));
 994         else
 995                 page = alloc_pages_exact_node(nid,
 996                         htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
 997                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
 998
 999         if (page && arch_prepare_hugepage(page)) {
1000                 __free_pages(page, huge_page_order(h));
1001                 page = NULL;
1002         }
1003
1004         spin_lock(&hugetlb_lock);
1005         if (page) {
1006                 INIT_LIST_HEAD(&page->lru);
1007                 r_nid = page_to_nid(page);
1008                 set_compound_page_dtor(page, free_huge_page);
1009                 set_hugetlb_cgroup(page, NULL);
1010                 /*
1011                  * We incremented the global counters already
1012                  */
1013                 h->nr_huge_pages_node[r_nid]++;
1014                 h->surplus_huge_pages_node[r_nid]++;
1015                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1016         } else {
1017                 h->nr_huge_pages--;
1018                 h->surplus_huge_pages--;
1019                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1020         }
1021         spin_unlock(&hugetlb_lock);
1022
1023         return page;
1024 }
1025
1026 /*
1027  * This allocation function is useful in the context where vma is irrelevant.
1028  * E.g. soft-offlining uses this function because it only cares physical
1029  * address of error page.
1030  */
1031 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1032 {
1033         struct page *page = NULL;
1034
1035         spin_lock(&hugetlb_lock);
1036         if (h->free_huge_pages - h->resv_huge_pages > 0)
1037                 page = dequeue_huge_page_node(h, nid);
1038         spin_unlock(&hugetlb_lock);
1039
1040         if (!page)
1041                 page = alloc_buddy_huge_page(h, nid);
1042
1043         return page;
1044 }
1045
1046 /*
1047  * Increase the hugetlb pool such that it can accommodate a reservation
1048  * of size 'delta'.
1049  */
1050 static int gather_surplus_pages(struct hstate *h, int delta)
1051 {
1052         struct list_head surplus_list;
1053         struct page *page, *tmp;
1054         int ret, i;
1055         int needed, allocated;
1056         bool alloc_ok = true;
1057
1058         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1059         if (needed <= 0) {
1060                 h->resv_huge_pages += delta;
1061                 return 0;
1062         }
1063
1064         allocated = 0;
1065         INIT_LIST_HEAD(&surplus_list);
1066
1067         ret = -ENOMEM;
1068 retry:
1069         spin_unlock(&hugetlb_lock);
1070         for (i = 0; i < needed; i++) {
1071                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1072                 if (!page) {
1073                         alloc_ok = false;
1074                         break;
1075                 }
1076                 list_add(&page->lru, &surplus_list);
1077         }
1078         allocated += i;
1079
1080         /*
1081          * After retaking hugetlb_lock, we need to recalculate 'needed'
1082          * because either resv_huge_pages or free_huge_pages may have changed.
1083          */
1084         spin_lock(&hugetlb_lock);
1085         needed = (h->resv_huge_pages + delta) -
1086                         (h->free_huge_pages + allocated);
1087         if (needed > 0) {
1088                 if (alloc_ok)
1089                         goto retry;
1090                 /*
1091                  * We were not able to allocate enough pages to
1092                  * satisfy the entire reservation so we free what
1093                  * we've allocated so far.
1094                  */
1095                 goto free;
1096         }
1097         /*
1098          * The surplus_list now contains _at_least_ the number of extra pages
1099          * needed to accommodate the reservation.  Add the appropriate number
1100          * of pages to the hugetlb pool and free the extras back to the buddy
1101          * allocator.  Commit the entire reservation here to prevent another
1102          * process from stealing the pages as they are added to the pool but
1103          * before they are reserved.
1104          */
1105         needed += allocated;
1106         h->resv_huge_pages += delta;
1107         ret = 0;
1108
1109         /* Free the needed pages to the hugetlb pool */
1110         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1111                 if ((--needed) < 0)
1112                         break;
1113                 /*
1114                  * This page is now managed by the hugetlb allocator and has
1115                  * no users -- drop the buddy allocator's reference.
1116                  */
1117                 put_page_testzero(page);
1118                 VM_BUG_ON(page_count(page));
1119                 enqueue_huge_page(h, page);
1120         }
1121 free:
1122         spin_unlock(&hugetlb_lock);
1123
1124         /* Free unnecessary surplus pages to the buddy allocator */
1125         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1126                 put_page(page);
1127         spin_lock(&hugetlb_lock);
1128
1129         return ret;
1130 }
1131
1132 /*
1133  * When releasing a hugetlb pool reservation, any surplus pages that were
1134  * allocated to satisfy the reservation must be explicitly freed if they were
1135  * never used.
1136  * Called with hugetlb_lock held.
1137  */
1138 static void return_unused_surplus_pages(struct hstate *h,
1139                                         unsigned long unused_resv_pages)
1140 {
1141         unsigned long nr_pages;
1142
1143         /* Uncommit the reservation */
1144         h->resv_huge_pages -= unused_resv_pages;
1145
1146         /* Cannot return gigantic pages currently */
1147         if (h->order >= MAX_ORDER)
1148                 return;
1149
1150         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1151
1152         /*
1153          * We want to release as many surplus pages as possible, spread
1154          * evenly across all nodes with memory. Iterate across these nodes
1155          * until we can no longer free unreserved surplus pages. This occurs
1156          * when the nodes with surplus pages have no free pages.
1157          * free_pool_huge_page() will balance the the freed pages across the
1158          * on-line nodes with memory and will handle the hstate accounting.
1159          */
1160         while (nr_pages--) {
1161                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1162                         break;
1163         }
1164 }
1165
1166 /*
1167  * Determine if the huge page at addr within the vma has an associated
1168  * reservation.  Where it does not we will need to logically increase
1169  * reservation and actually increase subpool usage before an allocation
1170  * can occur.  Where any new reservation would be required the
1171  * reservation change is prepared, but not committed.  Once the page
1172  * has been allocated from the subpool and instantiated the change should
1173  * be committed via vma_commit_reservation.  No action is required on
1174  * failure.
1175  */
1176 static long vma_needs_reservation(struct hstate *h,
1177                         struct vm_area_struct *vma, unsigned long addr)
1178 {
1179         struct address_space *mapping = vma->vm_file->f_mapping;
1180         struct inode *inode = mapping->host;
1181
1182         if (vma->vm_flags & VM_MAYSHARE) {
1183                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1184                 return region_chg(&inode->i_mapping->private_list,
1185                                                         idx, idx + 1);
1186
1187         } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1188                 return 1;
1189
1190         } else  {
1191                 long err;
1192                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1193                 struct resv_map *resv = vma_resv_map(vma);
1194
1195                 err = region_chg(&resv->regions, idx, idx + 1);
1196                 if (err < 0)
1197                         return err;
1198                 return 0;
1199         }
1200 }
1201 static void vma_commit_reservation(struct hstate *h,
1202                         struct vm_area_struct *vma, unsigned long addr)
1203 {
1204         struct address_space *mapping = vma->vm_file->f_mapping;
1205         struct inode *inode = mapping->host;
1206
1207         if (vma->vm_flags & VM_MAYSHARE) {
1208                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1209                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
1210
1211         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1212                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1213                 struct resv_map *resv = vma_resv_map(vma);
1214
1215                 /* Mark this page used in the map. */
1216                 region_add(&resv->regions, idx, idx + 1);
1217         }
1218 }
1219
1220 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1221                                     unsigned long addr, int avoid_reserve)
1222 {
1223         struct hugepage_subpool *spool = subpool_vma(vma);
1224         struct hstate *h = hstate_vma(vma);
1225         struct page *page;
1226         long chg;
1227         int ret, idx;
1228         struct hugetlb_cgroup *h_cg;
1229
1230         idx = hstate_index(h);
1231         /*
1232          * Processes that did not create the mapping will have no
1233          * reserves and will not have accounted against subpool
1234          * limit. Check that the subpool limit can be made before
1235          * satisfying the allocation MAP_NORESERVE mappings may also
1236          * need pages and subpool limit allocated allocated if no reserve
1237          * mapping overlaps.
1238          */
1239         chg = vma_needs_reservation(h, vma, addr);
1240         if (chg < 0)
1241                 return ERR_PTR(-ENOMEM);
1242         if (chg || avoid_reserve)
1243                 if (hugepage_subpool_get_pages(spool, 1))
1244                         return ERR_PTR(-ENOSPC);
1245
1246         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1247         if (ret) {
1248                 if (chg || avoid_reserve)
1249                         hugepage_subpool_put_pages(spool, 1);
1250                 return ERR_PTR(-ENOSPC);
1251         }
1252         spin_lock(&hugetlb_lock);
1253         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1254         if (!page) {
1255                 spin_unlock(&hugetlb_lock);
1256                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1257                 if (!page) {
1258                         hugetlb_cgroup_uncharge_cgroup(idx,
1259                                                        pages_per_huge_page(h),
1260                                                        h_cg);
1261                         if (chg || avoid_reserve)
1262                                 hugepage_subpool_put_pages(spool, 1);
1263                         return ERR_PTR(-ENOSPC);
1264                 }
1265                 spin_lock(&hugetlb_lock);
1266                 list_move(&page->lru, &h->hugepage_activelist);
1267                 /* Fall through */
1268         }
1269         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1270         spin_unlock(&hugetlb_lock);
1271
1272         set_page_private(page, (unsigned long)spool);
1273
1274         vma_commit_reservation(h, vma, addr);
1275         return page;
1276 }
1277
1278 /*
1279  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1280  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1281  * where no ERR_VALUE is expected to be returned.
1282  */
1283 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1284                                 unsigned long addr, int avoid_reserve)
1285 {
1286         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1287         if (IS_ERR(page))
1288                 page = NULL;
1289         return page;
1290 }
1291
1292 int __weak alloc_bootmem_huge_page(struct hstate *h)
1293 {
1294         struct huge_bootmem_page *m;
1295         int nr_nodes, node;
1296
1297         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1298                 void *addr;
1299
1300                 addr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
1301                                 huge_page_size(h), huge_page_size(h), 0);
1302
1303                 if (addr) {
1304                         /*
1305                          * Use the beginning of the huge page to store the
1306                          * huge_bootmem_page struct (until gather_bootmem
1307                          * puts them into the mem_map).
1308                          */
1309                         m = addr;
1310                         goto found;
1311                 }
1312         }
1313         return 0;
1314
1315 found:
1316         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1317         /* Put them into a private list first because mem_map is not up yet */
1318         list_add(&m->list, &huge_boot_pages);
1319         m->hstate = h;
1320         return 1;
1321 }
1322
1323 static void prep_compound_huge_page(struct page *page, int order)
1324 {
1325         if (unlikely(order > (MAX_ORDER - 1)))
1326                 prep_compound_gigantic_page(page, order);
1327         else
1328                 prep_compound_page(page, order);
1329 }
1330
1331 /* Put bootmem huge pages into the standard lists after mem_map is up */
1332 static void __init gather_bootmem_prealloc(void)
1333 {
1334         struct huge_bootmem_page *m;
1335
1336         list_for_each_entry(m, &huge_boot_pages, list) {
1337                 struct hstate *h = m->hstate;
1338                 struct page *page;
1339
1340 #ifdef CONFIG_HIGHMEM
1341                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1342                 free_bootmem_late((unsigned long)m,
1343                                   sizeof(struct huge_bootmem_page));
1344 #else
1345                 page = virt_to_page(m);
1346 #endif
1347                 WARN_ON(page_count(page) != 1);
1348                 prep_compound_huge_page(page, h->order);
1349                 WARN_ON(PageReserved(page));
1350                 prep_new_huge_page(h, page, page_to_nid(page));
1351                 /*
1352                  * If we had gigantic hugepages allocated at boot time, we need
1353                  * to restore the 'stolen' pages to totalram_pages in order to
1354                  * fix confusing memory reports from free(1) and another
1355                  * side-effects, like CommitLimit going negative.
1356                  */
1357                 if (h->order > (MAX_ORDER - 1))
1358                         adjust_managed_page_count(page, 1 << h->order);
1359         }
1360 }
1361
1362 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1363 {
1364         unsigned long i;
1365
1366         for (i = 0; i < h->max_huge_pages; ++i) {
1367                 if (h->order >= MAX_ORDER) {
1368                         if (!alloc_bootmem_huge_page(h))
1369                                 break;
1370                 } else if (!alloc_fresh_huge_page(h,
1371                                          &node_states[N_MEMORY]))
1372                         break;
1373         }
1374         h->max_huge_pages = i;
1375 }
1376
1377 static void __init hugetlb_init_hstates(void)
1378 {
1379         struct hstate *h;
1380
1381         for_each_hstate(h) {
1382                 /* oversize hugepages were init'ed in early boot */
1383                 if (h->order < MAX_ORDER)
1384                         hugetlb_hstate_alloc_pages(h);
1385         }
1386 }
1387
1388 static char * __init memfmt(char *buf, unsigned long n)
1389 {
1390         if (n >= (1UL << 30))
1391                 sprintf(buf, "%lu GB", n >> 30);
1392         else if (n >= (1UL << 20))
1393                 sprintf(buf, "%lu MB", n >> 20);
1394         else
1395                 sprintf(buf, "%lu KB", n >> 10);
1396         return buf;
1397 }
1398
1399 static void __init report_hugepages(void)
1400 {
1401         struct hstate *h;
1402
1403         for_each_hstate(h) {
1404                 char buf[32];
1405                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1406                         memfmt(buf, huge_page_size(h)),
1407                         h->free_huge_pages);
1408         }
1409 }
1410
1411 #ifdef CONFIG_HIGHMEM
1412 static void try_to_free_low(struct hstate *h, unsigned long count,
1413                                                 nodemask_t *nodes_allowed)
1414 {
1415         int i;
1416
1417         if (h->order >= MAX_ORDER)
1418                 return;
1419
1420         for_each_node_mask(i, *nodes_allowed) {
1421                 struct page *page, *next;
1422                 struct list_head *freel = &h->hugepage_freelists[i];
1423                 list_for_each_entry_safe(page, next, freel, lru) {
1424                         if (count >= h->nr_huge_pages)
1425                                 return;
1426                         if (PageHighMem(page))
1427                                 continue;
1428                         list_del(&page->lru);
1429                         update_and_free_page(h, page);
1430                         h->free_huge_pages--;
1431                         h->free_huge_pages_node[page_to_nid(page)]--;
1432                 }
1433         }
1434 }
1435 #else
1436 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1437                                                 nodemask_t *nodes_allowed)
1438 {
1439 }
1440 #endif
1441
1442 /*
1443  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1444  * balanced by operating on them in a round-robin fashion.
1445  * Returns 1 if an adjustment was made.
1446  */
1447 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1448                                 int delta)
1449 {
1450         int nr_nodes, node;
1451
1452         VM_BUG_ON(delta != -1 && delta != 1);
1453
1454         if (delta < 0) {
1455                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1456                         if (h->surplus_huge_pages_node[node])
1457                                 goto found;
1458                 }
1459         } else {
1460                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1461                         if (h->surplus_huge_pages_node[node] <
1462                                         h->nr_huge_pages_node[node])
1463                                 goto found;
1464                 }
1465         }
1466         return 0;
1467
1468 found:
1469         h->surplus_huge_pages += delta;
1470         h->surplus_huge_pages_node[node] += delta;
1471         return 1;
1472 }
1473
1474 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1475 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1476                                                 nodemask_t *nodes_allowed)
1477 {
1478         unsigned long min_count, ret;
1479
1480         if (h->order >= MAX_ORDER)
1481                 return h->max_huge_pages;
1482
1483         /*
1484          * Increase the pool size
1485          * First take pages out of surplus state.  Then make up the
1486          * remaining difference by allocating fresh huge pages.
1487          *
1488          * We might race with alloc_buddy_huge_page() here and be unable
1489          * to convert a surplus huge page to a normal huge page. That is
1490          * not critical, though, it just means the overall size of the
1491          * pool might be one hugepage larger than it needs to be, but
1492          * within all the constraints specified by the sysctls.
1493          */
1494         spin_lock(&hugetlb_lock);
1495         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1496                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1497                         break;
1498         }
1499
1500         while (count > persistent_huge_pages(h)) {
1501                 /*
1502                  * If this allocation races such that we no longer need the
1503                  * page, free_huge_page will handle it by freeing the page
1504                  * and reducing the surplus.
1505                  */
1506                 spin_unlock(&hugetlb_lock);
1507                 ret = alloc_fresh_huge_page(h, nodes_allowed);
1508                 spin_lock(&hugetlb_lock);
1509                 if (!ret)
1510                         goto out;
1511
1512                 /* Bail for signals. Probably ctrl-c from user */
1513                 if (signal_pending(current))
1514                         goto out;
1515         }
1516
1517         /*
1518          * Decrease the pool size
1519          * First return free pages to the buddy allocator (being careful
1520          * to keep enough around to satisfy reservations).  Then place
1521          * pages into surplus state as needed so the pool will shrink
1522          * to the desired size as pages become free.
1523          *
1524          * By placing pages into the surplus state independent of the
1525          * overcommit value, we are allowing the surplus pool size to
1526          * exceed overcommit. There are few sane options here. Since
1527          * alloc_buddy_huge_page() is checking the global counter,
1528          * though, we'll note that we're not allowed to exceed surplus
1529          * and won't grow the pool anywhere else. Not until one of the
1530          * sysctls are changed, or the surplus pages go out of use.
1531          */
1532         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1533         min_count = max(count, min_count);
1534         try_to_free_low(h, min_count, nodes_allowed);
1535         while (min_count < persistent_huge_pages(h)) {
1536                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1537                         break;
1538         }
1539         while (count < persistent_huge_pages(h)) {
1540                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1541                         break;
1542         }
1543 out:
1544         ret = persistent_huge_pages(h);
1545         spin_unlock(&hugetlb_lock);
1546         return ret;
1547 }
1548
1549 #define HSTATE_ATTR_RO(_name) \
1550         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1551
1552 #define HSTATE_ATTR(_name) \
1553         static struct kobj_attribute _name##_attr = \
1554                 __ATTR(_name, 0644, _name##_show, _name##_store)
1555
1556 static struct kobject *hugepages_kobj;
1557 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1558
1559 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1560
1561 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1562 {
1563         int i;
1564
1565         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1566                 if (hstate_kobjs[i] == kobj) {
1567                         if (nidp)
1568                                 *nidp = NUMA_NO_NODE;
1569                         return &hstates[i];
1570                 }
1571
1572         return kobj_to_node_hstate(kobj, nidp);
1573 }
1574
1575 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1576                                         struct kobj_attribute *attr, char *buf)
1577 {
1578         struct hstate *h;
1579         unsigned long nr_huge_pages;
1580         int nid;
1581
1582         h = kobj_to_hstate(kobj, &nid);
1583         if (nid == NUMA_NO_NODE)
1584                 nr_huge_pages = h->nr_huge_pages;
1585         else
1586                 nr_huge_pages = h->nr_huge_pages_node[nid];
1587
1588         return sprintf(buf, "%lu\n", nr_huge_pages);
1589 }
1590
1591 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1592                         struct kobject *kobj, struct kobj_attribute *attr,
1593                         const char *buf, size_t len)
1594 {
1595         int err;
1596         int nid;
1597         unsigned long count;
1598         struct hstate *h;
1599         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1600
1601         err = kstrtoul(buf, 10, &count);
1602         if (err)
1603                 goto out;
1604
1605         h = kobj_to_hstate(kobj, &nid);
1606         if (h->order >= MAX_ORDER) {
1607                 err = -EINVAL;
1608                 goto out;
1609         }
1610
1611         if (nid == NUMA_NO_NODE) {
1612                 /*
1613                  * global hstate attribute
1614                  */
1615                 if (!(obey_mempolicy &&
1616                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1617                         NODEMASK_FREE(nodes_allowed);
1618                         nodes_allowed = &node_states[N_MEMORY];
1619                 }
1620         } else if (nodes_allowed) {
1621                 /*
1622                  * per node hstate attribute: adjust count to global,
1623                  * but restrict alloc/free to the specified node.
1624                  */
1625                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1626                 init_nodemask_of_node(nodes_allowed, nid);
1627         } else
1628                 nodes_allowed = &node_states[N_MEMORY];
1629
1630         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1631
1632         if (nodes_allowed != &node_states[N_MEMORY])
1633                 NODEMASK_FREE(nodes_allowed);
1634
1635         return len;
1636 out:
1637         NODEMASK_FREE(nodes_allowed);
1638         return err;
1639 }
1640
1641 static ssize_t nr_hugepages_show(struct kobject *kobj,
1642                                        struct kobj_attribute *attr, char *buf)
1643 {
1644         return nr_hugepages_show_common(kobj, attr, buf);
1645 }
1646
1647 static ssize_t nr_hugepages_store(struct kobject *kobj,
1648                struct kobj_attribute *attr, const char *buf, size_t len)
1649 {
1650         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1651 }
1652 HSTATE_ATTR(nr_hugepages);
1653
1654 #ifdef CONFIG_NUMA
1655
1656 /*
1657  * hstate attribute for optionally mempolicy-based constraint on persistent
1658  * huge page alloc/free.
1659  */
1660 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1661                                        struct kobj_attribute *attr, char *buf)
1662 {
1663         return nr_hugepages_show_common(kobj, attr, buf);
1664 }
1665
1666 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1667                struct kobj_attribute *attr, const char *buf, size_t len)
1668 {
1669         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1670 }
1671 HSTATE_ATTR(nr_hugepages_mempolicy);
1672 #endif
1673
1674
1675 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1676                                         struct kobj_attribute *attr, char *buf)
1677 {
1678         struct hstate *h = kobj_to_hstate(kobj, NULL);
1679         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1680 }
1681
1682 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1683                 struct kobj_attribute *attr, const char *buf, size_t count)
1684 {
1685         int err;
1686         unsigned long input;
1687         struct hstate *h = kobj_to_hstate(kobj, NULL);
1688
1689         if (h->order >= MAX_ORDER)
1690                 return -EINVAL;
1691
1692         err = kstrtoul(buf, 10, &input);
1693         if (err)
1694                 return err;
1695
1696         spin_lock(&hugetlb_lock);
1697         h->nr_overcommit_huge_pages = input;
1698         spin_unlock(&hugetlb_lock);
1699
1700         return count;
1701 }
1702 HSTATE_ATTR(nr_overcommit_hugepages);
1703
1704 static ssize_t free_hugepages_show(struct kobject *kobj,
1705                                         struct kobj_attribute *attr, char *buf)
1706 {
1707         struct hstate *h;
1708         unsigned long free_huge_pages;
1709         int nid;
1710
1711         h = kobj_to_hstate(kobj, &nid);
1712         if (nid == NUMA_NO_NODE)
1713                 free_huge_pages = h->free_huge_pages;
1714         else
1715                 free_huge_pages = h->free_huge_pages_node[nid];
1716
1717         return sprintf(buf, "%lu\n", free_huge_pages);
1718 }
1719 HSTATE_ATTR_RO(free_hugepages);
1720
1721 static ssize_t resv_hugepages_show(struct kobject *kobj,
1722                                         struct kobj_attribute *attr, char *buf)
1723 {
1724         struct hstate *h = kobj_to_hstate(kobj, NULL);
1725         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1726 }
1727 HSTATE_ATTR_RO(resv_hugepages);
1728
1729 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1730                                         struct kobj_attribute *attr, char *buf)
1731 {
1732         struct hstate *h;
1733         unsigned long surplus_huge_pages;
1734         int nid;
1735
1736         h = kobj_to_hstate(kobj, &nid);
1737         if (nid == NUMA_NO_NODE)
1738                 surplus_huge_pages = h->surplus_huge_pages;
1739         else
1740                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1741
1742         return sprintf(buf, "%lu\n", surplus_huge_pages);
1743 }
1744 HSTATE_ATTR_RO(surplus_hugepages);
1745
1746 static struct attribute *hstate_attrs[] = {
1747         &nr_hugepages_attr.attr,
1748         &nr_overcommit_hugepages_attr.attr,
1749         &free_hugepages_attr.attr,
1750         &resv_hugepages_attr.attr,
1751         &surplus_hugepages_attr.attr,
1752 #ifdef CONFIG_NUMA
1753         &nr_hugepages_mempolicy_attr.attr,
1754 #endif
1755         NULL,
1756 };
1757
1758 static struct attribute_group hstate_attr_group = {
1759         .attrs = hstate_attrs,
1760 };
1761
1762 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1763                                     struct kobject **hstate_kobjs,
1764                                     struct attribute_group *hstate_attr_group)
1765 {
1766         int retval;
1767         int hi = hstate_index(h);
1768
1769         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1770         if (!hstate_kobjs[hi])
1771                 return -ENOMEM;
1772
1773         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1774         if (retval)
1775                 kobject_put(hstate_kobjs[hi]);
1776
1777         return retval;
1778 }
1779
1780 static void __init hugetlb_sysfs_init(void)
1781 {
1782         struct hstate *h;
1783         int err;
1784
1785         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1786         if (!hugepages_kobj)
1787                 return;
1788
1789         for_each_hstate(h) {
1790                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1791                                          hstate_kobjs, &hstate_attr_group);
1792                 if (err)
1793                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
1794         }
1795 }
1796
1797 #ifdef CONFIG_NUMA
1798
1799 /*
1800  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1801  * with node devices in node_devices[] using a parallel array.  The array
1802  * index of a node device or _hstate == node id.
1803  * This is here to avoid any static dependency of the node device driver, in
1804  * the base kernel, on the hugetlb module.
1805  */
1806 struct node_hstate {
1807         struct kobject          *hugepages_kobj;
1808         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1809 };
1810 struct node_hstate node_hstates[MAX_NUMNODES];
1811
1812 /*
1813  * A subset of global hstate attributes for node devices
1814  */
1815 static struct attribute *per_node_hstate_attrs[] = {
1816         &nr_hugepages_attr.attr,
1817         &free_hugepages_attr.attr,
1818         &surplus_hugepages_attr.attr,
1819         NULL,
1820 };
1821
1822 static struct attribute_group per_node_hstate_attr_group = {
1823         .attrs = per_node_hstate_attrs,
1824 };
1825
1826 /*
1827  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1828  * Returns node id via non-NULL nidp.
1829  */
1830 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1831 {
1832         int nid;
1833
1834         for (nid = 0; nid < nr_node_ids; nid++) {
1835                 struct node_hstate *nhs = &node_hstates[nid];
1836                 int i;
1837                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1838                         if (nhs->hstate_kobjs[i] == kobj) {
1839                                 if (nidp)
1840                                         *nidp = nid;
1841                                 return &hstates[i];
1842                         }
1843         }
1844
1845         BUG();
1846         return NULL;
1847 }
1848
1849 /*
1850  * Unregister hstate attributes from a single node device.
1851  * No-op if no hstate attributes attached.
1852  */
1853 static void hugetlb_unregister_node(struct node *node)
1854 {
1855         struct hstate *h;
1856         struct node_hstate *nhs = &node_hstates[node->dev.id];
1857
1858         if (!nhs->hugepages_kobj)
1859                 return;         /* no hstate attributes */
1860
1861         for_each_hstate(h) {
1862                 int idx = hstate_index(h);
1863                 if (nhs->hstate_kobjs[idx]) {
1864                         kobject_put(nhs->hstate_kobjs[idx]);
1865                         nhs->hstate_kobjs[idx] = NULL;
1866                 }
1867         }
1868
1869         kobject_put(nhs->hugepages_kobj);
1870         nhs->hugepages_kobj = NULL;
1871 }
1872
1873 /*
1874  * hugetlb module exit:  unregister hstate attributes from node devices
1875  * that have them.
1876  */
1877 static void hugetlb_unregister_all_nodes(void)
1878 {
1879         int nid;
1880
1881         /*
1882          * disable node device registrations.
1883          */
1884         register_hugetlbfs_with_node(NULL, NULL);
1885
1886         /*
1887          * remove hstate attributes from any nodes that have them.
1888          */
1889         for (nid = 0; nid < nr_node_ids; nid++)
1890                 hugetlb_unregister_node(node_devices[nid]);
1891 }
1892
1893 /*
1894  * Register hstate attributes for a single node device.
1895  * No-op if attributes already registered.
1896  */
1897 static void hugetlb_register_node(struct node *node)
1898 {
1899         struct hstate *h;
1900         struct node_hstate *nhs = &node_hstates[node->dev.id];
1901         int err;
1902
1903         if (nhs->hugepages_kobj)
1904                 return;         /* already allocated */
1905
1906         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1907                                                         &node->dev.kobj);
1908         if (!nhs->hugepages_kobj)
1909                 return;
1910
1911         for_each_hstate(h) {
1912                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1913                                                 nhs->hstate_kobjs,
1914                                                 &per_node_hstate_attr_group);
1915                 if (err) {
1916                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
1917                                 h->name, node->dev.id);
1918                         hugetlb_unregister_node(node);
1919                         break;
1920                 }
1921         }
1922 }
1923
1924 /*
1925  * hugetlb init time:  register hstate attributes for all registered node
1926  * devices of nodes that have memory.  All on-line nodes should have
1927  * registered their associated device by this time.
1928  */
1929 static void hugetlb_register_all_nodes(void)
1930 {
1931         int nid;
1932
1933         for_each_node_state(nid, N_MEMORY) {
1934                 struct node *node = node_devices[nid];
1935                 if (node->dev.id == nid)
1936                         hugetlb_register_node(node);
1937         }
1938
1939         /*
1940          * Let the node device driver know we're here so it can
1941          * [un]register hstate attributes on node hotplug.
1942          */
1943         register_hugetlbfs_with_node(hugetlb_register_node,
1944                                      hugetlb_unregister_node);
1945 }
1946 #else   /* !CONFIG_NUMA */
1947
1948 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1949 {
1950         BUG();
1951         if (nidp)
1952                 *nidp = -1;
1953         return NULL;
1954 }
1955
1956 static void hugetlb_unregister_all_nodes(void) { }
1957
1958 static void hugetlb_register_all_nodes(void) { }
1959
1960 #endif
1961
1962 static void __exit hugetlb_exit(void)
1963 {
1964         struct hstate *h;
1965
1966         hugetlb_unregister_all_nodes();
1967
1968         for_each_hstate(h) {
1969                 kobject_put(hstate_kobjs[hstate_index(h)]);
1970         }
1971
1972         kobject_put(hugepages_kobj);
1973 }
1974 module_exit(hugetlb_exit);
1975
1976 static int __init hugetlb_init(void)
1977 {
1978         /* Some platform decide whether they support huge pages at boot
1979          * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1980          * there is no such support
1981          */
1982         if (HPAGE_SHIFT == 0)
1983                 return 0;
1984
1985         if (!size_to_hstate(default_hstate_size)) {
1986                 default_hstate_size = HPAGE_SIZE;
1987                 if (!size_to_hstate(default_hstate_size))
1988                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1989         }
1990         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
1991         if (default_hstate_max_huge_pages)
1992                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1993
1994         hugetlb_init_hstates();
1995         gather_bootmem_prealloc();
1996         report_hugepages();
1997
1998         hugetlb_sysfs_init();
1999         hugetlb_register_all_nodes();
2000         hugetlb_cgroup_file_init();
2001
2002         return 0;
2003 }
2004 module_init(hugetlb_init);
2005
2006 /* Should be called on processing a hugepagesz=... option */
2007 void __init hugetlb_add_hstate(unsigned order)
2008 {
2009         struct hstate *h;
2010         unsigned long i;
2011
2012         if (size_to_hstate(PAGE_SIZE << order)) {
2013                 pr_warning("hugepagesz= specified twice, ignoring\n");
2014                 return;
2015         }
2016         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2017         BUG_ON(order == 0);
2018         h = &hstates[hugetlb_max_hstate++];
2019         h->order = order;
2020         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2021         h->nr_huge_pages = 0;
2022         h->free_huge_pages = 0;
2023         for (i = 0; i < MAX_NUMNODES; ++i)
2024                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2025         INIT_LIST_HEAD(&h->hugepage_activelist);
2026         h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2027         h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2028         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2029                                         huge_page_size(h)/1024);
2030
2031         parsed_hstate = h;
2032 }
2033
2034 static int __init hugetlb_nrpages_setup(char *s)
2035 {
2036         unsigned long *mhp;
2037         static unsigned long *last_mhp;
2038
2039         /*
2040          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2041          * so this hugepages= parameter goes to the "default hstate".
2042          */
2043         if (!hugetlb_max_hstate)
2044                 mhp = &default_hstate_max_huge_pages;
2045         else
2046                 mhp = &parsed_hstate->max_huge_pages;
2047
2048         if (mhp == last_mhp) {
2049                 pr_warning("hugepages= specified twice without "
2050                            "interleaving hugepagesz=, ignoring\n");
2051                 return 1;
2052         }
2053
2054         if (sscanf(s, "%lu", mhp) <= 0)
2055                 *mhp = 0;
2056
2057         /*
2058          * Global state is always initialized later in hugetlb_init.
2059          * But we need to allocate >= MAX_ORDER hstates here early to still
2060          * use the bootmem allocator.
2061          */
2062         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2063                 hugetlb_hstate_alloc_pages(parsed_hstate);
2064
2065         last_mhp = mhp;
2066
2067         return 1;
2068 }
2069 __setup("hugepages=", hugetlb_nrpages_setup);
2070
2071 static int __init hugetlb_default_setup(char *s)
2072 {
2073         default_hstate_size = memparse(s, &s);
2074         return 1;
2075 }
2076 __setup("default_hugepagesz=", hugetlb_default_setup);
2077
2078 static unsigned int cpuset_mems_nr(unsigned int *array)
2079 {
2080         int node;
2081         unsigned int nr = 0;
2082
2083         for_each_node_mask(node, cpuset_current_mems_allowed)
2084                 nr += array[node];
2085
2086         return nr;
2087 }
2088
2089 #ifdef CONFIG_SYSCTL
2090 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2091                          struct ctl_table *table, int write,
2092                          void __user *buffer, size_t *length, loff_t *ppos)
2093 {
2094         struct hstate *h = &default_hstate;
2095         unsigned long tmp;
2096         int ret;
2097
2098         tmp = h->max_huge_pages;
2099
2100         if (write && h->order >= MAX_ORDER)
2101                 return -EINVAL;
2102
2103         table->data = &tmp;
2104         table->maxlen = sizeof(unsigned long);
2105         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2106         if (ret)
2107                 goto out;
2108
2109         if (write) {
2110                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
2111                                                 GFP_KERNEL | __GFP_NORETRY);
2112                 if (!(obey_mempolicy &&
2113                                init_nodemask_of_mempolicy(nodes_allowed))) {
2114                         NODEMASK_FREE(nodes_allowed);
2115                         nodes_allowed = &node_states[N_MEMORY];
2116                 }
2117                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2118
2119                 if (nodes_allowed != &node_states[N_MEMORY])
2120                         NODEMASK_FREE(nodes_allowed);
2121         }
2122 out:
2123         return ret;
2124 }
2125
2126 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2127                           void __user *buffer, size_t *length, loff_t *ppos)
2128 {
2129
2130         return hugetlb_sysctl_handler_common(false, table, write,
2131                                                         buffer, length, ppos);
2132 }
2133
2134 #ifdef CONFIG_NUMA
2135 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2136                           void __user *buffer, size_t *length, loff_t *ppos)
2137 {
2138         return hugetlb_sysctl_handler_common(true, table, write,
2139                                                         buffer, length, ppos);
2140 }
2141 #endif /* CONFIG_NUMA */
2142
2143 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2144                         void __user *buffer,
2145                         size_t *length, loff_t *ppos)
2146 {
2147         struct hstate *h = &default_hstate;
2148         unsigned long tmp;
2149         int ret;
2150
2151         tmp = h->nr_overcommit_huge_pages;
2152
2153         if (write && h->order >= MAX_ORDER)
2154                 return -EINVAL;
2155
2156         table->data = &tmp;
2157         table->maxlen = sizeof(unsigned long);
2158         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2159         if (ret)
2160                 goto out;
2161
2162         if (write) {
2163                 spin_lock(&hugetlb_lock);
2164                 h->nr_overcommit_huge_pages = tmp;
2165                 spin_unlock(&hugetlb_lock);
2166         }
2167 out:
2168         return ret;
2169 }
2170
2171 #endif /* CONFIG_SYSCTL */
2172
2173 void hugetlb_report_meminfo(struct seq_file *m)
2174 {
2175         struct hstate *h = &default_hstate;
2176         seq_printf(m,
2177                         "HugePages_Total:   %5lu\n"
2178                         "HugePages_Free:    %5lu\n"
2179                         "HugePages_Rsvd:    %5lu\n"
2180                         "HugePages_Surp:    %5lu\n"
2181                         "Hugepagesize:   %8lu kB\n",
2182                         h->nr_huge_pages,
2183                         h->free_huge_pages,
2184                         h->resv_huge_pages,
2185                         h->surplus_huge_pages,
2186                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2187 }
2188
2189 int hugetlb_report_node_meminfo(int nid, char *buf)
2190 {
2191         struct hstate *h = &default_hstate;
2192         return sprintf(buf,
2193                 "Node %d HugePages_Total: %5u\n"
2194                 "Node %d HugePages_Free:  %5u\n"
2195                 "Node %d HugePages_Surp:  %5u\n",
2196                 nid, h->nr_huge_pages_node[nid],
2197                 nid, h->free_huge_pages_node[nid],
2198                 nid, h->surplus_huge_pages_node[nid]);
2199 }
2200
2201 void hugetlb_show_meminfo(void)
2202 {
2203         struct hstate *h;
2204         int nid;
2205
2206         for_each_node_state(nid, N_MEMORY)
2207                 for_each_hstate(h)
2208                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2209                                 nid,
2210                                 h->nr_huge_pages_node[nid],
2211                                 h->free_huge_pages_node[nid],
2212                                 h->surplus_huge_pages_node[nid],
2213                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2214 }
2215
2216 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2217 unsigned long hugetlb_total_pages(void)
2218 {
2219         struct hstate *h;
2220         unsigned long nr_total_pages = 0;
2221
2222         for_each_hstate(h)
2223                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2224         return nr_total_pages;
2225 }
2226
2227 static int hugetlb_acct_memory(struct hstate *h, long delta)
2228 {
2229         int ret = -ENOMEM;
2230
2231         spin_lock(&hugetlb_lock);
2232         /*
2233          * When cpuset is configured, it breaks the strict hugetlb page
2234          * reservation as the accounting is done on a global variable. Such
2235          * reservation is completely rubbish in the presence of cpuset because
2236          * the reservation is not checked against page availability for the
2237          * current cpuset. Application can still potentially OOM'ed by kernel
2238          * with lack of free htlb page in cpuset that the task is in.
2239          * Attempt to enforce strict accounting with cpuset is almost
2240          * impossible (or too ugly) because cpuset is too fluid that
2241          * task or memory node can be dynamically moved between cpusets.
2242          *
2243          * The change of semantics for shared hugetlb mapping with cpuset is
2244          * undesirable. However, in order to preserve some of the semantics,
2245          * we fall back to check against current free page availability as
2246          * a best attempt and hopefully to minimize the impact of changing
2247          * semantics that cpuset has.
2248          */
2249         if (delta > 0) {
2250                 if (gather_surplus_pages(h, delta) < 0)
2251                         goto out;
2252
2253                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2254                         return_unused_surplus_pages(h, delta);
2255                         goto out;
2256                 }
2257         }
2258
2259         ret = 0;
2260         if (delta < 0)
2261                 return_unused_surplus_pages(h, (unsigned long) -delta);
2262
2263 out:
2264         spin_unlock(&hugetlb_lock);
2265         return ret;
2266 }
2267
2268 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2269 {
2270         struct resv_map *resv = vma_resv_map(vma);
2271
2272         /*
2273          * This new VMA should share its siblings reservation map if present.
2274          * The VMA will only ever have a valid reservation map pointer where
2275          * it is being copied for another still existing VMA.  As that VMA
2276          * has a reference to the reservation map it cannot disappear until
2277          * after this open call completes.  It is therefore safe to take a
2278          * new reference here without additional locking.
2279          */
2280         if (resv)
2281                 kref_get(&resv->refs);
2282 }
2283
2284 static void resv_map_put(struct vm_area_struct *vma)
2285 {
2286         struct resv_map *resv = vma_resv_map(vma);
2287
2288         if (!resv)
2289                 return;
2290         kref_put(&resv->refs, resv_map_release);
2291 }
2292
2293 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2294 {
2295         struct hstate *h = hstate_vma(vma);
2296         struct resv_map *resv = vma_resv_map(vma);
2297         struct hugepage_subpool *spool = subpool_vma(vma);
2298         unsigned long reserve;
2299         unsigned long start;
2300         unsigned long end;
2301
2302         if (resv) {
2303                 start = vma_hugecache_offset(h, vma, vma->vm_start);
2304                 end = vma_hugecache_offset(h, vma, vma->vm_end);
2305
2306                 reserve = (end - start) -
2307                         region_count(&resv->regions, start, end);
2308
2309                 resv_map_put(vma);
2310
2311                 if (reserve) {
2312                         hugetlb_acct_memory(h, -reserve);
2313                         hugepage_subpool_put_pages(spool, reserve);
2314                 }
2315         }
2316 }
2317
2318 /*
2319  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2320  * handle_mm_fault() to try to instantiate regular-sized pages in the
2321  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2322  * this far.
2323  */
2324 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2325 {
2326         BUG();
2327         return 0;
2328 }
2329
2330 const struct vm_operations_struct hugetlb_vm_ops = {
2331         .fault = hugetlb_vm_op_fault,
2332         .open = hugetlb_vm_op_open,
2333         .close = hugetlb_vm_op_close,
2334 };
2335
2336 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2337                                 int writable)
2338 {
2339         pte_t entry;
2340
2341         if (writable) {
2342                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2343                                          vma->vm_page_prot)));
2344         } else {
2345                 entry = huge_pte_wrprotect(mk_huge_pte(page,
2346                                            vma->vm_page_prot));
2347         }
2348         entry = pte_mkyoung(entry);
2349         entry = pte_mkhuge(entry);
2350         entry = arch_make_huge_pte(entry, vma, page, writable);
2351
2352         return entry;
2353 }
2354
2355 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2356                                    unsigned long address, pte_t *ptep)
2357 {
2358         pte_t entry;
2359
2360         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2361         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2362                 update_mmu_cache(vma, address, ptep);
2363 }
2364
2365
2366 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2367                             struct vm_area_struct *vma)
2368 {
2369         pte_t *src_pte, *dst_pte, entry;
2370         struct page *ptepage;
2371         unsigned long addr;
2372         int cow;
2373         struct hstate *h = hstate_vma(vma);
2374         unsigned long sz = huge_page_size(h);
2375
2376         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2377
2378         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2379                 spinlock_t *src_ptl, *dst_ptl;
2380                 src_pte = huge_pte_offset(src, addr);
2381                 if (!src_pte)
2382                         continue;
2383                 dst_pte = huge_pte_alloc(dst, addr, sz);
2384                 if (!dst_pte)
2385                         goto nomem;
2386
2387                 /* If the pagetables are shared don't copy or take references */
2388                 if (dst_pte == src_pte)
2389                         continue;
2390
2391                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2392                 src_ptl = huge_pte_lockptr(h, src, src_pte);
2393                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
2394                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
2395                         if (cow)
2396                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2397                         entry = huge_ptep_get(src_pte);
2398                         ptepage = pte_page(entry);
2399                         get_page(ptepage);
2400                         page_dup_rmap(ptepage);
2401                         set_huge_pte_at(dst, addr, dst_pte, entry);
2402                 }
2403                 spin_unlock(src_ptl);
2404                 spin_unlock(dst_ptl);
2405         }
2406         return 0;
2407
2408 nomem:
2409         return -ENOMEM;
2410 }
2411
2412 static int is_hugetlb_entry_migration(pte_t pte)
2413 {
2414         swp_entry_t swp;
2415
2416         if (huge_pte_none(pte) || pte_present(pte))
2417                 return 0;
2418         swp = pte_to_swp_entry(pte);
2419         if (non_swap_entry(swp) && is_migration_entry(swp))
2420                 return 1;
2421         else
2422                 return 0;
2423 }
2424
2425 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2426 {
2427         swp_entry_t swp;
2428
2429         if (huge_pte_none(pte) || pte_present(pte))
2430                 return 0;
2431         swp = pte_to_swp_entry(pte);
2432         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2433                 return 1;
2434         else
2435                 return 0;
2436 }
2437
2438 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2439                             unsigned long start, unsigned long end,
2440                             struct page *ref_page)
2441 {
2442         int force_flush = 0;
2443         struct mm_struct *mm = vma->vm_mm;
2444         unsigned long address;
2445         pte_t *ptep;
2446         pte_t pte;
2447         spinlock_t *ptl;
2448         struct page *page;
2449         struct hstate *h = hstate_vma(vma);
2450         unsigned long sz = huge_page_size(h);
2451         const unsigned long mmun_start = start; /* For mmu_notifiers */
2452         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
2453
2454         WARN_ON(!is_vm_hugetlb_page(vma));
2455         BUG_ON(start & ~huge_page_mask(h));
2456         BUG_ON(end & ~huge_page_mask(h));
2457
2458         tlb_start_vma(tlb, vma);
2459         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2460 again:
2461         for (address = start; address < end; address += sz) {
2462                 ptep = huge_pte_offset(mm, address);
2463                 if (!ptep)
2464                         continue;
2465
2466                 ptl = huge_pte_lock(h, mm, ptep);
2467                 if (huge_pmd_unshare(mm, &address, ptep))
2468                         goto unlock;
2469
2470                 pte = huge_ptep_get(ptep);
2471                 if (huge_pte_none(pte))
2472                         goto unlock;
2473
2474                 /*
2475                  * HWPoisoned hugepage is already unmapped and dropped reference
2476                  */
2477                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2478                         huge_pte_clear(mm, address, ptep);
2479                         goto unlock;
2480                 }
2481
2482                 page = pte_page(pte);
2483                 /*
2484                  * If a reference page is supplied, it is because a specific
2485                  * page is being unmapped, not a range. Ensure the page we
2486                  * are about to unmap is the actual page of interest.
2487                  */
2488                 if (ref_page) {
2489                         if (page != ref_page)
2490                                 goto unlock;
2491
2492                         /*
2493                          * Mark the VMA as having unmapped its page so that
2494                          * future faults in this VMA will fail rather than
2495                          * looking like data was lost
2496                          */
2497                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2498                 }
2499
2500                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2501                 tlb_remove_tlb_entry(tlb, ptep, address);
2502                 if (huge_pte_dirty(pte))
2503                         set_page_dirty(page);
2504
2505                 page_remove_rmap(page);
2506                 force_flush = !__tlb_remove_page(tlb, page);
2507                 if (force_flush) {
2508                         spin_unlock(ptl);
2509                         break;
2510                 }
2511                 /* Bail out after unmapping reference page if supplied */
2512                 if (ref_page) {
2513                         spin_unlock(ptl);
2514                         break;
2515                 }
2516 unlock:
2517                 spin_unlock(ptl);
2518         }
2519         /*
2520          * mmu_gather ran out of room to batch pages, we break out of
2521          * the PTE lock to avoid doing the potential expensive TLB invalidate
2522          * and page-free while holding it.
2523          */
2524         if (force_flush) {
2525                 force_flush = 0;
2526                 tlb_flush_mmu(tlb);
2527                 if (address < end && !ref_page)
2528                         goto again;
2529         }
2530         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2531         tlb_end_vma(tlb, vma);
2532 }
2533
2534 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2535                           struct vm_area_struct *vma, unsigned long start,
2536                           unsigned long end, struct page *ref_page)
2537 {
2538         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
2539
2540         /*
2541          * Clear this flag so that x86's huge_pmd_share page_table_shareable
2542          * test will fail on a vma being torn down, and not grab a page table
2543          * on its way out.  We're lucky that the flag has such an appropriate
2544          * name, and can in fact be safely cleared here. We could clear it
2545          * before the __unmap_hugepage_range above, but all that's necessary
2546          * is to clear it before releasing the i_mmap_mutex. This works
2547          * because in the context this is called, the VMA is about to be
2548          * destroyed and the i_mmap_mutex is held.
2549          */
2550         vma->vm_flags &= ~VM_MAYSHARE;
2551 }
2552
2553 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2554                           unsigned long end, struct page *ref_page)
2555 {
2556         struct mm_struct *mm;
2557         struct mmu_gather tlb;
2558
2559         mm = vma->vm_mm;
2560
2561         tlb_gather_mmu(&tlb, mm, start, end);
2562         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2563         tlb_finish_mmu(&tlb, start, end);
2564 }
2565
2566 /*
2567  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2568  * mappping it owns the reserve page for. The intention is to unmap the page
2569  * from other VMAs and let the children be SIGKILLed if they are faulting the
2570  * same region.
2571  */
2572 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2573                                 struct page *page, unsigned long address)
2574 {
2575         struct hstate *h = hstate_vma(vma);
2576         struct vm_area_struct *iter_vma;
2577         struct address_space *mapping;
2578         pgoff_t pgoff;
2579
2580         /*
2581          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2582          * from page cache lookup which is in HPAGE_SIZE units.
2583          */
2584         address = address & huge_page_mask(h);
2585         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2586                         vma->vm_pgoff;
2587         mapping = file_inode(vma->vm_file)->i_mapping;
2588
2589         /*
2590          * Take the mapping lock for the duration of the table walk. As
2591          * this mapping should be shared between all the VMAs,
2592          * __unmap_hugepage_range() is called as the lock is already held
2593          */
2594         mutex_lock(&mapping->i_mmap_mutex);
2595         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2596                 /* Do not unmap the current VMA */
2597                 if (iter_vma == vma)
2598                         continue;
2599
2600                 /*
2601                  * Unmap the page from other VMAs without their own reserves.
2602                  * They get marked to be SIGKILLed if they fault in these
2603                  * areas. This is because a future no-page fault on this VMA
2604                  * could insert a zeroed page instead of the data existing
2605                  * from the time of fork. This would look like data corruption
2606                  */
2607                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2608                         unmap_hugepage_range(iter_vma, address,
2609                                              address + huge_page_size(h), page);
2610         }
2611         mutex_unlock(&mapping->i_mmap_mutex);
2612
2613         return 1;
2614 }
2615
2616 /*
2617  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2618  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2619  * cannot race with other handlers or page migration.
2620  * Keep the pte_same checks anyway to make transition from the mutex easier.
2621  */
2622 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2623                         unsigned long address, pte_t *ptep, pte_t pte,
2624                         struct page *pagecache_page, spinlock_t *ptl)
2625 {
2626         struct hstate *h = hstate_vma(vma);
2627         struct page *old_page, *new_page;
2628         int outside_reserve = 0;
2629         unsigned long mmun_start;       /* For mmu_notifiers */
2630         unsigned long mmun_end;         /* For mmu_notifiers */
2631
2632         old_page = pte_page(pte);
2633
2634 retry_avoidcopy:
2635         /* If no-one else is actually using this page, avoid the copy
2636          * and just make the page writable */
2637         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2638                 page_move_anon_rmap(old_page, vma, address);
2639                 set_huge_ptep_writable(vma, address, ptep);
2640                 return 0;
2641         }
2642
2643         /*
2644          * If the process that created a MAP_PRIVATE mapping is about to
2645          * perform a COW due to a shared page count, attempt to satisfy
2646          * the allocation without using the existing reserves. The pagecache
2647          * page is used to determine if the reserve at this address was
2648          * consumed or not. If reserves were used, a partial faulted mapping
2649          * at the time of fork() could consume its reserves on COW instead
2650          * of the full address range.
2651          */
2652         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2653                         old_page != pagecache_page)
2654                 outside_reserve = 1;
2655
2656         page_cache_get(old_page);
2657
2658         /* Drop page table lock as buddy allocator may be called */
2659         spin_unlock(ptl);
2660         new_page = alloc_huge_page(vma, address, outside_reserve);
2661
2662         if (IS_ERR(new_page)) {
2663                 long err = PTR_ERR(new_page);
2664                 page_cache_release(old_page);
2665
2666                 /*
2667                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2668                  * it is due to references held by a child and an insufficient
2669                  * huge page pool. To guarantee the original mappers
2670                  * reliability, unmap the page from child processes. The child
2671                  * may get SIGKILLed if it later faults.
2672                  */
2673                 if (outside_reserve) {
2674                         BUG_ON(huge_pte_none(pte));
2675                         if (unmap_ref_private(mm, vma, old_page, address)) {
2676                                 BUG_ON(huge_pte_none(pte));
2677                                 spin_lock(ptl);
2678                                 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2679                                 if (likely(pte_same(huge_ptep_get(ptep), pte)))
2680                                         goto retry_avoidcopy;
2681                                 /*
2682                                  * race occurs while re-acquiring page table
2683                                  * lock, and our job is done.
2684                                  */
2685                                 return 0;
2686                         }
2687                         WARN_ON_ONCE(1);
2688                 }
2689
2690                 /* Caller expects lock to be held */
2691                 spin_lock(ptl);
2692                 if (err == -ENOMEM)
2693                         return VM_FAULT_OOM;
2694                 else
2695                         return VM_FAULT_SIGBUS;
2696         }
2697
2698         /*
2699          * When the original hugepage is shared one, it does not have
2700          * anon_vma prepared.
2701          */
2702         if (unlikely(anon_vma_prepare(vma))) {
2703                 page_cache_release(new_page);
2704                 page_cache_release(old_page);
2705                 /* Caller expects lock to be held */
2706                 spin_lock(ptl);
2707                 return VM_FAULT_OOM;
2708         }
2709
2710         copy_user_huge_page(new_page, old_page, address, vma,
2711                             pages_per_huge_page(h));
2712         __SetPageUptodate(new_page);
2713
2714         mmun_start = address & huge_page_mask(h);
2715         mmun_end = mmun_start + huge_page_size(h);
2716         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2717         /*
2718          * Retake the page table lock to check for racing updates
2719          * before the page tables are altered
2720          */
2721         spin_lock(ptl);
2722         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2723         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2724                 ClearPagePrivate(new_page);
2725
2726                 /* Break COW */
2727                 huge_ptep_clear_flush(vma, address, ptep);
2728                 set_huge_pte_at(mm, address, ptep,
2729                                 make_huge_pte(vma, new_page, 1));
2730                 page_remove_rmap(old_page);
2731                 hugepage_add_new_anon_rmap(new_page, vma, address);
2732                 /* Make the old page be freed below */
2733                 new_page = old_page;
2734         }
2735         spin_unlock(ptl);
2736         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2737         page_cache_release(new_page);
2738         page_cache_release(old_page);
2739
2740         /* Caller expects lock to be held */
2741         spin_lock(ptl);
2742         return 0;
2743 }
2744
2745 /* Return the pagecache page at a given address within a VMA */
2746 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2747                         struct vm_area_struct *vma, unsigned long address)
2748 {
2749         struct address_space *mapping;
2750         pgoff_t idx;
2751
2752         mapping = vma->vm_file->f_mapping;
2753         idx = vma_hugecache_offset(h, vma, address);
2754
2755         return find_lock_page(mapping, idx);
2756 }
2757
2758 /*
2759  * Return whether there is a pagecache page to back given address within VMA.
2760  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2761  */
2762 static bool hugetlbfs_pagecache_present(struct hstate *h,
2763                         struct vm_area_struct *vma, unsigned long address)
2764 {
2765         struct address_space *mapping;
2766         pgoff_t idx;
2767         struct page *page;
2768
2769         mapping = vma->vm_file->f_mapping;
2770         idx = vma_hugecache_offset(h, vma, address);
2771
2772         page = find_get_page(mapping, idx);
2773         if (page)
2774                 put_page(page);
2775         return page != NULL;
2776 }
2777
2778 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2779                         unsigned long address, pte_t *ptep, unsigned int flags)
2780 {
2781         struct hstate *h = hstate_vma(vma);
2782         int ret = VM_FAULT_SIGBUS;
2783         int anon_rmap = 0;
2784         pgoff_t idx;
2785         unsigned long size;
2786         struct page *page;
2787         struct address_space *mapping;
2788         pte_t new_pte;
2789         spinlock_t *ptl;
2790
2791         /*
2792          * Currently, we are forced to kill the process in the event the
2793          * original mapper has unmapped pages from the child due to a failed
2794          * COW. Warn that such a situation has occurred as it may not be obvious
2795          */
2796         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2797                 pr_warning("PID %d killed due to inadequate hugepage pool\n",
2798                            current->pid);
2799                 return ret;
2800         }
2801
2802         mapping = vma->vm_file->f_mapping;
2803         idx = vma_hugecache_offset(h, vma, address);
2804
2805         /*
2806          * Use page lock to guard against racing truncation
2807          * before we get page_table_lock.
2808          */
2809 retry:
2810         page = find_lock_page(mapping, idx);
2811         if (!page) {
2812                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2813                 if (idx >= size)
2814                         goto out;
2815                 page = alloc_huge_page(vma, address, 0);
2816                 if (IS_ERR(page)) {
2817                         ret = PTR_ERR(page);
2818                         if (ret == -ENOMEM)
2819                                 ret = VM_FAULT_OOM;
2820                         else
2821                                 ret = VM_FAULT_SIGBUS;
2822                         goto out;
2823                 }
2824                 clear_huge_page(page, address, pages_per_huge_page(h));
2825                 __SetPageUptodate(page);
2826
2827                 if (vma->vm_flags & VM_MAYSHARE) {
2828                         int err;
2829                         struct inode *inode = mapping->host;
2830
2831                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2832                         if (err) {
2833                                 put_page(page);
2834                                 if (err == -EEXIST)
2835                                         goto retry;
2836                                 goto out;
2837                         }
2838                         ClearPagePrivate(page);
2839
2840                         spin_lock(&inode->i_lock);
2841                         inode->i_blocks += blocks_per_huge_page(h);
2842                         spin_unlock(&inode->i_lock);
2843                 } else {
2844                         lock_page(page);
2845                         if (unlikely(anon_vma_prepare(vma))) {
2846                                 ret = VM_FAULT_OOM;
2847                                 goto backout_unlocked;
2848                         }
2849                         anon_rmap = 1;
2850                 }
2851         } else {
2852                 /*
2853                  * If memory error occurs between mmap() and fault, some process
2854                  * don't have hwpoisoned swap entry for errored virtual address.
2855                  * So we need to block hugepage fault by PG_hwpoison bit check.
2856                  */
2857                 if (unlikely(PageHWPoison(page))) {
2858                         ret = VM_FAULT_HWPOISON |
2859                                 VM_FAULT_SET_HINDEX(hstate_index(h));
2860                         goto backout_unlocked;
2861                 }
2862         }
2863
2864         /*
2865          * If we are going to COW a private mapping later, we examine the
2866          * pending reservations for this page now. This will ensure that
2867          * any allocations necessary to record that reservation occur outside
2868          * the spinlock.
2869          */
2870         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2871                 if (vma_needs_reservation(h, vma, address) < 0) {
2872                         ret = VM_FAULT_OOM;
2873                         goto backout_unlocked;
2874                 }
2875
2876         ptl = huge_pte_lockptr(h, mm, ptep);
2877         spin_lock(ptl);
2878         size = i_size_read(mapping->host) >> huge_page_shift(h);
2879         if (idx >= size)
2880                 goto backout;
2881
2882         ret = 0;
2883         if (!huge_pte_none(huge_ptep_get(ptep)))
2884                 goto backout;
2885
2886         if (anon_rmap) {
2887                 ClearPagePrivate(page);
2888                 hugepage_add_new_anon_rmap(page, vma, address);
2889         }
2890         else
2891                 page_dup_rmap(page);
2892         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2893                                 && (vma->vm_flags & VM_SHARED)));
2894         set_huge_pte_at(mm, address, ptep, new_pte);
2895
2896         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2897                 /* Optimization, do the COW without a second fault */
2898                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
2899         }
2900
2901         spin_unlock(ptl);
2902         unlock_page(page);
2903 out:
2904         return ret;
2905
2906 backout:
2907         spin_unlock(ptl);
2908 backout_unlocked:
2909         unlock_page(page);
2910         put_page(page);
2911         goto out;
2912 }
2913
2914 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2915                         unsigned long address, unsigned int flags)
2916 {
2917         pte_t *ptep;
2918         pte_t entry;
2919         spinlock_t *ptl;
2920         int ret;
2921         struct page *page = NULL;
2922         struct page *pagecache_page = NULL;
2923         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2924         struct hstate *h = hstate_vma(vma);
2925
2926         address &= huge_page_mask(h);
2927
2928         ptep = huge_pte_offset(mm, address);
2929         if (ptep) {
2930                 entry = huge_ptep_get(ptep);
2931                 if (unlikely(is_hugetlb_entry_migration(entry))) {
2932                         migration_entry_wait_huge(vma, mm, ptep);
2933                         return 0;
2934                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2935                         return VM_FAULT_HWPOISON_LARGE |
2936                                 VM_FAULT_SET_HINDEX(hstate_index(h));
2937         }
2938
2939         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2940         if (!ptep)
2941                 return VM_FAULT_OOM;
2942
2943         /*
2944          * Serialize hugepage allocation and instantiation, so that we don't
2945          * get spurious allocation failures if two CPUs race to instantiate
2946          * the same page in the page cache.
2947          */
2948         mutex_lock(&hugetlb_instantiation_mutex);
2949         entry = huge_ptep_get(ptep);
2950         if (huge_pte_none(entry)) {
2951                 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2952                 goto out_mutex;
2953         }
2954
2955         ret = 0;
2956
2957         /*
2958          * If we are going to COW the mapping later, we examine the pending
2959          * reservations for this page now. This will ensure that any
2960          * allocations necessary to record that reservation occur outside the
2961          * spinlock. For private mappings, we also lookup the pagecache
2962          * page now as it is used to determine if a reservation has been
2963          * consumed.
2964          */
2965         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
2966                 if (vma_needs_reservation(h, vma, address) < 0) {
2967                         ret = VM_FAULT_OOM;
2968                         goto out_mutex;
2969                 }
2970
2971                 if (!(vma->vm_flags & VM_MAYSHARE))
2972                         pagecache_page = hugetlbfs_pagecache_page(h,
2973                                                                 vma, address);
2974         }
2975
2976         /*
2977          * hugetlb_cow() requires page locks of pte_page(entry) and
2978          * pagecache_page, so here we need take the former one
2979          * when page != pagecache_page or !pagecache_page.
2980          * Note that locking order is always pagecache_page -> page,
2981          * so no worry about deadlock.
2982          */
2983         page = pte_page(entry);
2984         get_page(page);
2985         if (page != pagecache_page)
2986                 lock_page(page);
2987
2988         ptl = huge_pte_lockptr(h, mm, ptep);
2989         spin_lock(ptl);
2990         /* Check for a racing update before calling hugetlb_cow */
2991         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2992                 goto out_ptl;
2993
2994
2995         if (flags & FAULT_FLAG_WRITE) {
2996                 if (!huge_pte_write(entry)) {
2997                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
2998                                         pagecache_page, ptl);
2999                         goto out_ptl;
3000                 }
3001                 entry = huge_pte_mkdirty(entry);
3002         }
3003         entry = pte_mkyoung(entry);
3004         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3005                                                 flags & FAULT_FLAG_WRITE))
3006                 update_mmu_cache(vma, address, ptep);
3007
3008 out_ptl:
3009         spin_unlock(ptl);
3010
3011         if (pagecache_page) {
3012                 unlock_page(pagecache_page);
3013                 put_page(pagecache_page);
3014         }
3015         if (page != pagecache_page)
3016                 unlock_page(page);
3017         put_page(page);
3018
3019 out_mutex:
3020         mutex_unlock(&hugetlb_instantiation_mutex);
3021
3022         return ret;
3023 }
3024
3025 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3026                          struct page **pages, struct vm_area_struct **vmas,
3027                          unsigned long *position, unsigned long *nr_pages,
3028                          long i, unsigned int flags)
3029 {
3030         unsigned long pfn_offset;
3031         unsigned long vaddr = *position;
3032         unsigned long remainder = *nr_pages;
3033         struct hstate *h = hstate_vma(vma);
3034
3035         while (vaddr < vma->vm_end && remainder) {
3036                 pte_t *pte;
3037                 spinlock_t *ptl = NULL;
3038                 int absent;
3039                 struct page *page;
3040
3041                 /*
3042                  * Some archs (sparc64, sh*) have multiple pte_ts to
3043                  * each hugepage.  We have to make sure we get the
3044                  * first, for the page indexing below to work.
3045                  *
3046                  * Note that page table lock is not held when pte is null.
3047                  */
3048                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3049                 if (pte)
3050                         ptl = huge_pte_lock(h, mm, pte);
3051                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3052
3053                 /*
3054                  * When coredumping, it suits get_dump_page if we just return
3055                  * an error where there's an empty slot with no huge pagecache
3056                  * to back it.  This way, we avoid allocating a hugepage, and
3057                  * the sparse dumpfile avoids allocating disk blocks, but its
3058                  * huge holes still show up with zeroes where they need to be.
3059                  */
3060                 if (absent && (flags & FOLL_DUMP) &&
3061                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3062                         if (pte)
3063                                 spin_unlock(ptl);
3064                         remainder = 0;
3065                         break;
3066                 }
3067
3068                 /*
3069                  * We need call hugetlb_fault for both hugepages under migration
3070                  * (in which case hugetlb_fault waits for the migration,) and
3071                  * hwpoisoned hugepages (in which case we need to prevent the
3072                  * caller from accessing to them.) In order to do this, we use
3073                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3074                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3075                  * both cases, and because we can't follow correct pages
3076                  * directly from any kind of swap entries.
3077                  */
3078                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3079                     ((flags & FOLL_WRITE) &&
3080                       !huge_pte_write(huge_ptep_get(pte)))) {
3081                         int ret;
3082
3083                         if (pte)
3084                                 spin_unlock(ptl);
3085                         ret = hugetlb_fault(mm, vma, vaddr,
3086                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3087                         if (!(ret & VM_FAULT_ERROR))
3088                                 continue;
3089
3090                         remainder = 0;
3091                         break;
3092                 }
3093
3094                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3095                 page = pte_page(huge_ptep_get(pte));
3096 same_page:
3097                 if (pages) {
3098                         pages[i] = mem_map_offset(page, pfn_offset);
3099                         get_page(pages[i]);
3100                 }
3101
3102                 if (vmas)
3103                         vmas[i] = vma;
3104
3105                 vaddr += PAGE_SIZE;
3106                 ++pfn_offset;
3107                 --remainder;
3108                 ++i;
3109                 if (vaddr < vma->vm_end && remainder &&
3110                                 pfn_offset < pages_per_huge_page(h)) {
3111                         /*
3112                          * We use pfn_offset to avoid touching the pageframes
3113                          * of this compound page.
3114                          */
3115                         goto same_page;
3116                 }
3117                 spin_unlock(ptl);
3118         }
3119         *nr_pages = remainder;
3120         *position = vaddr;
3121
3122         return i ? i : -EFAULT;
3123 }
3124
3125 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3126                 unsigned long address, unsigned long end, pgprot_t newprot)
3127 {
3128         struct mm_struct *mm = vma->vm_mm;
3129         unsigned long start = address;
3130         pte_t *ptep;
3131         pte_t pte;
3132         struct hstate *h = hstate_vma(vma);
3133         unsigned long pages = 0;
3134
3135         BUG_ON(address >= end);
3136         flush_cache_range(vma, address, end);
3137
3138         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
3139         for (; address < end; address += huge_page_size(h)) {
3140                 spinlock_t *ptl;
3141                 ptep = huge_pte_offset(mm, address);
3142                 if (!ptep)
3143                         continue;
3144                 ptl = huge_pte_lock(h, mm, ptep);
3145                 if (huge_pmd_unshare(mm, &address, ptep)) {
3146                         pages++;
3147                         spin_unlock(ptl);
3148                         continue;
3149                 }
3150                 if (!huge_pte_none(huge_ptep_get(ptep))) {
3151                         pte = huge_ptep_get_and_clear(mm, address, ptep);
3152                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3153                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
3154                         set_huge_pte_at(mm, address, ptep, pte);
3155                         pages++;
3156                 }
3157                 spin_unlock(ptl);
3158         }
3159         /*
3160          * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
3161          * may have cleared our pud entry and done put_page on the page table:
3162          * once we release i_mmap_mutex, another task can do the final put_page
3163          * and that page table be reused and filled with junk.
3164          */
3165         flush_tlb_range(vma, start, end);
3166         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
3167
3168         return pages << h->order;
3169 }
3170
3171 int hugetlb_reserve_pages(struct inode *inode,
3172                                         long from, long to,
3173                                         struct vm_area_struct *vma,
3174                                         vm_flags_t vm_flags)
3175 {
3176         long ret, chg;
3177         struct hstate *h = hstate_inode(inode);
3178         struct hugepage_subpool *spool = subpool_inode(inode);
3179
3180         /*
3181          * Only apply hugepage reservation if asked. At fault time, an
3182          * attempt will be made for VM_NORESERVE to allocate a page
3183          * without using reserves
3184          */
3185         if (vm_flags & VM_NORESERVE)
3186                 return 0;
3187
3188         /*
3189          * Shared mappings base their reservation on the number of pages that
3190          * are already allocated on behalf of the file. Private mappings need
3191          * to reserve the full area even if read-only as mprotect() may be
3192          * called to make the mapping read-write. Assume !vma is a shm mapping
3193          */
3194         if (!vma || vma->vm_flags & VM_MAYSHARE)
3195                 chg = region_chg(&inode->i_mapping->private_list, from, to);
3196         else {
3197                 struct resv_map *resv_map = resv_map_alloc();
3198                 if (!resv_map)
3199                         return -ENOMEM;
3200
3201                 chg = to - from;
3202
3203                 set_vma_resv_map(vma, resv_map);
3204                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3205         }
3206
3207         if (chg < 0) {
3208                 ret = chg;
3209                 goto out_err;
3210         }
3211
3212         /* There must be enough pages in the subpool for the mapping */
3213         if (hugepage_subpool_get_pages(spool, chg)) {
3214                 ret = -ENOSPC;
3215                 goto out_err;
3216         }
3217
3218         /*
3219          * Check enough hugepages are available for the reservation.
3220          * Hand the pages back to the subpool if there are not
3221          */
3222         ret = hugetlb_acct_memory(h, chg);
3223         if (ret < 0) {
3224                 hugepage_subpool_put_pages(spool, chg);
3225                 goto out_err;
3226         }
3227
3228         /*
3229          * Account for the reservations made. Shared mappings record regions
3230          * that have reservations as they are shared by multiple VMAs.
3231          * When the last VMA disappears, the region map says how much
3232          * the reservation was and the page cache tells how much of
3233          * the reservation was consumed. Private mappings are per-VMA and
3234          * only the consumed reservations are tracked. When the VMA
3235          * disappears, the original reservation is the VMA size and the
3236          * consumed reservations are stored in the map. Hence, nothing
3237          * else has to be done for private mappings here
3238          */
3239         if (!vma || vma->vm_flags & VM_MAYSHARE)
3240                 region_add(&inode->i_mapping->private_list, from, to);
3241         return 0;
3242 out_err:
3243         if (vma)
3244                 resv_map_put(vma);
3245         return ret;
3246 }
3247
3248 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3249 {
3250         struct hstate *h = hstate_inode(inode);
3251         long chg = region_truncate(&inode->i_mapping->private_list, offset);
3252         struct hugepage_subpool *spool = subpool_inode(inode);
3253
3254         spin_lock(&inode->i_lock);
3255         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3256         spin_unlock(&inode->i_lock);
3257
3258         hugepage_subpool_put_pages(spool, (chg - freed));
3259         hugetlb_acct_memory(h, -(chg - freed));
3260 }
3261
3262 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3263 static unsigned long page_table_shareable(struct vm_area_struct *svma,
3264                                 struct vm_area_struct *vma,
3265                                 unsigned long addr, pgoff_t idx)
3266 {
3267         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3268                                 svma->vm_start;
3269         unsigned long sbase = saddr & PUD_MASK;
3270         unsigned long s_end = sbase + PUD_SIZE;
3271
3272         /* Allow segments to share if only one is marked locked */
3273         unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3274         unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3275
3276         /*
3277          * match the virtual addresses, permission and the alignment of the
3278          * page table page.
3279          */
3280         if (pmd_index(addr) != pmd_index(saddr) ||
3281             vm_flags != svm_flags ||
3282             sbase < svma->vm_start || svma->vm_end < s_end)
3283                 return 0;
3284
3285         return saddr;
3286 }
3287
3288 static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3289 {
3290         unsigned long base = addr & PUD_MASK;
3291         unsigned long end = base + PUD_SIZE;
3292
3293         /*
3294          * check on proper vm_flags and page table alignment
3295          */
3296         if (vma->vm_flags & VM_MAYSHARE &&
3297             vma->vm_start <= base && end <= vma->vm_end)
3298                 return 1;
3299         return 0;
3300 }
3301
3302 /*
3303  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3304  * and returns the corresponding pte. While this is not necessary for the
3305  * !shared pmd case because we can allocate the pmd later as well, it makes the
3306  * code much cleaner. pmd allocation is essential for the shared case because
3307  * pud has to be populated inside the same i_mmap_mutex section - otherwise
3308  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3309  * bad pmd for sharing.
3310  */
3311 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3312 {
3313         struct vm_area_struct *vma = find_vma(mm, addr);
3314         struct address_space *mapping = vma->vm_file->f_mapping;
3315         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3316                         vma->vm_pgoff;
3317         struct vm_area_struct *svma;
3318         unsigned long saddr;
3319         pte_t *spte = NULL;
3320         pte_t *pte;
3321         spinlock_t *ptl;
3322
3323         if (!vma_shareable(vma, addr))
3324                 return (pte_t *)pmd_alloc(mm, pud, addr);
3325
3326         mutex_lock(&mapping->i_mmap_mutex);
3327         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3328                 if (svma == vma)
3329                         continue;
3330
3331                 saddr = page_table_shareable(svma, vma, addr, idx);
3332                 if (saddr) {
3333                         spte = huge_pte_offset(svma->vm_mm, saddr);
3334                         if (spte) {
3335                                 get_page(virt_to_page(spte));
3336                                 break;
3337                         }
3338                 }
3339         }
3340
3341         if (!spte)
3342                 goto out;
3343
3344         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3345         spin_lock(ptl);
3346         if (pud_none(*pud))
3347                 pud_populate(mm, pud,
3348                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
3349         else
3350                 put_page(virt_to_page(spte));
3351         spin_unlock(ptl);
3352 out:
3353         pte = (pte_t *)pmd_alloc(mm, pud, addr);
3354         mutex_unlock(&mapping->i_mmap_mutex);
3355         return pte;
3356 }
3357
3358 /*
3359  * unmap huge page backed by shared pte.
3360  *
3361  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
3362  * indicated by page_count > 1, unmap is achieved by clearing pud and
3363  * decrementing the ref count. If count == 1, the pte page is not shared.
3364  *
3365  * called with page table lock held.
3366  *
3367  * returns: 1 successfully unmapped a shared pte page
3368  *          0 the underlying pte page is not shared, or it is the last user
3369  */
3370 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3371 {
3372         pgd_t *pgd = pgd_offset(mm, *addr);
3373         pud_t *pud = pud_offset(pgd, *addr);
3374
3375         BUG_ON(page_count(virt_to_page(ptep)) == 0);
3376         if (page_count(virt_to_page(ptep)) == 1)
3377                 return 0;
3378
3379         pud_clear(pud);
3380         put_page(virt_to_page(ptep));
3381         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3382         return 1;
3383 }
3384 #define want_pmd_share()        (1)
3385 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3386 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3387 {
3388         return NULL;
3389 }
3390 #define want_pmd_share()        (0)
3391 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3392
3393 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3394 pte_t *huge_pte_alloc(struct mm_struct *mm,
3395                         unsigned long addr, unsigned long sz)
3396 {
3397         pgd_t *pgd;
3398         pud_t *pud;
3399         pte_t *pte = NULL;
3400
3401         pgd = pgd_offset(mm, addr);
3402         pud = pud_alloc(mm, pgd, addr);
3403         if (pud) {
3404                 if (sz == PUD_SIZE) {
3405                         pte = (pte_t *)pud;
3406                 } else {
3407                         BUG_ON(sz != PMD_SIZE);
3408                         if (want_pmd_share() && pud_none(*pud))
3409                                 pte = huge_pmd_share(mm, addr, pud);
3410                         else
3411                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3412                 }
3413         }
3414         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3415
3416         return pte;
3417 }
3418
3419 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3420 {
3421         pgd_t *pgd;
3422         pud_t *pud;
3423         pmd_t *pmd = NULL;
3424
3425         pgd = pgd_offset(mm, addr);
3426         if (pgd_present(*pgd)) {
3427                 pud = pud_offset(pgd, addr);
3428                 if (pud_present(*pud)) {
3429                         if (pud_huge(*pud))
3430                                 return (pte_t *)pud;
3431                         pmd = pmd_offset(pud, addr);
3432                 }
3433         }
3434         return (pte_t *) pmd;
3435 }
3436
3437 struct page *
3438 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3439                 pmd_t *pmd, int write)
3440 {
3441         struct page *page;
3442
3443         page = pte_page(*(pte_t *)pmd);
3444         if (page)
3445                 page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
3446         return page;
3447 }
3448
3449 struct page *
3450 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3451                 pud_t *pud, int write)
3452 {
3453         struct page *page;
3454
3455         page = pte_page(*(pte_t *)pud);
3456         if (page)
3457                 page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
3458         return page;
3459 }
3460
3461 #else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3462
3463 /* Can be overriden by architectures */
3464 __attribute__((weak)) struct page *
3465 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3466                pud_t *pud, int write)
3467 {
3468         BUG();
3469         return NULL;
3470 }
3471
3472 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3473
3474 #ifdef CONFIG_MEMORY_FAILURE
3475
3476 /* Should be called in hugetlb_lock */
3477 static int is_hugepage_on_freelist(struct page *hpage)
3478 {
3479         struct page *page;
3480         struct page *tmp;
3481         struct hstate *h = page_hstate(hpage);
3482         int nid = page_to_nid(hpage);
3483
3484         list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3485                 if (page == hpage)
3486                         return 1;
3487         return 0;
3488 }
3489
3490 /*
3491  * This function is called from memory failure code.
3492  * Assume the caller holds page lock of the head page.
3493  */
3494 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3495 {
3496         struct hstate *h = page_hstate(hpage);
3497         int nid = page_to_nid(hpage);
3498         int ret = -EBUSY;
3499
3500         spin_lock(&hugetlb_lock);
3501         if (is_hugepage_on_freelist(hpage)) {
3502                 /*
3503                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
3504                  * but dangling hpage->lru can trigger list-debug warnings
3505                  * (this happens when we call unpoison_memory() on it),
3506                  * so let it point to itself with list_del_init().
3507                  */
3508                 list_del_init(&hpage->lru);
3509                 set_page_refcounted(hpage);
3510                 h->free_huge_pages--;
3511                 h->free_huge_pages_node[nid]--;
3512                 ret = 0;
3513         }
3514         spin_unlock(&hugetlb_lock);
3515         return ret;
3516 }
3517 #endif
3518
3519 bool isolate_huge_page(struct page *page, struct list_head *list)
3520 {
3521         VM_BUG_ON(!PageHead(page));
3522         if (!get_page_unless_zero(page))
3523                 return false;
3524         spin_lock(&hugetlb_lock);
3525         list_move_tail(&page->lru, list);
3526         spin_unlock(&hugetlb_lock);
3527         return true;
3528 }
3529
3530 void putback_active_hugepage(struct page *page)
3531 {
3532         VM_BUG_ON(!PageHead(page));
3533         spin_lock(&hugetlb_lock);
3534         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3535         spin_unlock(&hugetlb_lock);
3536         put_page(page);
3537 }
3538
3539 bool is_hugepage_active(struct page *page)
3540 {
3541         VM_BUG_ON(!PageHuge(page));
3542         /*
3543          * This function can be called for a tail page because the caller,
3544          * scan_movable_pages, scans through a given pfn-range which typically
3545          * covers one memory block. In systems using gigantic hugepage (1GB
3546          * for x86_64,) a hugepage is larger than a memory block, and we don't
3547          * support migrating such large hugepages for now, so return false
3548          * when called for tail pages.
3549          */
3550         if (PageTail(page))
3551                 return false;
3552         /*
3553          * Refcount of a hwpoisoned hugepages is 1, but they are not active,
3554          * so we should return false for them.
3555          */
3556         if (unlikely(PageHWPoison(page)))
3557                 return false;
3558         return page_count(page) > 0;
3559 }