mm/mremap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *      mm/mremap.c
   4  *
   5  *      (C) Copyright 1996 Linus Torvalds
   6  *
   7  *      Address space accounting code   <alan@lxorguk.ukuu.org.uk>
   8  *      (C) Copyright 2002 Red Hat Inc, All Rights Reserved
   9  */
  10
  11 #include <linux/mm.h>
  12 #include <linux/hugetlb.h>
  13 #include <linux/shm.h>
  14 #include <linux/ksm.h>
  15 #include <linux/mman.h>
  16 #include <linux/swap.h>
  17 #include <linux/capability.h>
  18 #include <linux/fs.h>
  19 #include <linux/swapops.h>
  20 #include <linux/highmem.h>
  21 #include <linux/security.h>
  22 #include <linux/syscalls.h>
  23 #include <linux/mmu_notifier.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/userfaultfd_k.h>
  26
  27 #include <asm/cacheflush.h>
  28 #include <asm/tlbflush.h>
  29
  30 #include "internal.h"
  31
  32 static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
  33 {
  34         pgd_t *pgd;
  35         p4d_t *p4d;
  36         pud_t *pud;
  37
  38         pgd = pgd_offset(mm, addr);
  39         if (pgd_none_or_clear_bad(pgd))
  40                 return NULL;
  41
  42         p4d = p4d_offset(pgd, addr);
  43         if (p4d_none_or_clear_bad(p4d))
  44                 return NULL;
  45
  46         pud = pud_offset(p4d, addr);
  47         if (pud_none_or_clear_bad(pud))
  48                 return NULL;
  49
  50         return pud;
  51 }
  52
  53 static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
  54 {
  55         pud_t *pud;
  56         pmd_t *pmd;
  57
  58         pud = get_old_pud(mm, addr);
  59         if (!pud)
  60                 return NULL;
  61
  62         pmd = pmd_offset(pud, addr);
  63         if (pmd_none(*pmd))
  64                 return NULL;
  65
  66         return pmd;
  67 }
  68
  69 static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma,
  70                             unsigned long addr)
  71 {
  72         pgd_t *pgd;
  73         p4d_t *p4d;
  74
  75         pgd = pgd_offset(mm, addr);
  76         p4d = p4d_alloc(mm, pgd, addr);
  77         if (!p4d)
  78                 return NULL;
  79
  80         return pud_alloc(mm, p4d, addr);
  81 }
  82
  83 static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
  84                             unsigned long addr)
  85 {
  86         pud_t *pud;
  87         pmd_t *pmd;
  88
  89         pud = alloc_new_pud(mm, vma, addr);
  90         if (!pud)
  91                 return NULL;
  92
  93         pmd = pmd_alloc(mm, pud, addr);
  94         if (!pmd)
  95                 return NULL;
  96
  97         VM_BUG_ON(pmd_trans_huge(*pmd));
  98
  99         return pmd;
 100 }
 101
 102 static void take_rmap_locks(struct vm_area_struct *vma)
 103 {
 104         if (vma->vm_file)
 105                 i_mmap_lock_write(vma->vm_file->f_mapping);
 106         if (vma->anon_vma)
 107                 anon_vma_lock_write(vma->anon_vma);
 108 }
 109
 110 static void drop_rmap_locks(struct vm_area_struct *vma)
 111 {
 112         if (vma->anon_vma)
 113                 anon_vma_unlock_write(vma->anon_vma);
 114         if (vma->vm_file)
 115                 i_mmap_unlock_write(vma->vm_file->f_mapping);
 116 }
 117
 118 static pte_t move_soft_dirty_pte(pte_t pte)
 119 {
 120         /*
 121          * Set soft dirty bit so we can notice
 122          * in userspace the ptes were moved.
 123          */
 124 #ifdef CONFIG_MEM_SOFT_DIRTY
 125         if (pte_present(pte))
 126                 pte = pte_mksoft_dirty(pte);
 127         else if (is_swap_pte(pte))
 128                 pte = pte_swp_mksoft_dirty(pte);
 129 #endif
 130         return pte;
 131 }
 132
 133 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
 134                 unsigned long old_addr, unsigned long old_end,
 135                 struct vm_area_struct *new_vma, pmd_t *new_pmd,
 136                 unsigned long new_addr, bool need_rmap_locks)
 137 {
 138         struct mm_struct *mm = vma->vm_mm;
 139         pte_t *old_pte, *new_pte, pte;
 140         spinlock_t *old_ptl, *new_ptl;
 141         bool force_flush = false;
 142         unsigned long len = old_end - old_addr;
 143
 144         /*
 145          * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma
 146          * locks to ensure that rmap will always observe either the old or the
 147          * new ptes. This is the easiest way to avoid races with
 148          * truncate_pagecache(), page migration, etc...
 149          *
 150          * When need_rmap_locks is false, we use other ways to avoid
 151          * such races:
 152          *
 153          * - During exec() shift_arg_pages(), we use a specially tagged vma
 154          *   which rmap call sites look for using vma_is_temporary_stack().
 155          *
 156          * - During mremap(), new_vma is often known to be placed after vma
 157          *   in rmap traversal order. This ensures rmap will always observe
 158          *   either the old pte, or the new pte, or both (the page table locks
 159          *   serialize access to individual ptes, but only rmap traversal
 160          *   order guarantees that we won't miss both the old and new ptes).
 161          */
 162         if (need_rmap_locks)
 163                 take_rmap_locks(vma);
 164
 165         /*
 166          * We don't have to worry about the ordering of src and dst
 167          * pte locks because exclusive mmap_lock prevents deadlock.
 168          */
 169         old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
 170         new_pte = pte_offset_map(new_pmd, new_addr);
 171         new_ptl = pte_lockptr(mm, new_pmd);
 172         if (new_ptl != old_ptl)
 173                 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
 174         flush_tlb_batched_pending(vma->vm_mm);
 175         arch_enter_lazy_mmu_mode();
 176
 177         for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
 178                                    new_pte++, new_addr += PAGE_SIZE) {
 179                 if (pte_none(*old_pte))
 180                         continue;
 181
 182                 pte = ptep_get_and_clear(mm, old_addr, old_pte);
 183                 /*
 184                  * If we are remapping a valid PTE, make sure
 185                  * to flush TLB before we drop the PTL for the
 186                  * PTE.
 187                  *
 188                  * NOTE! Both old and new PTL matter: the old one
 189                  * for racing with page_mkclean(), the new one to
 190                  * make sure the physical page stays valid until
 191                  * the TLB entry for the old mapping has been
 192                  * flushed.
 193                  */
 194                 if (pte_present(pte))
 195                         force_flush = true;
 196                 pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
 197                 pte = move_soft_dirty_pte(pte);
 198                 set_pte_at(mm, new_addr, new_pte, pte);
 199         }
 200
 201         arch_leave_lazy_mmu_mode();
 202         if (force_flush)
 203                 flush_tlb_range(vma, old_end - len, old_end);
 204         if (new_ptl != old_ptl)
 205                 spin_unlock(new_ptl);
 206         pte_unmap(new_pte - 1);
 207         pte_unmap_unlock(old_pte - 1, old_ptl);
 208         if (need_rmap_locks)
 209                 drop_rmap_locks(vma);
 210 }
 211
 212 #ifdef CONFIG_HAVE_MOVE_PMD
 213 static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 214                   unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
 215 {
 216         spinlock_t *old_ptl, *new_ptl;
 217         struct mm_struct *mm = vma->vm_mm;
 218         pmd_t pmd;
 219
 220         /*
 221          * The destination pmd shouldn't be established, free_pgtables()
 222          * should have released it.
 223          *
 224          * However, there's a case during execve() where we use mremap
 225          * to move the initial stack, and in that case the target area
 226          * may overlap the source area (always moving down).
 227          *
 228          * If everything is PMD-aligned, that works fine, as moving
 229          * each pmd down will clear the source pmd. But if we first
 230          * have a few 4kB-only pages that get moved down, and then
 231          * hit the "now the rest is PMD-aligned, let's do everything
 232          * one pmd at a time", we will still have the old (now empty
 233          * of any 4kB pages, but still there) PMD in the page table
 234          * tree.
 235          *
 236          * Warn on it once - because we really should try to figure
 237          * out how to do this better - but then say "I won't move
 238          * this pmd".
 239          *
 240          * One alternative might be to just unmap the target pmd at
 241          * this point, and verify that it really is empty. We'll see.
 242          */
 243         if (WARN_ON_ONCE(!pmd_none(*new_pmd)))
 244                 return false;
 245
 246         /*
 247          * We don't have to worry about the ordering of src and dst
 248          * ptlocks because exclusive mmap_lock prevents deadlock.
 249          */
 250         old_ptl = pmd_lock(vma->vm_mm, old_pmd);
 251         new_ptl = pmd_lockptr(mm, new_pmd);
 252         if (new_ptl != old_ptl)
 253                 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
 254
 255         /* Clear the pmd */
 256         pmd = *old_pmd;
 257         pmd_clear(old_pmd);
 258
 259         VM_BUG_ON(!pmd_none(*new_pmd));
 260
 261         /* Set the new pmd */
 262         set_pmd_at(mm, new_addr, new_pmd, pmd);
 263         flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
 264         if (new_ptl != old_ptl)
 265                 spin_unlock(new_ptl);
 266         spin_unlock(old_ptl);
 267
 268         return true;
 269 }
 270 #else
 271 static inline bool move_normal_pmd(struct vm_area_struct *vma,
 272                 unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd,
 273                 pmd_t *new_pmd)
 274 {
 275         return false;
 276 }
 277 #endif
 278
 279 #ifdef CONFIG_HAVE_MOVE_PUD
 280 static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
 281                   unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
 282 {
 283         spinlock_t *old_ptl, *new_ptl;
 284         struct mm_struct *mm = vma->vm_mm;
 285         pud_t pud;
 286
 287         /*
 288          * The destination pud shouldn't be established, free_pgtables()
 289          * should have released it.
 290          */
 291         if (WARN_ON_ONCE(!pud_none(*new_pud)))
 292                 return false;
 293
 294         /*
 295          * We don't have to worry about the ordering of src and dst
 296          * ptlocks because exclusive mmap_lock prevents deadlock.
 297          */
 298         old_ptl = pud_lock(vma->vm_mm, old_pud);
 299         new_ptl = pud_lockptr(mm, new_pud);
 300         if (new_ptl != old_ptl)
 301                 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
 302
 303         /* Clear the pud */
 304         pud = *old_pud;
 305         pud_clear(old_pud);
 306
 307         VM_BUG_ON(!pud_none(*new_pud));
 308
 309         /* Set the new pud */
 310         set_pud_at(mm, new_addr, new_pud, pud);
 311         flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE);
 312         if (new_ptl != old_ptl)
 313                 spin_unlock(new_ptl);
 314         spin_unlock(old_ptl);
 315
 316         return true;
 317 }
 318 #else
 319 static inline bool move_normal_pud(struct vm_area_struct *vma,
 320                 unsigned long old_addr, unsigned long new_addr, pud_t *old_pud,
 321                 pud_t *new_pud)
 322 {
 323         return false;
 324 }
 325 #endif
 326
 327 enum pgt_entry {
 328         NORMAL_PMD,
 329         HPAGE_PMD,
 330         NORMAL_PUD,
 331 };
 332
 333 /*
 334  * Returns an extent of the corresponding size for the pgt_entry specified if
 335  * valid. Else returns a smaller extent bounded by the end of the source and
 336  * destination pgt_entry.
 337  */
 338 static __always_inline unsigned long get_extent(enum pgt_entry entry,
 339                         unsigned long old_addr, unsigned long old_end,
 340                         unsigned long new_addr)
 341 {
 342         unsigned long next, extent, mask, size;
 343
 344         switch (entry) {
 345         case HPAGE_PMD:
 346         case NORMAL_PMD:
 347                 mask = PMD_MASK;
 348                 size = PMD_SIZE;
 349                 break;
 350         case NORMAL_PUD:
 351                 mask = PUD_MASK;
 352                 size = PUD_SIZE;
 353                 break;
 354         default:
 355                 BUILD_BUG();
 356                 break;
 357         }
 358
 359         next = (old_addr + size) & mask;
 360         /* even if next overflowed, extent below will be ok */
 361         extent = next - old_addr;
 362         if (extent > old_end - old_addr)
 363                 extent = old_end - old_addr;
 364         next = (new_addr + size) & mask;
 365         if (extent > next - new_addr)
 366                 extent = next - new_addr;
 367         return extent;
 368 }
 369
 370 /*
 371  * Attempts to speedup the move by moving entry at the level corresponding to
 372  * pgt_entry. Returns true if the move was successful, else false.
 373  */
 374 static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma,
 375                         unsigned long old_addr, unsigned long new_addr,
 376                         void *old_entry, void *new_entry, bool need_rmap_locks)
 377 {
 378         bool moved = false;
 379
 380         /* See comment in move_ptes() */
 381         if (need_rmap_locks)
 382                 take_rmap_locks(vma);
 383
 384         switch (entry) {
 385         case NORMAL_PMD:
 386                 moved = move_normal_pmd(vma, old_addr, new_addr, old_entry,
 387                                         new_entry);
 388                 break;
 389         case NORMAL_PUD:
 390                 moved = move_normal_pud(vma, old_addr, new_addr, old_entry,
 391                                         new_entry);
 392                 break;
 393         case HPAGE_PMD:
 394                 moved = IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
 395                         move_huge_pmd(vma, old_addr, new_addr, old_entry,
 396                                       new_entry);
 397                 break;
 398         default:
 399                 WARN_ON_ONCE(1);
 400                 break;
 401         }
 402
 403         if (need_rmap_locks)
 404                 drop_rmap_locks(vma);
 405
 406         return moved;
 407 }
 408
 409 unsigned long move_page_tables(struct vm_area_struct *vma,
 410                 unsigned long old_addr, struct vm_area_struct *new_vma,
 411                 unsigned long new_addr, unsigned long len,
 412                 bool need_rmap_locks)
 413 {
 414         unsigned long extent, old_end;
 415         struct mmu_notifier_range range;
 416         pmd_t *old_pmd, *new_pmd;
 417
 418         old_end = old_addr + len;
 419         flush_cache_range(vma, old_addr, old_end);
 420
 421         mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
 422                                 old_addr, old_end);
 423         mmu_notifier_invalidate_range_start(&range);
 424
 425         for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
 426                 cond_resched();
 427                 /*
 428                  * If extent is PUD-sized try to speed up the move by moving at the
 429                  * PUD level if possible.
 430                  */
 431                 extent = get_extent(NORMAL_PUD, old_addr, old_end, new_addr);
 432                 if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) {
 433                         pud_t *old_pud, *new_pud;
 434
 435                         old_pud = get_old_pud(vma->vm_mm, old_addr);
 436                         if (!old_pud)
 437                                 continue;
 438                         new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr);
 439                         if (!new_pud)
 440                                 break;
 441                         if (move_pgt_entry(NORMAL_PUD, vma, old_addr, new_addr,
 442                                            old_pud, new_pud, need_rmap_locks))
 443                                 continue;
 444                 }
 445
 446                 extent = get_extent(NORMAL_PMD, old_addr, old_end, new_addr);
 447                 old_pmd = get_old_pmd(vma->vm_mm, old_addr);
 448                 if (!old_pmd)
 449                         continue;
 450                 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
 451                 if (!new_pmd)
 452                         break;
 453                 if (is_swap_pmd(*old_pmd) || pmd_trans_huge(*old_pmd) ||
 454                     pmd_devmap(*old_pmd)) {
 455                         if (extent == HPAGE_PMD_SIZE &&
 456                             move_pgt_entry(HPAGE_PMD, vma, old_addr, new_addr,
 457                                            old_pmd, new_pmd, need_rmap_locks))
 458                                 continue;
 459                         split_huge_pmd(vma, old_pmd, old_addr);
 460                         if (pmd_trans_unstable(old_pmd))
 461                                 continue;
 462                 } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) &&
 463                            extent == PMD_SIZE) {
 464                         /*
 465                          * If the extent is PMD-sized, try to speed the move by
 466                          * moving at the PMD level if possible.
 467                          */
 468                         if (move_pgt_entry(NORMAL_PMD, vma, old_addr, new_addr,
 469                                            old_pmd, new_pmd, need_rmap_locks))
 470                                 continue;
 471                 }
 472
 473                 if (pte_alloc(new_vma->vm_mm, new_pmd))
 474                         break;
 475                 move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma,
 476                           new_pmd, new_addr, need_rmap_locks);
 477         }
 478
 479         mmu_notifier_invalidate_range_end(&range);
 480
 481         return len + old_addr - old_end;        /* how much done */
 482 }
 483
 484 static unsigned long move_vma(struct vm_area_struct *vma,
 485                 unsigned long old_addr, unsigned long old_len,
 486                 unsigned long new_len, unsigned long new_addr,
 487                 bool *locked, unsigned long flags,
 488                 struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
 489 {
 490         struct mm_struct *mm = vma->vm_mm;
 491         struct vm_area_struct *new_vma;
 492         unsigned long vm_flags = vma->vm_flags;
 493         unsigned long new_pgoff;
 494         unsigned long moved_len;
 495         unsigned long excess = 0;
 496         unsigned long hiwater_vm;
 497         int split = 0;
 498         int err = 0;
 499         bool need_rmap_locks;
 500
 501         /*
 502          * We'd prefer to avoid failure later on in do_munmap:
 503          * which may split one vma into three before unmapping.
 504          */
 505         if (mm->map_count >= sysctl_max_map_count - 3)
 506                 return -ENOMEM;
 507
 508         if (vma->vm_ops && vma->vm_ops->may_split) {
 509                 if (vma->vm_start != old_addr)
 510                         err = vma->vm_ops->may_split(vma, old_addr);
 511                 if (!err && vma->vm_end != old_addr + old_len)
 512                         err = vma->vm_ops->may_split(vma, old_addr + old_len);
 513                 if (err)
 514                         return err;
 515         }
 516
 517         /*
 518          * Advise KSM to break any KSM pages in the area to be moved:
 519          * it would be confusing if they were to turn up at the new
 520          * location, where they happen to coincide with different KSM
 521          * pages recently unmapped.  But leave vma->vm_flags as it was,
 522          * so KSM can come around to merge on vma and new_vma afterwards.
 523          */
 524         err = ksm_madvise(vma, old_addr, old_addr + old_len,
 525                                                 MADV_UNMERGEABLE, &vm_flags);
 526         if (err)
 527                 return err;
 528
 529         if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT)) {
 530                 if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
 531                         return -ENOMEM;
 532         }
 533
 534         new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
 535         new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
 536                            &need_rmap_locks);
 537         if (!new_vma) {
 538                 if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT))
 539                         vm_unacct_memory(new_len >> PAGE_SHIFT);
 540                 return -ENOMEM;
 541         }
 542
 543         moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
 544                                      need_rmap_locks);
 545         if (moved_len < old_len) {
 546                 err = -ENOMEM;
 547         } else if (vma->vm_ops && vma->vm_ops->mremap) {
 548                 err = vma->vm_ops->mremap(new_vma);
 549         }
 550
 551         if (unlikely(err)) {
 552                 /*
 553                  * On error, move entries back from new area to old,
 554                  * which will succeed since page tables still there,
 555                  * and then proceed to unmap new area instead of old.
 556                  */
 557                 move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
 558                                  true);
 559                 vma = new_vma;
 560                 old_len = new_len;
 561                 old_addr = new_addr;
 562                 new_addr = err;
 563         } else {
 564                 mremap_userfaultfd_prep(new_vma, uf);
 565         }
 566
 567         /* Conceal VM_ACCOUNT so old reservation is not undone */
 568         if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP)) {
 569                 vma->vm_flags &= ~VM_ACCOUNT;
 570                 excess = vma->vm_end - vma->vm_start - old_len;
 571                 if (old_addr > vma->vm_start &&
 572                     old_addr + old_len < vma->vm_end)
 573                         split = 1;
 574         }
 575
 576         /*
 577          * If we failed to move page tables we still do total_vm increment
 578          * since do_munmap() will decrement it by old_len == new_len.
 579          *
 580          * Since total_vm is about to be raised artificially high for a
 581          * moment, we need to restore high watermark afterwards: if stats
 582          * are taken meanwhile, total_vm and hiwater_vm appear too high.
 583          * If this were a serious issue, we'd add a flag to do_munmap().
 584          */
 585         hiwater_vm = mm->hiwater_vm;
 586         vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT);
 587
 588         /* Tell pfnmap has moved from this vma */
 589         if (unlikely(vma->vm_flags & VM_PFNMAP))
 590                 untrack_pfn_moved(vma);
 591
 592         if (unlikely(!err && (flags & MREMAP_DONTUNMAP))) {
 593                 /* We always clear VM_LOCKED[ONFAULT] on the old vma */
 594                 vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
 595
 596                 /*
 597                  * anon_vma links of the old vma is no longer needed after its page
 598                  * table has been moved.
 599                  */
 600                 if (new_vma != vma && vma->vm_start == old_addr &&
 601                         vma->vm_end == (old_addr + old_len))
 602                         unlink_anon_vmas(vma);
 603
 604                 /* Because we won't unmap we don't need to touch locked_vm */
 605                 return new_addr;
 606         }
 607
 608         if (do_munmap(mm, old_addr, old_len, uf_unmap) < 0) {
 609                 /* OOM: unable to split vma, just get accounts right */
 610                 if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP))
 611                         vm_acct_memory(new_len >> PAGE_SHIFT);
 612                 excess = 0;
 613         }
 614
 615         if (vm_flags & VM_LOCKED) {
 616                 mm->locked_vm += new_len >> PAGE_SHIFT;
 617                 *locked = true;
 618         }
 619
 620         mm->hiwater_vm = hiwater_vm;
 621
 622         /* Restore VM_ACCOUNT if one or two pieces of vma left */
 623         if (excess) {
 624                 vma->vm_flags |= VM_ACCOUNT;
 625                 if (split)
 626                         vma->vm_next->vm_flags |= VM_ACCOUNT;
 627         }
 628
 629         return new_addr;
 630 }
 631
 632 static struct vm_area_struct *vma_to_resize(unsigned long addr,
 633         unsigned long old_len, unsigned long new_len, unsigned long flags,
 634         unsigned long *p)
 635 {
 636         struct mm_struct *mm = current->mm;
 637         struct vm_area_struct *vma = find_vma(mm, addr);
 638         unsigned long pgoff;
 639
 640         if (!vma || vma->vm_start > addr)
 641                 return ERR_PTR(-EFAULT);
 642
 643         /*
 644          * !old_len is a special case where an attempt is made to 'duplicate'
 645          * a mapping.  This makes no sense for private mappings as it will
 646          * instead create a fresh/new mapping unrelated to the original.  This
 647          * is contrary to the basic idea of mremap which creates new mappings
 648          * based on the original.  There are no known use cases for this
 649          * behavior.  As a result, fail such attempts.
 650          */
 651         if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) {
 652                 pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap.  This is not supported.\n", current->comm, current->pid);
 653                 return ERR_PTR(-EINVAL);
 654         }
 655
 656         if ((flags & MREMAP_DONTUNMAP) &&
 657                         (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))
 658                 return ERR_PTR(-EINVAL);
 659
 660         if (is_vm_hugetlb_page(vma))
 661                 return ERR_PTR(-EINVAL);
 662
 663         /* We can't remap across vm area boundaries */
 664         if (old_len > vma->vm_end - addr)
 665                 return ERR_PTR(-EFAULT);
 666
 667         if (new_len == old_len)
 668                 return vma;
 669
 670         /* Need to be careful about a growing mapping */
 671         pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
 672         pgoff += vma->vm_pgoff;
 673         if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
 674                 return ERR_PTR(-EINVAL);
 675
 676         if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
 677                 return ERR_PTR(-EFAULT);
 678
 679         if (vma->vm_flags & VM_LOCKED) {
 680                 unsigned long locked, lock_limit;
 681                 locked = mm->locked_vm << PAGE_SHIFT;
 682                 lock_limit = rlimit(RLIMIT_MEMLOCK);
 683                 locked += new_len - old_len;
 684                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
 685                         return ERR_PTR(-EAGAIN);
 686         }
 687
 688         if (!may_expand_vm(mm, vma->vm_flags,
 689                                 (new_len - old_len) >> PAGE_SHIFT))
 690                 return ERR_PTR(-ENOMEM);
 691
 692         if (vma->vm_flags & VM_ACCOUNT) {
 693                 unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
 694                 if (security_vm_enough_memory_mm(mm, charged))
 695                         return ERR_PTR(-ENOMEM);
 696                 *p = charged;
 697         }
 698
 699         return vma;
 700 }
 701
 702 static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 703                 unsigned long new_addr, unsigned long new_len, bool *locked,
 704                 unsigned long flags, struct vm_userfaultfd_ctx *uf,
 705                 struct list_head *uf_unmap_early,
 706                 struct list_head *uf_unmap)
 707 {
 708         struct mm_struct *mm = current->mm;
 709         struct vm_area_struct *vma;
 710         unsigned long ret = -EINVAL;
 711         unsigned long charged = 0;
 712         unsigned long map_flags = 0;
 713
 714         if (offset_in_page(new_addr))
 715                 goto out;
 716
 717         if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
 718                 goto out;
 719
 720         /* Ensure the old/new locations do not overlap */
 721         if (addr + old_len > new_addr && new_addr + new_len > addr)
 722                 goto out;
 723
 724         /*
 725          * move_vma() need us to stay 4 maps below the threshold, otherwise
 726          * it will bail out at the very beginning.
 727          * That is a problem if we have already unmaped the regions here
 728          * (new_addr, and old_addr), because userspace will not know the
 729          * state of the vma's after it gets -ENOMEM.
 730          * So, to avoid such scenario we can pre-compute if the whole
 731          * operation has high chances to success map-wise.
 732          * Worst-scenario case is when both vma's (new_addr and old_addr) get
 733          * split in 3 before unmapping it.
 734          * That means 2 more maps (1 for each) to the ones we already hold.
 735          * Check whether current map count plus 2 still leads us to 4 maps below
 736          * the threshold, otherwise return -ENOMEM here to be more safe.
 737          */
 738         if ((mm->map_count + 2) >= sysctl_max_map_count - 3)
 739                 return -ENOMEM;
 740
 741         if (flags & MREMAP_FIXED) {
 742                 ret = do_munmap(mm, new_addr, new_len, uf_unmap_early);
 743                 if (ret)
 744                         goto out;
 745         }
 746
 747         if (old_len >= new_len) {
 748                 ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap);
 749                 if (ret && old_len != new_len)
 750                         goto out;
 751                 old_len = new_len;
 752         }
 753
 754         vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
 755         if (IS_ERR(vma)) {
 756                 ret = PTR_ERR(vma);
 757                 goto out;
 758         }
 759
 760         /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */
 761         if (flags & MREMAP_DONTUNMAP &&
 762                 !may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) {
 763                 ret = -ENOMEM;
 764                 goto out;
 765         }
 766
 767         if (flags & MREMAP_FIXED)
 768                 map_flags |= MAP_FIXED;
 769
 770         if (vma->vm_flags & VM_MAYSHARE)
 771                 map_flags |= MAP_SHARED;
 772
 773         ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
 774                                 ((addr - vma->vm_start) >> PAGE_SHIFT),
 775                                 map_flags);
 776         if (IS_ERR_VALUE(ret))
 777                 goto out1;
 778
 779         /* We got a new mapping */
 780         if (!(flags & MREMAP_FIXED))
 781                 new_addr = ret;
 782
 783         ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
 784                        uf_unmap);
 785
 786         if (!(offset_in_page(ret)))
 787                 goto out;
 788
 789 out1:
 790         vm_unacct_memory(charged);
 791
 792 out:
 793         return ret;
 794 }
 795
 796 static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
 797 {
 798         unsigned long end = vma->vm_end + delta;
 799         if (end < vma->vm_end) /* overflow */
 800                 return 0;
 801         if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
 802                 return 0;
 803         if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
 804                               0, MAP_FIXED) & ~PAGE_MASK)
 805                 return 0;
 806         return 1;
 807 }
 808
 809 /*
 810  * Expand (or shrink) an existing mapping, potentially moving it at the
 811  * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 812  *
 813  * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 814  * This option implies MREMAP_MAYMOVE.
 815  */
 816 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 817                 unsigned long, new_len, unsigned long, flags,
 818                 unsigned long, new_addr)
 819 {
 820         struct mm_struct *mm = current->mm;
 821         struct vm_area_struct *vma;
 822         unsigned long ret = -EINVAL;
 823         unsigned long charged = 0;
 824         bool locked = false;
 825         bool downgraded = false;
 826         struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
 827         LIST_HEAD(uf_unmap_early);
 828         LIST_HEAD(uf_unmap);
 829
 830         /*
 831          * There is a deliberate asymmetry here: we strip the pointer tag
 832          * from the old address but leave the new address alone. This is
 833          * for consistency with mmap(), where we prevent the creation of
 834          * aliasing mappings in userspace by leaving the tag bits of the
 835          * mapping address intact. A non-zero tag will cause the subsequent
 836          * range checks to reject the address as invalid.
 837          *
 838          * See Documentation/arm64/tagged-address-abi.rst for more information.
 839          */
 840         addr = untagged_addr(addr);
 841
 842         if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP))
 843                 return ret;
 844
 845         if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE))
 846                 return ret;
 847
 848         /*
 849          * MREMAP_DONTUNMAP is always a move and it does not allow resizing
 850          * in the process.
 851          */
 852         if (flags & MREMAP_DONTUNMAP &&
 853                         (!(flags & MREMAP_MAYMOVE) || old_len != new_len))
 854                 return ret;
 855
 856
 857         if (offset_in_page(addr))
 858                 return ret;
 859
 860         old_len = PAGE_ALIGN(old_len);
 861         new_len = PAGE_ALIGN(new_len);
 862
 863         /*
 864          * We allow a zero old-len as a special case
 865          * for DOS-emu "duplicate shm area" thing. But
 866          * a zero new-len is nonsensical.
 867          */
 868         if (!new_len)
 869                 return ret;
 870
 871         if (mmap_write_lock_killable(current->mm))
 872                 return -EINTR;
 873
 874         if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) {
 875                 ret = mremap_to(addr, old_len, new_addr, new_len,
 876                                 &locked, flags, &uf, &uf_unmap_early,
 877                                 &uf_unmap);
 878                 goto out;
 879         }
 880
 881         /*
 882          * Always allow a shrinking remap: that just unmaps
 883          * the unnecessary pages..
 884          * __do_munmap does all the needed commit accounting, and
 885          * downgrades mmap_lock to read if so directed.
 886          */
 887         if (old_len >= new_len) {
 888                 int retval;
 889
 890                 retval = __do_munmap(mm, addr+new_len, old_len - new_len,
 891                                   &uf_unmap, true);
 892                 if (retval < 0 && old_len != new_len) {
 893                         ret = retval;
 894                         goto out;
 895                 /* Returning 1 indicates mmap_lock is downgraded to read. */
 896                 } else if (retval == 1)
 897                         downgraded = true;
 898                 ret = addr;
 899                 goto out;
 900         }
 901
 902         /*
 903          * Ok, we need to grow..
 904          */
 905         vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
 906         if (IS_ERR(vma)) {
 907                 ret = PTR_ERR(vma);
 908                 goto out;
 909         }
 910
 911         /* old_len exactly to the end of the area..
 912          */
 913         if (old_len == vma->vm_end - addr) {
 914                 /* can we just expand the current mapping? */
 915                 if (vma_expandable(vma, new_len - old_len)) {
 916                         int pages = (new_len - old_len) >> PAGE_SHIFT;
 917
 918                         if (vma_adjust(vma, vma->vm_start, addr + new_len,
 919                                        vma->vm_pgoff, NULL)) {
 920                                 ret = -ENOMEM;
 921                                 goto out;
 922                         }
 923
 924                         vm_stat_account(mm, vma->vm_flags, pages);
 925                         if (vma->vm_flags & VM_LOCKED) {
 926                                 mm->locked_vm += pages;
 927                                 locked = true;
 928                                 new_addr = addr;
 929                         }
 930                         ret = addr;
 931                         goto out;
 932                 }
 933         }
 934
 935         /*
 936          * We weren't able to just expand or shrink the area,
 937          * we need to create a new one and move it..
 938          */
 939         ret = -ENOMEM;
 940         if (flags & MREMAP_MAYMOVE) {
 941                 unsigned long map_flags = 0;
 942                 if (vma->vm_flags & VM_MAYSHARE)
 943                         map_flags |= MAP_SHARED;
 944
 945                 new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
 946                                         vma->vm_pgoff +
 947                                         ((addr - vma->vm_start) >> PAGE_SHIFT),
 948                                         map_flags);
 949                 if (IS_ERR_VALUE(new_addr)) {
 950                         ret = new_addr;
 951                         goto out;
 952                 }
 953
 954                 ret = move_vma(vma, addr, old_len, new_len, new_addr,
 955                                &locked, flags, &uf, &uf_unmap);
 956         }
 957 out:
 958         if (offset_in_page(ret)) {
 959                 vm_unacct_memory(charged);
 960                 locked = false;
 961         }
 962         if (downgraded)
 963                 mmap_read_unlock(current->mm);
 964         else
 965                 mmap_write_unlock(current->mm);
 966         if (locked && new_len > old_len)
 967                 mm_populate(new_addr + old_len, new_len - old_len);
 968         userfaultfd_unmap_complete(mm, &uf_unmap_early);
 969         mremap_userfaultfd_complete(&uf, addr, ret, old_len);
 970         userfaultfd_unmap_complete(mm, &uf_unmap);
 971         return ret;
 972 }