fs/userfaultfd.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  fs/userfaultfd.c
   4  *
   5  *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
   6  *  Copyright (C) 2008-2009 Red Hat, Inc.
   7  *  Copyright (C) 2015  Red Hat, Inc.
   8  *
   9  *  Some part derived from fs/eventfd.c (anon inode setup) and
  10  *  mm/ksm.c (mm hashing).
  11  */
  12
  13 #include <linux/list.h>
  14 #include <linux/hashtable.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/sched/mm.h>
  17 #include <linux/mm.h>
  18 #include <linux/mm_inline.h>
  19 #include <linux/mmu_notifier.h>
  20 #include <linux/poll.h>
  21 #include <linux/slab.h>
  22 #include <linux/seq_file.h>
  23 #include <linux/file.h>
  24 #include <linux/bug.h>
  25 #include <linux/anon_inodes.h>
  26 #include <linux/syscalls.h>
  27 #include <linux/userfaultfd_k.h>
  28 #include <linux/mempolicy.h>
  29 #include <linux/ioctl.h>
  30 #include <linux/security.h>
  31 #include <linux/hugetlb.h>
  32 #include <linux/swapops.h>
  33 #include <linux/miscdevice.h>
  34
  35 int sysctl_unprivileged_userfaultfd __read_mostly;
  36
  37 static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
  38
  39 /*
  40  * Start with fault_pending_wqh and fault_wqh so they're more likely
  41  * to be in the same cacheline.
  42  *
  43  * Locking order:
  44  *      fd_wqh.lock
  45  *              fault_pending_wqh.lock
  46  *                      fault_wqh.lock
  47  *              event_wqh.lock
  48  *
  49  * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
  50  * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
  51  * also taken in IRQ context.
  52  */
  53 struct userfaultfd_ctx {
  54         /* waitqueue head for the pending (i.e. not read) userfaults */
  55         wait_queue_head_t fault_pending_wqh;
  56         /* waitqueue head for the userfaults */
  57         wait_queue_head_t fault_wqh;
  58         /* waitqueue head for the pseudo fd to wakeup poll/read */
  59         wait_queue_head_t fd_wqh;
  60         /* waitqueue head for events */
  61         wait_queue_head_t event_wqh;
  62         /* a refile sequence protected by fault_pending_wqh lock */
  63         seqcount_spinlock_t refile_seq;
  64         /* pseudo fd refcounting */
  65         refcount_t refcount;
  66         /* userfaultfd syscall flags */
  67         unsigned int flags;
  68         /* features requested from the userspace */
  69         unsigned int features;
  70         /* released */
  71         bool released;
  72         /* memory mappings are changing because of non-cooperative event */
  73         atomic_t mmap_changing;
  74         /* mm with one ore more vmas attached to this userfaultfd_ctx */
  75         struct mm_struct *mm;
  76 };
  77
  78 struct userfaultfd_fork_ctx {
  79         struct userfaultfd_ctx *orig;
  80         struct userfaultfd_ctx *new;
  81         struct list_head list;
  82 };
  83
  84 struct userfaultfd_unmap_ctx {
  85         struct userfaultfd_ctx *ctx;
  86         unsigned long start;
  87         unsigned long end;
  88         struct list_head list;
  89 };
  90
  91 struct userfaultfd_wait_queue {
  92         struct uffd_msg msg;
  93         wait_queue_entry_t wq;
  94         struct userfaultfd_ctx *ctx;
  95         bool waken;
  96 };
  97
  98 struct userfaultfd_wake_range {
  99         unsigned long start;
 100         unsigned long len;
 101 };
 102
 103 /* internal indication that UFFD_API ioctl was successfully executed */
 104 #define UFFD_FEATURE_INITIALIZED                (1u << 31)
 105
 106 static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
 107 {
 108         return ctx->features & UFFD_FEATURE_INITIALIZED;
 109 }
 110
 111 static void userfaultfd_set_vm_flags(struct vm_area_struct *vma,
 112                                      vm_flags_t flags)
 113 {
 114         const bool uffd_wp_changed = (vma->vm_flags ^ flags) & VM_UFFD_WP;
 115
 116         vm_flags_reset(vma, flags);
 117         /*
 118          * For shared mappings, we want to enable writenotify while
 119          * userfaultfd-wp is enabled (see vma_wants_writenotify()). We'll simply
 120          * recalculate vma->vm_page_prot whenever userfaultfd-wp changes.
 121          */
 122         if ((vma->vm_flags & VM_SHARED) && uffd_wp_changed)
 123                 vma_set_page_prot(vma);
 124 }
 125
 126 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
 127                                      int wake_flags, void *key)
 128 {
 129         struct userfaultfd_wake_range *range = key;
 130         int ret;
 131         struct userfaultfd_wait_queue *uwq;
 132         unsigned long start, len;
 133
 134         uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
 135         ret = 0;
 136         /* len == 0 means wake all */
 137         start = range->start;
 138         len = range->len;
 139         if (len && (start > uwq->msg.arg.pagefault.address ||
 140                     start + len <= uwq->msg.arg.pagefault.address))
 141                 goto out;
 142         WRITE_ONCE(uwq->waken, true);
 143         /*
 144          * The Program-Order guarantees provided by the scheduler
 145          * ensure uwq->waken is visible before the task is woken.
 146          */
 147         ret = wake_up_state(wq->private, mode);
 148         if (ret) {
 149                 /*
 150                  * Wake only once, autoremove behavior.
 151                  *
 152                  * After the effect of list_del_init is visible to the other
 153                  * CPUs, the waitqueue may disappear from under us, see the
 154                  * !list_empty_careful() in handle_userfault().
 155                  *
 156                  * try_to_wake_up() has an implicit smp_mb(), and the
 157                  * wq->private is read before calling the extern function
 158                  * "wake_up_state" (which in turns calls try_to_wake_up).
 159                  */
 160                 list_del_init(&wq->entry);
 161         }
 162 out:
 163         return ret;
 164 }
 165
 166 /**
 167  * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
 168  * context.
 169  * @ctx: [in] Pointer to the userfaultfd context.
 170  */
 171 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
 172 {
 173         refcount_inc(&ctx->refcount);
 174 }
 175
 176 /**
 177  * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
 178  * context.
 179  * @ctx: [in] Pointer to userfaultfd context.
 180  *
 181  * The userfaultfd context reference must have been previously acquired either
 182  * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
 183  */
 184 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
 185 {
 186         if (refcount_dec_and_test(&ctx->refcount)) {
 187                 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
 188                 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
 189                 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
 190                 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
 191                 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
 192                 VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
 193                 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
 194                 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
 195                 mmdrop(ctx->mm);
 196                 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
 197         }
 198 }
 199
 200 static inline void msg_init(struct uffd_msg *msg)
 201 {
 202         BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
 203         /*
 204          * Must use memset to zero out the paddings or kernel data is
 205          * leaked to userland.
 206          */
 207         memset(msg, 0, sizeof(struct uffd_msg));
 208 }
 209
 210 static inline struct uffd_msg userfault_msg(unsigned long address,
 211                                             unsigned long real_address,
 212                                             unsigned int flags,
 213                                             unsigned long reason,
 214                                             unsigned int features)
 215 {
 216         struct uffd_msg msg;
 217
 218         msg_init(&msg);
 219         msg.event = UFFD_EVENT_PAGEFAULT;
 220
 221         msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ?
 222                                     real_address : address;
 223
 224         /*
 225          * These flags indicate why the userfault occurred:
 226          * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
 227          * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
 228          * - Neither of these flags being set indicates a MISSING fault.
 229          *
 230          * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
 231          * fault. Otherwise, it was a read fault.
 232          */
 233         if (flags & FAULT_FLAG_WRITE)
 234                 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
 235         if (reason & VM_UFFD_WP)
 236                 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
 237         if (reason & VM_UFFD_MINOR)
 238                 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
 239         if (features & UFFD_FEATURE_THREAD_ID)
 240                 msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
 241         return msg;
 242 }
 243
 244 #ifdef CONFIG_HUGETLB_PAGE
 245 /*
 246  * Same functionality as userfaultfd_must_wait below with modifications for
 247  * hugepmd ranges.
 248  */
 249 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
 250                                          struct vm_area_struct *vma,
 251                                          unsigned long address,
 252                                          unsigned long flags,
 253                                          unsigned long reason)
 254 {
 255         pte_t *ptep, pte;
 256         bool ret = true;
 257
 258         mmap_assert_locked(ctx->mm);
 259
 260         ptep = hugetlb_walk(vma, address, vma_mmu_pagesize(vma));
 261         if (!ptep)
 262                 goto out;
 263
 264         ret = false;
 265         pte = huge_ptep_get(ptep);
 266
 267         /*
 268          * Lockless access: we're in a wait_event so it's ok if it
 269          * changes under us.  PTE markers should be handled the same as none
 270          * ptes here.
 271          */
 272         if (huge_pte_none_mostly(pte))
 273                 ret = true;
 274         if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
 275                 ret = true;
 276 out:
 277         return ret;
 278 }
 279 #else
 280 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
 281                                          struct vm_area_struct *vma,
 282                                          unsigned long address,
 283                                          unsigned long flags,
 284                                          unsigned long reason)
 285 {
 286         return false;   /* should never get here */
 287 }
 288 #endif /* CONFIG_HUGETLB_PAGE */
 289
 290 /*
 291  * Verify the pagetables are still not ok after having reigstered into
 292  * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
 293  * userfault that has already been resolved, if userfaultfd_read and
 294  * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
 295  * threads.
 296  */
 297 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
 298                                          unsigned long address,
 299                                          unsigned long flags,
 300                                          unsigned long reason)
 301 {
 302         struct mm_struct *mm = ctx->mm;
 303         pgd_t *pgd;
 304         p4d_t *p4d;
 305         pud_t *pud;
 306         pmd_t *pmd, _pmd;
 307         pte_t *pte;
 308         bool ret = true;
 309
 310         mmap_assert_locked(mm);
 311
 312         pgd = pgd_offset(mm, address);
 313         if (!pgd_present(*pgd))
 314                 goto out;
 315         p4d = p4d_offset(pgd, address);
 316         if (!p4d_present(*p4d))
 317                 goto out;
 318         pud = pud_offset(p4d, address);
 319         if (!pud_present(*pud))
 320                 goto out;
 321         pmd = pmd_offset(pud, address);
 322         /*
 323          * READ_ONCE must function as a barrier with narrower scope
 324          * and it must be equivalent to:
 325          *      _pmd = *pmd; barrier();
 326          *
 327          * This is to deal with the instability (as in
 328          * pmd_trans_unstable) of the pmd.
 329          */
 330         _pmd = READ_ONCE(*pmd);
 331         if (pmd_none(_pmd))
 332                 goto out;
 333
 334         ret = false;
 335         if (!pmd_present(_pmd))
 336                 goto out;
 337
 338         if (pmd_trans_huge(_pmd)) {
 339                 if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
 340                         ret = true;
 341                 goto out;
 342         }
 343
 344         /*
 345          * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
 346          * and use the standard pte_offset_map() instead of parsing _pmd.
 347          */
 348         pte = pte_offset_map(pmd, address);
 349         /*
 350          * Lockless access: we're in a wait_event so it's ok if it
 351          * changes under us.  PTE markers should be handled the same as none
 352          * ptes here.
 353          */
 354         if (pte_none_mostly(*pte))
 355                 ret = true;
 356         if (!pte_write(*pte) && (reason & VM_UFFD_WP))
 357                 ret = true;
 358         pte_unmap(pte);
 359
 360 out:
 361         return ret;
 362 }
 363
 364 static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)
 365 {
 366         if (flags & FAULT_FLAG_INTERRUPTIBLE)
 367                 return TASK_INTERRUPTIBLE;
 368
 369         if (flags & FAULT_FLAG_KILLABLE)
 370                 return TASK_KILLABLE;
 371
 372         return TASK_UNINTERRUPTIBLE;
 373 }
 374
 375 /*
 376  * The locking rules involved in returning VM_FAULT_RETRY depending on
 377  * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
 378  * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
 379  * recommendation in __lock_page_or_retry is not an understatement.
 380  *
 381  * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
 382  * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
 383  * not set.
 384  *
 385  * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
 386  * set, VM_FAULT_RETRY can still be returned if and only if there are
 387  * fatal_signal_pending()s, and the mmap_lock must be released before
 388  * returning it.
 389  */
 390 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
 391 {
 392         struct vm_area_struct *vma = vmf->vma;
 393         struct mm_struct *mm = vma->vm_mm;
 394         struct userfaultfd_ctx *ctx;
 395         struct userfaultfd_wait_queue uwq;
 396         vm_fault_t ret = VM_FAULT_SIGBUS;
 397         bool must_wait;
 398         unsigned int blocking_state;
 399
 400         /*
 401          * We don't do userfault handling for the final child pid update.
 402          *
 403          * We also don't do userfault handling during
 404          * coredumping. hugetlbfs has the special
 405          * follow_hugetlb_page() to skip missing pages in the
 406          * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
 407          * the no_page_table() helper in follow_page_mask(), but the
 408          * shmem_vm_ops->fault method is invoked even during
 409          * coredumping without mmap_lock and it ends up here.
 410          */
 411         if (current->flags & (PF_EXITING|PF_DUMPCORE))
 412                 goto out;
 413
 414         /*
 415          * Coredumping runs without mmap_lock so we can only check that
 416          * the mmap_lock is held, if PF_DUMPCORE was not set.
 417          */
 418         mmap_assert_locked(mm);
 419
 420         ctx = vma->vm_userfaultfd_ctx.ctx;
 421         if (!ctx)
 422                 goto out;
 423
 424         BUG_ON(ctx->mm != mm);
 425
 426         /* Any unrecognized flag is a bug. */
 427         VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);
 428         /* 0 or > 1 flags set is a bug; we expect exactly 1. */
 429         VM_BUG_ON(!reason || (reason & (reason - 1)));
 430
 431         if (ctx->features & UFFD_FEATURE_SIGBUS)
 432                 goto out;
 433         if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))
 434                 goto out;
 435
 436         /*
 437          * If it's already released don't get it. This avoids to loop
 438          * in __get_user_pages if userfaultfd_release waits on the
 439          * caller of handle_userfault to release the mmap_lock.
 440          */
 441         if (unlikely(READ_ONCE(ctx->released))) {
 442                 /*
 443                  * Don't return VM_FAULT_SIGBUS in this case, so a non
 444                  * cooperative manager can close the uffd after the
 445                  * last UFFDIO_COPY, without risking to trigger an
 446                  * involuntary SIGBUS if the process was starting the
 447                  * userfaultfd while the userfaultfd was still armed
 448                  * (but after the last UFFDIO_COPY). If the uffd
 449                  * wasn't already closed when the userfault reached
 450                  * this point, that would normally be solved by
 451                  * userfaultfd_must_wait returning 'false'.
 452                  *
 453                  * If we were to return VM_FAULT_SIGBUS here, the non
 454                  * cooperative manager would be instead forced to
 455                  * always call UFFDIO_UNREGISTER before it can safely
 456                  * close the uffd.
 457                  */
 458                 ret = VM_FAULT_NOPAGE;
 459                 goto out;
 460         }
 461
 462         /*
 463          * Check that we can return VM_FAULT_RETRY.
 464          *
 465          * NOTE: it should become possible to return VM_FAULT_RETRY
 466          * even if FAULT_FLAG_TRIED is set without leading to gup()
 467          * -EBUSY failures, if the userfaultfd is to be extended for
 468          * VM_UFFD_WP tracking and we intend to arm the userfault
 469          * without first stopping userland access to the memory. For
 470          * VM_UFFD_MISSING userfaults this is enough for now.
 471          */
 472         if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
 473                 /*
 474                  * Validate the invariant that nowait must allow retry
 475                  * to be sure not to return SIGBUS erroneously on
 476                  * nowait invocations.
 477                  */
 478                 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
 479 #ifdef CONFIG_DEBUG_VM
 480                 if (printk_ratelimit()) {
 481                         printk(KERN_WARNING
 482                                "FAULT_FLAG_ALLOW_RETRY missing %x\n",
 483                                vmf->flags);
 484                         dump_stack();
 485                 }
 486 #endif
 487                 goto out;
 488         }
 489
 490         /*
 491          * Handle nowait, not much to do other than tell it to retry
 492          * and wait.
 493          */
 494         ret = VM_FAULT_RETRY;
 495         if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
 496                 goto out;
 497
 498         /* take the reference before dropping the mmap_lock */
 499         userfaultfd_ctx_get(ctx);
 500
 501         init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
 502         uwq.wq.private = current;
 503         uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags,
 504                                 reason, ctx->features);
 505         uwq.ctx = ctx;
 506         uwq.waken = false;
 507
 508         blocking_state = userfaultfd_get_blocking_state(vmf->flags);
 509
 510         /*
 511          * Take the vma lock now, in order to safely call
 512          * userfaultfd_huge_must_wait() later. Since acquiring the
 513          * (sleepable) vma lock can modify the current task state, that
 514          * must be before explicitly calling set_current_state().
 515          */
 516         if (is_vm_hugetlb_page(vma))
 517                 hugetlb_vma_lock_read(vma);
 518
 519         spin_lock_irq(&ctx->fault_pending_wqh.lock);
 520         /*
 521          * After the __add_wait_queue the uwq is visible to userland
 522          * through poll/read().
 523          */
 524         __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
 525         /*
 526          * The smp_mb() after __set_current_state prevents the reads
 527          * following the spin_unlock to happen before the list_add in
 528          * __add_wait_queue.
 529          */
 530         set_current_state(blocking_state);
 531         spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 532
 533         if (!is_vm_hugetlb_page(vma))
 534                 must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
 535                                                   reason);
 536         else
 537                 must_wait = userfaultfd_huge_must_wait(ctx, vma,
 538                                                        vmf->address,
 539                                                        vmf->flags, reason);
 540         if (is_vm_hugetlb_page(vma))
 541                 hugetlb_vma_unlock_read(vma);
 542         mmap_read_unlock(mm);
 543
 544         if (likely(must_wait && !READ_ONCE(ctx->released))) {
 545                 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
 546                 schedule();
 547         }
 548
 549         __set_current_state(TASK_RUNNING);
 550
 551         /*
 552          * Here we race with the list_del; list_add in
 553          * userfaultfd_ctx_read(), however because we don't ever run
 554          * list_del_init() to refile across the two lists, the prev
 555          * and next pointers will never point to self. list_add also
 556          * would never let any of the two pointers to point to
 557          * self. So list_empty_careful won't risk to see both pointers
 558          * pointing to self at any time during the list refile. The
 559          * only case where list_del_init() is called is the full
 560          * removal in the wake function and there we don't re-list_add
 561          * and it's fine not to block on the spinlock. The uwq on this
 562          * kernel stack can be released after the list_del_init.
 563          */
 564         if (!list_empty_careful(&uwq.wq.entry)) {
 565                 spin_lock_irq(&ctx->fault_pending_wqh.lock);
 566                 /*
 567                  * No need of list_del_init(), the uwq on the stack
 568                  * will be freed shortly anyway.
 569                  */
 570                 list_del(&uwq.wq.entry);
 571                 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 572         }
 573
 574         /*
 575          * ctx may go away after this if the userfault pseudo fd is
 576          * already released.
 577          */
 578         userfaultfd_ctx_put(ctx);
 579
 580 out:
 581         return ret;
 582 }
 583
 584 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
 585                                               struct userfaultfd_wait_queue *ewq)
 586 {
 587         struct userfaultfd_ctx *release_new_ctx;
 588
 589         if (WARN_ON_ONCE(current->flags & PF_EXITING))
 590                 goto out;
 591
 592         ewq->ctx = ctx;
 593         init_waitqueue_entry(&ewq->wq, current);
 594         release_new_ctx = NULL;
 595
 596         spin_lock_irq(&ctx->event_wqh.lock);
 597         /*
 598          * After the __add_wait_queue the uwq is visible to userland
 599          * through poll/read().
 600          */
 601         __add_wait_queue(&ctx->event_wqh, &ewq->wq);
 602         for (;;) {
 603                 set_current_state(TASK_KILLABLE);
 604                 if (ewq->msg.event == 0)
 605                         break;
 606                 if (READ_ONCE(ctx->released) ||
 607                     fatal_signal_pending(current)) {
 608                         /*
 609                          * &ewq->wq may be queued in fork_event, but
 610                          * __remove_wait_queue ignores the head
 611                          * parameter. It would be a problem if it
 612                          * didn't.
 613                          */
 614                         __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
 615                         if (ewq->msg.event == UFFD_EVENT_FORK) {
 616                                 struct userfaultfd_ctx *new;
 617
 618                                 new = (struct userfaultfd_ctx *)
 619                                         (unsigned long)
 620                                         ewq->msg.arg.reserved.reserved1;
 621                                 release_new_ctx = new;
 622                         }
 623                         break;
 624                 }
 625
 626                 spin_unlock_irq(&ctx->event_wqh.lock);
 627
 628                 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
 629                 schedule();
 630
 631                 spin_lock_irq(&ctx->event_wqh.lock);
 632         }
 633         __set_current_state(TASK_RUNNING);
 634         spin_unlock_irq(&ctx->event_wqh.lock);
 635
 636         if (release_new_ctx) {
 637                 struct vm_area_struct *vma;
 638                 struct mm_struct *mm = release_new_ctx->mm;
 639                 VMA_ITERATOR(vmi, mm, 0);
 640
 641                 /* the various vma->vm_userfaultfd_ctx still points to it */
 642                 mmap_write_lock(mm);
 643                 for_each_vma(vmi, vma) {
 644                         if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
 645                                 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 646                                 userfaultfd_set_vm_flags(vma,
 647                                                          vma->vm_flags & ~__VM_UFFD_FLAGS);
 648                         }
 649                 }
 650                 mmap_write_unlock(mm);
 651
 652                 userfaultfd_ctx_put(release_new_ctx);
 653         }
 654
 655         /*
 656          * ctx may go away after this if the userfault pseudo fd is
 657          * already released.
 658          */
 659 out:
 660         atomic_dec(&ctx->mmap_changing);
 661         VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0);
 662         userfaultfd_ctx_put(ctx);
 663 }
 664
 665 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
 666                                        struct userfaultfd_wait_queue *ewq)
 667 {
 668         ewq->msg.event = 0;
 669         wake_up_locked(&ctx->event_wqh);
 670         __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
 671 }
 672
 673 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
 674 {
 675         struct userfaultfd_ctx *ctx = NULL, *octx;
 676         struct userfaultfd_fork_ctx *fctx;
 677
 678         octx = vma->vm_userfaultfd_ctx.ctx;
 679         if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
 680                 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 681                 userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS);
 682                 return 0;
 683         }
 684
 685         list_for_each_entry(fctx, fcs, list)
 686                 if (fctx->orig == octx) {
 687                         ctx = fctx->new;
 688                         break;
 689                 }
 690
 691         if (!ctx) {
 692                 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
 693                 if (!fctx)
 694                         return -ENOMEM;
 695
 696                 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
 697                 if (!ctx) {
 698                         kfree(fctx);
 699                         return -ENOMEM;
 700                 }
 701
 702                 refcount_set(&ctx->refcount, 1);
 703                 ctx->flags = octx->flags;
 704                 ctx->features = octx->features;
 705                 ctx->released = false;
 706                 atomic_set(&ctx->mmap_changing, 0);
 707                 ctx->mm = vma->vm_mm;
 708                 mmgrab(ctx->mm);
 709
 710                 userfaultfd_ctx_get(octx);
 711                 atomic_inc(&octx->mmap_changing);
 712                 fctx->orig = octx;
 713                 fctx->new = ctx;
 714                 list_add_tail(&fctx->list, fcs);
 715         }
 716
 717         vma->vm_userfaultfd_ctx.ctx = ctx;
 718         return 0;
 719 }
 720
 721 static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
 722 {
 723         struct userfaultfd_ctx *ctx = fctx->orig;
 724         struct userfaultfd_wait_queue ewq;
 725
 726         msg_init(&ewq.msg);
 727
 728         ewq.msg.event = UFFD_EVENT_FORK;
 729         ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
 730
 731         userfaultfd_event_wait_completion(ctx, &ewq);
 732 }
 733
 734 void dup_userfaultfd_complete(struct list_head *fcs)
 735 {
 736         struct userfaultfd_fork_ctx *fctx, *n;
 737
 738         list_for_each_entry_safe(fctx, n, fcs, list) {
 739                 dup_fctx(fctx);
 740                 list_del(&fctx->list);
 741                 kfree(fctx);
 742         }
 743 }
 744
 745 void mremap_userfaultfd_prep(struct vm_area_struct *vma,
 746                              struct vm_userfaultfd_ctx *vm_ctx)
 747 {
 748         struct userfaultfd_ctx *ctx;
 749
 750         ctx = vma->vm_userfaultfd_ctx.ctx;
 751
 752         if (!ctx)
 753                 return;
 754
 755         if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
 756                 vm_ctx->ctx = ctx;
 757                 userfaultfd_ctx_get(ctx);
 758                 atomic_inc(&ctx->mmap_changing);
 759         } else {
 760                 /* Drop uffd context if remap feature not enabled */
 761                 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 762                 userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS);
 763         }
 764 }
 765
 766 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
 767                                  unsigned long from, unsigned long to,
 768                                  unsigned long len)
 769 {
 770         struct userfaultfd_ctx *ctx = vm_ctx->ctx;
 771         struct userfaultfd_wait_queue ewq;
 772
 773         if (!ctx)
 774                 return;
 775
 776         if (to & ~PAGE_MASK) {
 777                 userfaultfd_ctx_put(ctx);
 778                 return;
 779         }
 780
 781         msg_init(&ewq.msg);
 782
 783         ewq.msg.event = UFFD_EVENT_REMAP;
 784         ewq.msg.arg.remap.from = from;
 785         ewq.msg.arg.remap.to = to;
 786         ewq.msg.arg.remap.len = len;
 787
 788         userfaultfd_event_wait_completion(ctx, &ewq);
 789 }
 790
 791 bool userfaultfd_remove(struct vm_area_struct *vma,
 792                         unsigned long start, unsigned long end)
 793 {
 794         struct mm_struct *mm = vma->vm_mm;
 795         struct userfaultfd_ctx *ctx;
 796         struct userfaultfd_wait_queue ewq;
 797
 798         ctx = vma->vm_userfaultfd_ctx.ctx;
 799         if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
 800                 return true;
 801
 802         userfaultfd_ctx_get(ctx);
 803         atomic_inc(&ctx->mmap_changing);
 804         mmap_read_unlock(mm);
 805
 806         msg_init(&ewq.msg);
 807
 808         ewq.msg.event = UFFD_EVENT_REMOVE;
 809         ewq.msg.arg.remove.start = start;
 810         ewq.msg.arg.remove.end = end;
 811
 812         userfaultfd_event_wait_completion(ctx, &ewq);
 813
 814         return false;
 815 }
 816
 817 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
 818                           unsigned long start, unsigned long end)
 819 {
 820         struct userfaultfd_unmap_ctx *unmap_ctx;
 821
 822         list_for_each_entry(unmap_ctx, unmaps, list)
 823                 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
 824                     unmap_ctx->end == end)
 825                         return true;
 826
 827         return false;
 828 }
 829
 830 int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
 831                            unsigned long end, struct list_head *unmaps)
 832 {
 833         VMA_ITERATOR(vmi, mm, start);
 834         struct vm_area_struct *vma;
 835
 836         for_each_vma_range(vmi, vma, end) {
 837                 struct userfaultfd_unmap_ctx *unmap_ctx;
 838                 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
 839
 840                 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
 841                     has_unmap_ctx(ctx, unmaps, start, end))
 842                         continue;
 843
 844                 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
 845                 if (!unmap_ctx)
 846                         return -ENOMEM;
 847
 848                 userfaultfd_ctx_get(ctx);
 849                 atomic_inc(&ctx->mmap_changing);
 850                 unmap_ctx->ctx = ctx;
 851                 unmap_ctx->start = start;
 852                 unmap_ctx->end = end;
 853                 list_add_tail(&unmap_ctx->list, unmaps);
 854         }
 855
 856         return 0;
 857 }
 858
 859 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
 860 {
 861         struct userfaultfd_unmap_ctx *ctx, *n;
 862         struct userfaultfd_wait_queue ewq;
 863
 864         list_for_each_entry_safe(ctx, n, uf, list) {
 865                 msg_init(&ewq.msg);
 866
 867                 ewq.msg.event = UFFD_EVENT_UNMAP;
 868                 ewq.msg.arg.remove.start = ctx->start;
 869                 ewq.msg.arg.remove.end = ctx->end;
 870
 871                 userfaultfd_event_wait_completion(ctx->ctx, &ewq);
 872
 873                 list_del(&ctx->list);
 874                 kfree(ctx);
 875         }
 876 }
 877
 878 static int userfaultfd_release(struct inode *inode, struct file *file)
 879 {
 880         struct userfaultfd_ctx *ctx = file->private_data;
 881         struct mm_struct *mm = ctx->mm;
 882         struct vm_area_struct *vma, *prev;
 883         /* len == 0 means wake all */
 884         struct userfaultfd_wake_range range = { .len = 0, };
 885         unsigned long new_flags;
 886         VMA_ITERATOR(vmi, mm, 0);
 887
 888         WRITE_ONCE(ctx->released, true);
 889
 890         if (!mmget_not_zero(mm))
 891                 goto wakeup;
 892
 893         /*
 894          * Flush page faults out of all CPUs. NOTE: all page faults
 895          * must be retried without returning VM_FAULT_SIGBUS if
 896          * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
 897          * changes while handle_userfault released the mmap_lock. So
 898          * it's critical that released is set to true (above), before
 899          * taking the mmap_lock for writing.
 900          */
 901         mmap_write_lock(mm);
 902         prev = NULL;
 903         for_each_vma(vmi, vma) {
 904                 cond_resched();
 905                 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
 906                        !!(vma->vm_flags & __VM_UFFD_FLAGS));
 907                 if (vma->vm_userfaultfd_ctx.ctx != ctx) {
 908                         prev = vma;
 909                         continue;
 910                 }
 911                 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
 912                 prev = vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
 913                                  new_flags, vma->anon_vma,
 914                                  vma->vm_file, vma->vm_pgoff,
 915                                  vma_policy(vma),
 916                                  NULL_VM_UFFD_CTX, anon_vma_name(vma));
 917                 if (prev) {
 918                         vma = prev;
 919                 } else {
 920                         prev = vma;
 921                 }
 922
 923                 userfaultfd_set_vm_flags(vma, new_flags);
 924                 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 925         }
 926         mmap_write_unlock(mm);
 927         mmput(mm);
 928 wakeup:
 929         /*
 930          * After no new page faults can wait on this fault_*wqh, flush
 931          * the last page faults that may have been already waiting on
 932          * the fault_*wqh.
 933          */
 934         spin_lock_irq(&ctx->fault_pending_wqh.lock);
 935         __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
 936         __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
 937         spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 938
 939         /* Flush pending events that may still wait on event_wqh */
 940         wake_up_all(&ctx->event_wqh);
 941
 942         wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
 943         userfaultfd_ctx_put(ctx);
 944         return 0;
 945 }
 946
 947 /* fault_pending_wqh.lock must be hold by the caller */
 948 static inline struct userfaultfd_wait_queue *find_userfault_in(
 949                 wait_queue_head_t *wqh)
 950 {
 951         wait_queue_entry_t *wq;
 952         struct userfaultfd_wait_queue *uwq;
 953
 954         lockdep_assert_held(&wqh->lock);
 955
 956         uwq = NULL;
 957         if (!waitqueue_active(wqh))
 958                 goto out;
 959         /* walk in reverse to provide FIFO behavior to read userfaults */
 960         wq = list_last_entry(&wqh->head, typeof(*wq), entry);
 961         uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
 962 out:
 963         return uwq;
 964 }
 965
 966 static inline struct userfaultfd_wait_queue *find_userfault(
 967                 struct userfaultfd_ctx *ctx)
 968 {
 969         return find_userfault_in(&ctx->fault_pending_wqh);
 970 }
 971
 972 static inline struct userfaultfd_wait_queue *find_userfault_evt(
 973                 struct userfaultfd_ctx *ctx)
 974 {
 975         return find_userfault_in(&ctx->event_wqh);
 976 }
 977
 978 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
 979 {
 980         struct userfaultfd_ctx *ctx = file->private_data;
 981         __poll_t ret;
 982
 983         poll_wait(file, &ctx->fd_wqh, wait);
 984
 985         if (!userfaultfd_is_initialized(ctx))
 986                 return EPOLLERR;
 987
 988         /*
 989          * poll() never guarantees that read won't block.
 990          * userfaults can be waken before they're read().
 991          */
 992         if (unlikely(!(file->f_flags & O_NONBLOCK)))
 993                 return EPOLLERR;
 994         /*
 995          * lockless access to see if there are pending faults
 996          * __pollwait last action is the add_wait_queue but
 997          * the spin_unlock would allow the waitqueue_active to
 998          * pass above the actual list_add inside
 999          * add_wait_queue critical section. So use a full
1000          * memory barrier to serialize the list_add write of
1001          * add_wait_queue() with the waitqueue_active read
1002          * below.
1003          */
1004         ret = 0;
1005         smp_mb();
1006         if (waitqueue_active(&ctx->fault_pending_wqh))
1007                 ret = EPOLLIN;
1008         else if (waitqueue_active(&ctx->event_wqh))
1009                 ret = EPOLLIN;
1010
1011         return ret;
1012 }
1013
1014 static const struct file_operations userfaultfd_fops;
1015
1016 static int resolve_userfault_fork(struct userfaultfd_ctx *new,
1017                                   struct inode *inode,
1018                                   struct uffd_msg *msg)
1019 {
1020         int fd;
1021
1022         fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
1023                         O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
1024         if (fd < 0)
1025                 return fd;
1026
1027         msg->arg.reserved.reserved1 = 0;
1028         msg->arg.fork.ufd = fd;
1029         return 0;
1030 }
1031
1032 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
1033                                     struct uffd_msg *msg, struct inode *inode)
1034 {
1035         ssize_t ret;
1036         DECLARE_WAITQUEUE(wait, current);
1037         struct userfaultfd_wait_queue *uwq;
1038         /*
1039          * Handling fork event requires sleeping operations, so
1040          * we drop the event_wqh lock, then do these ops, then
1041          * lock it back and wake up the waiter. While the lock is
1042          * dropped the ewq may go away so we keep track of it
1043          * carefully.
1044          */
1045         LIST_HEAD(fork_event);
1046         struct userfaultfd_ctx *fork_nctx = NULL;
1047
1048         /* always take the fd_wqh lock before the fault_pending_wqh lock */
1049         spin_lock_irq(&ctx->fd_wqh.lock);
1050         __add_wait_queue(&ctx->fd_wqh, &wait);
1051         for (;;) {
1052                 set_current_state(TASK_INTERRUPTIBLE);
1053                 spin_lock(&ctx->fault_pending_wqh.lock);
1054                 uwq = find_userfault(ctx);
1055                 if (uwq) {
1056                         /*
1057                          * Use a seqcount to repeat the lockless check
1058                          * in wake_userfault() to avoid missing
1059                          * wakeups because during the refile both
1060                          * waitqueue could become empty if this is the
1061                          * only userfault.
1062                          */
1063                         write_seqcount_begin(&ctx->refile_seq);
1064
1065                         /*
1066                          * The fault_pending_wqh.lock prevents the uwq
1067                          * to disappear from under us.
1068                          *
1069                          * Refile this userfault from
1070                          * fault_pending_wqh to fault_wqh, it's not
1071                          * pending anymore after we read it.
1072                          *
1073                          * Use list_del() by hand (as
1074                          * userfaultfd_wake_function also uses
1075                          * list_del_init() by hand) to be sure nobody
1076                          * changes __remove_wait_queue() to use
1077                          * list_del_init() in turn breaking the
1078                          * !list_empty_careful() check in
1079                          * handle_userfault(). The uwq->wq.head list
1080                          * must never be empty at any time during the
1081                          * refile, or the waitqueue could disappear
1082                          * from under us. The "wait_queue_head_t"
1083                          * parameter of __remove_wait_queue() is unused
1084                          * anyway.
1085                          */
1086                         list_del(&uwq->wq.entry);
1087                         add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1088
1089                         write_seqcount_end(&ctx->refile_seq);
1090
1091                         /* careful to always initialize msg if ret == 0 */
1092                         *msg = uwq->msg;
1093                         spin_unlock(&ctx->fault_pending_wqh.lock);
1094                         ret = 0;
1095                         break;
1096                 }
1097                 spin_unlock(&ctx->fault_pending_wqh.lock);
1098
1099                 spin_lock(&ctx->event_wqh.lock);
1100                 uwq = find_userfault_evt(ctx);
1101                 if (uwq) {
1102                         *msg = uwq->msg;
1103
1104                         if (uwq->msg.event == UFFD_EVENT_FORK) {
1105                                 fork_nctx = (struct userfaultfd_ctx *)
1106                                         (unsigned long)
1107                                         uwq->msg.arg.reserved.reserved1;
1108                                 list_move(&uwq->wq.entry, &fork_event);
1109                                 /*
1110                                  * fork_nctx can be freed as soon as
1111                                  * we drop the lock, unless we take a
1112                                  * reference on it.
1113                                  */
1114                                 userfaultfd_ctx_get(fork_nctx);
1115                                 spin_unlock(&ctx->event_wqh.lock);
1116                                 ret = 0;
1117                                 break;
1118                         }
1119
1120                         userfaultfd_event_complete(ctx, uwq);
1121                         spin_unlock(&ctx->event_wqh.lock);
1122                         ret = 0;
1123                         break;
1124                 }
1125                 spin_unlock(&ctx->event_wqh.lock);
1126
1127                 if (signal_pending(current)) {
1128                         ret = -ERESTARTSYS;
1129                         break;
1130                 }
1131                 if (no_wait) {
1132                         ret = -EAGAIN;
1133                         break;
1134                 }
1135                 spin_unlock_irq(&ctx->fd_wqh.lock);
1136                 schedule();
1137                 spin_lock_irq(&ctx->fd_wqh.lock);
1138         }
1139         __remove_wait_queue(&ctx->fd_wqh, &wait);
1140         __set_current_state(TASK_RUNNING);
1141         spin_unlock_irq(&ctx->fd_wqh.lock);
1142
1143         if (!ret && msg->event == UFFD_EVENT_FORK) {
1144                 ret = resolve_userfault_fork(fork_nctx, inode, msg);
1145                 spin_lock_irq(&ctx->event_wqh.lock);
1146                 if (!list_empty(&fork_event)) {
1147                         /*
1148                          * The fork thread didn't abort, so we can
1149                          * drop the temporary refcount.
1150                          */
1151                         userfaultfd_ctx_put(fork_nctx);
1152
1153                         uwq = list_first_entry(&fork_event,
1154                                                typeof(*uwq),
1155                                                wq.entry);
1156                         /*
1157                          * If fork_event list wasn't empty and in turn
1158                          * the event wasn't already released by fork
1159                          * (the event is allocated on fork kernel
1160                          * stack), put the event back to its place in
1161                          * the event_wq. fork_event head will be freed
1162                          * as soon as we return so the event cannot
1163                          * stay queued there no matter the current
1164                          * "ret" value.
1165                          */
1166                         list_del(&uwq->wq.entry);
1167                         __add_wait_queue(&ctx->event_wqh, &uwq->wq);
1168
1169                         /*
1170                          * Leave the event in the waitqueue and report
1171                          * error to userland if we failed to resolve
1172                          * the userfault fork.
1173                          */
1174                         if (likely(!ret))
1175                                 userfaultfd_event_complete(ctx, uwq);
1176                 } else {
1177                         /*
1178                          * Here the fork thread aborted and the
1179                          * refcount from the fork thread on fork_nctx
1180                          * has already been released. We still hold
1181                          * the reference we took before releasing the
1182                          * lock above. If resolve_userfault_fork
1183                          * failed we've to drop it because the
1184                          * fork_nctx has to be freed in such case. If
1185                          * it succeeded we'll hold it because the new
1186                          * uffd references it.
1187                          */
1188                         if (ret)
1189                                 userfaultfd_ctx_put(fork_nctx);
1190                 }
1191                 spin_unlock_irq(&ctx->event_wqh.lock);
1192         }
1193
1194         return ret;
1195 }
1196
1197 static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1198                                 size_t count, loff_t *ppos)
1199 {
1200         struct userfaultfd_ctx *ctx = file->private_data;
1201         ssize_t _ret, ret = 0;
1202         struct uffd_msg msg;
1203         int no_wait = file->f_flags & O_NONBLOCK;
1204         struct inode *inode = file_inode(file);
1205
1206         if (!userfaultfd_is_initialized(ctx))
1207                 return -EINVAL;
1208
1209         for (;;) {
1210                 if (count < sizeof(msg))
1211                         return ret ? ret : -EINVAL;
1212                 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
1213                 if (_ret < 0)
1214                         return ret ? ret : _ret;
1215                 if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
1216                         return ret ? ret : -EFAULT;
1217                 ret += sizeof(msg);
1218                 buf += sizeof(msg);
1219                 count -= sizeof(msg);
1220                 /*
1221                  * Allow to read more than one fault at time but only
1222                  * block if waiting for the very first one.
1223                  */
1224                 no_wait = O_NONBLOCK;
1225         }
1226 }
1227
1228 static void __wake_userfault(struct userfaultfd_ctx *ctx,
1229                              struct userfaultfd_wake_range *range)
1230 {
1231         spin_lock_irq(&ctx->fault_pending_wqh.lock);
1232         /* wake all in the range and autoremove */
1233         if (waitqueue_active(&ctx->fault_pending_wqh))
1234                 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1235                                      range);
1236         if (waitqueue_active(&ctx->fault_wqh))
1237                 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1238         spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1239 }
1240
1241 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1242                                            struct userfaultfd_wake_range *range)
1243 {
1244         unsigned seq;
1245         bool need_wakeup;
1246
1247         /*
1248          * To be sure waitqueue_active() is not reordered by the CPU
1249          * before the pagetable update, use an explicit SMP memory
1250          * barrier here. PT lock release or mmap_read_unlock(mm) still
1251          * have release semantics that can allow the
1252          * waitqueue_active() to be reordered before the pte update.
1253          */
1254         smp_mb();
1255
1256         /*
1257          * Use waitqueue_active because it's very frequent to
1258          * change the address space atomically even if there are no
1259          * userfaults yet. So we take the spinlock only when we're
1260          * sure we've userfaults to wake.
1261          */
1262         do {
1263                 seq = read_seqcount_begin(&ctx->refile_seq);
1264                 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1265                         waitqueue_active(&ctx->fault_wqh);
1266                 cond_resched();
1267         } while (read_seqcount_retry(&ctx->refile_seq, seq));
1268         if (need_wakeup)
1269                 __wake_userfault(ctx, range);
1270 }
1271
1272 static __always_inline int validate_range(struct mm_struct *mm,
1273                                           __u64 start, __u64 len)
1274 {
1275         __u64 task_size = mm->task_size;
1276
1277         if (start & ~PAGE_MASK)
1278                 return -EINVAL;
1279         if (len & ~PAGE_MASK)
1280                 return -EINVAL;
1281         if (!len)
1282                 return -EINVAL;
1283         if (start < mmap_min_addr)
1284                 return -EINVAL;
1285         if (start >= task_size)
1286                 return -EINVAL;
1287         if (len > task_size - start)
1288                 return -EINVAL;
1289         return 0;
1290 }
1291
1292 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1293                                 unsigned long arg)
1294 {
1295         struct mm_struct *mm = ctx->mm;
1296         struct vm_area_struct *vma, *prev, *cur;
1297         int ret;
1298         struct uffdio_register uffdio_register;
1299         struct uffdio_register __user *user_uffdio_register;
1300         unsigned long vm_flags, new_flags;
1301         bool found;
1302         bool basic_ioctls;
1303         unsigned long start, end, vma_end;
1304         struct vma_iterator vmi;
1305
1306         user_uffdio_register = (struct uffdio_register __user *) arg;
1307
1308         ret = -EFAULT;
1309         if (copy_from_user(&uffdio_register, user_uffdio_register,
1310                            sizeof(uffdio_register)-sizeof(__u64)))
1311                 goto out;
1312
1313         ret = -EINVAL;
1314         if (!uffdio_register.mode)
1315                 goto out;
1316         if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
1317                 goto out;
1318         vm_flags = 0;
1319         if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1320                 vm_flags |= VM_UFFD_MISSING;
1321         if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1322 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1323                 goto out;
1324 #endif
1325                 vm_flags |= VM_UFFD_WP;
1326         }
1327         if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
1328 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1329                 goto out;
1330 #endif
1331                 vm_flags |= VM_UFFD_MINOR;
1332         }
1333
1334         ret = validate_range(mm, uffdio_register.range.start,
1335                              uffdio_register.range.len);
1336         if (ret)
1337                 goto out;
1338
1339         start = uffdio_register.range.start;
1340         end = start + uffdio_register.range.len;
1341
1342         ret = -ENOMEM;
1343         if (!mmget_not_zero(mm))
1344                 goto out;
1345
1346         ret = -EINVAL;
1347         mmap_write_lock(mm);
1348         vma_iter_init(&vmi, mm, start);
1349         vma = vma_find(&vmi, end);
1350         if (!vma)
1351                 goto out_unlock;
1352
1353         /*
1354          * If the first vma contains huge pages, make sure start address
1355          * is aligned to huge page size.
1356          */
1357         if (is_vm_hugetlb_page(vma)) {
1358                 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1359
1360                 if (start & (vma_hpagesize - 1))
1361                         goto out_unlock;
1362         }
1363
1364         /*
1365          * Search for not compatible vmas.
1366          */
1367         found = false;
1368         basic_ioctls = false;
1369         cur = vma;
1370         do {
1371                 cond_resched();
1372
1373                 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1374                        !!(cur->vm_flags & __VM_UFFD_FLAGS));
1375
1376                 /* check not compatible vmas */
1377                 ret = -EINVAL;
1378                 if (!vma_can_userfault(cur, vm_flags))
1379                         goto out_unlock;
1380
1381                 /*
1382                  * UFFDIO_COPY will fill file holes even without
1383                  * PROT_WRITE. This check enforces that if this is a
1384                  * MAP_SHARED, the process has write permission to the backing
1385                  * file. If VM_MAYWRITE is set it also enforces that on a
1386                  * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1387                  * F_WRITE_SEAL can be taken until the vma is destroyed.
1388                  */
1389                 ret = -EPERM;
1390                 if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1391                         goto out_unlock;
1392
1393                 /*
1394                  * If this vma contains ending address, and huge pages
1395                  * check alignment.
1396                  */
1397                 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1398                     end > cur->vm_start) {
1399                         unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1400
1401                         ret = -EINVAL;
1402
1403                         if (end & (vma_hpagesize - 1))
1404                                 goto out_unlock;
1405                 }
1406                 if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
1407                         goto out_unlock;
1408
1409                 /*
1410                  * Check that this vma isn't already owned by a
1411                  * different userfaultfd. We can't allow more than one
1412                  * userfaultfd to own a single vma simultaneously or we
1413                  * wouldn't know which one to deliver the userfaults to.
1414                  */
1415                 ret = -EBUSY;
1416                 if (cur->vm_userfaultfd_ctx.ctx &&
1417                     cur->vm_userfaultfd_ctx.ctx != ctx)
1418                         goto out_unlock;
1419
1420                 /*
1421                  * Note vmas containing huge pages
1422                  */
1423                 if (is_vm_hugetlb_page(cur))
1424                         basic_ioctls = true;
1425
1426                 found = true;
1427         } for_each_vma_range(vmi, cur, end);
1428         BUG_ON(!found);
1429
1430         vma_iter_set(&vmi, start);
1431         prev = vma_prev(&vmi);
1432
1433         ret = 0;
1434         for_each_vma_range(vmi, vma, end) {
1435                 cond_resched();
1436
1437                 BUG_ON(!vma_can_userfault(vma, vm_flags));
1438                 BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1439                        vma->vm_userfaultfd_ctx.ctx != ctx);
1440                 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1441
1442                 /*
1443                  * Nothing to do: this vma is already registered into this
1444                  * userfaultfd and with the right tracking mode too.
1445                  */
1446                 if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1447                     (vma->vm_flags & vm_flags) == vm_flags)
1448                         goto skip;
1449
1450                 if (vma->vm_start > start)
1451                         start = vma->vm_start;
1452                 vma_end = min(end, vma->vm_end);
1453
1454                 new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
1455                 prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
1456                                  vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1457                                  vma_policy(vma),
1458                                  ((struct vm_userfaultfd_ctx){ ctx }),
1459                                  anon_vma_name(vma));
1460                 if (prev) {
1461                         /* vma_merge() invalidated the mas */
1462                         vma = prev;
1463                         goto next;
1464                 }
1465                 if (vma->vm_start < start) {
1466                         ret = split_vma(&vmi, vma, start, 1);
1467                         if (ret)
1468                                 break;
1469                 }
1470                 if (vma->vm_end > end) {
1471                         ret = split_vma(&vmi, vma, end, 0);
1472                         if (ret)
1473                                 break;
1474                 }
1475         next:
1476                 /*
1477                  * In the vma_merge() successful mprotect-like case 8:
1478                  * the next vma was merged into the current one and
1479                  * the current one has not been updated yet.
1480                  */
1481                 userfaultfd_set_vm_flags(vma, new_flags);
1482                 vma->vm_userfaultfd_ctx.ctx = ctx;
1483
1484                 if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
1485                         hugetlb_unshare_all_pmds(vma);
1486
1487         skip:
1488                 prev = vma;
1489                 start = vma->vm_end;
1490         }
1491
1492 out_unlock:
1493         mmap_write_unlock(mm);
1494         mmput(mm);
1495         if (!ret) {
1496                 __u64 ioctls_out;
1497
1498                 ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1499                     UFFD_API_RANGE_IOCTLS;
1500
1501                 /*
1502                  * Declare the WP ioctl only if the WP mode is
1503                  * specified and all checks passed with the range
1504                  */
1505                 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
1506                         ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
1507
1508                 /* CONTINUE ioctl is only supported for MINOR ranges. */
1509                 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
1510                         ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
1511
1512                 /*
1513                  * Now that we scanned all vmas we can already tell
1514                  * userland which ioctls methods are guaranteed to
1515                  * succeed on this range.
1516                  */
1517                 if (put_user(ioctls_out, &user_uffdio_register->ioctls))
1518                         ret = -EFAULT;
1519         }
1520 out:
1521         return ret;
1522 }
1523
1524 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1525                                   unsigned long arg)
1526 {
1527         struct mm_struct *mm = ctx->mm;
1528         struct vm_area_struct *vma, *prev, *cur;
1529         int ret;
1530         struct uffdio_range uffdio_unregister;
1531         unsigned long new_flags;
1532         bool found;
1533         unsigned long start, end, vma_end;
1534         const void __user *buf = (void __user *)arg;
1535         struct vma_iterator vmi;
1536
1537         ret = -EFAULT;
1538         if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1539                 goto out;
1540
1541         ret = validate_range(mm, uffdio_unregister.start,
1542                              uffdio_unregister.len);
1543         if (ret)
1544                 goto out;
1545
1546         start = uffdio_unregister.start;
1547         end = start + uffdio_unregister.len;
1548
1549         ret = -ENOMEM;
1550         if (!mmget_not_zero(mm))
1551                 goto out;
1552
1553         mmap_write_lock(mm);
1554         ret = -EINVAL;
1555         vma_iter_init(&vmi, mm, start);
1556         vma = vma_find(&vmi, end);
1557         if (!vma)
1558                 goto out_unlock;
1559
1560         /*
1561          * If the first vma contains huge pages, make sure start address
1562          * is aligned to huge page size.
1563          */
1564         if (is_vm_hugetlb_page(vma)) {
1565                 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1566
1567                 if (start & (vma_hpagesize - 1))
1568                         goto out_unlock;
1569         }
1570
1571         /*
1572          * Search for not compatible vmas.
1573          */
1574         found = false;
1575         cur = vma;
1576         do {
1577                 cond_resched();
1578
1579                 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1580                        !!(cur->vm_flags & __VM_UFFD_FLAGS));
1581
1582                 /*
1583                  * Check not compatible vmas, not strictly required
1584                  * here as not compatible vmas cannot have an
1585                  * userfaultfd_ctx registered on them, but this
1586                  * provides for more strict behavior to notice
1587                  * unregistration errors.
1588                  */
1589                 if (!vma_can_userfault(cur, cur->vm_flags))
1590                         goto out_unlock;
1591
1592                 found = true;
1593         } for_each_vma_range(vmi, cur, end);
1594         BUG_ON(!found);
1595
1596         vma_iter_set(&vmi, start);
1597         prev = vma_prev(&vmi);
1598         ret = 0;
1599         for_each_vma_range(vmi, vma, end) {
1600                 cond_resched();
1601
1602                 BUG_ON(!vma_can_userfault(vma, vma->vm_flags));
1603
1604                 /*
1605                  * Nothing to do: this vma is already registered into this
1606                  * userfaultfd and with the right tracking mode too.
1607                  */
1608                 if (!vma->vm_userfaultfd_ctx.ctx)
1609                         goto skip;
1610
1611                 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1612
1613                 if (vma->vm_start > start)
1614                         start = vma->vm_start;
1615                 vma_end = min(end, vma->vm_end);
1616
1617                 if (userfaultfd_missing(vma)) {
1618                         /*
1619                          * Wake any concurrent pending userfault while
1620                          * we unregister, so they will not hang
1621                          * permanently and it avoids userland to call
1622                          * UFFDIO_WAKE explicitly.
1623                          */
1624                         struct userfaultfd_wake_range range;
1625                         range.start = start;
1626                         range.len = vma_end - start;
1627                         wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1628                 }
1629
1630                 /* Reset ptes for the whole vma range if wr-protected */
1631                 if (userfaultfd_wp(vma))
1632                         uffd_wp_range(mm, vma, start, vma_end - start, false);
1633
1634                 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
1635                 prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
1636                                  vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1637                                  vma_policy(vma),
1638                                  NULL_VM_UFFD_CTX, anon_vma_name(vma));
1639                 if (prev) {
1640                         vma = prev;
1641                         goto next;
1642                 }
1643                 if (vma->vm_start < start) {
1644                         ret = split_vma(&vmi, vma, start, 1);
1645                         if (ret)
1646                                 break;
1647                 }
1648                 if (vma->vm_end > end) {
1649                         ret = split_vma(&vmi, vma, end, 0);
1650                         if (ret)
1651                                 break;
1652                 }
1653         next:
1654                 /*
1655                  * In the vma_merge() successful mprotect-like case 8:
1656                  * the next vma was merged into the current one and
1657                  * the current one has not been updated yet.
1658                  */
1659                 userfaultfd_set_vm_flags(vma, new_flags);
1660                 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1661
1662         skip:
1663                 prev = vma;
1664                 start = vma->vm_end;
1665         }
1666
1667 out_unlock:
1668         mmap_write_unlock(mm);
1669         mmput(mm);
1670 out:
1671         return ret;
1672 }
1673
1674 /*
1675  * userfaultfd_wake may be used in combination with the
1676  * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1677  */
1678 static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1679                             unsigned long arg)
1680 {
1681         int ret;
1682         struct uffdio_range uffdio_wake;
1683         struct userfaultfd_wake_range range;
1684         const void __user *buf = (void __user *)arg;
1685
1686         ret = -EFAULT;
1687         if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1688                 goto out;
1689
1690         ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
1691         if (ret)
1692                 goto out;
1693
1694         range.start = uffdio_wake.start;
1695         range.len = uffdio_wake.len;
1696
1697         /*
1698          * len == 0 means wake all and we don't want to wake all here,
1699          * so check it again to be sure.
1700          */
1701         VM_BUG_ON(!range.len);
1702
1703         wake_userfault(ctx, &range);
1704         ret = 0;
1705
1706 out:
1707         return ret;
1708 }
1709
1710 static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1711                             unsigned long arg)
1712 {
1713         __s64 ret;
1714         struct uffdio_copy uffdio_copy;
1715         struct uffdio_copy __user *user_uffdio_copy;
1716         struct userfaultfd_wake_range range;
1717
1718         user_uffdio_copy = (struct uffdio_copy __user *) arg;
1719
1720         ret = -EAGAIN;
1721         if (atomic_read(&ctx->mmap_changing))
1722                 goto out;
1723
1724         ret = -EFAULT;
1725         if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1726                            /* don't copy "copy" last field */
1727                            sizeof(uffdio_copy)-sizeof(__s64)))
1728                 goto out;
1729
1730         ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
1731         if (ret)
1732                 goto out;
1733         /*
1734          * double check for wraparound just in case. copy_from_user()
1735          * will later check uffdio_copy.src + uffdio_copy.len to fit
1736          * in the userland range.
1737          */
1738         ret = -EINVAL;
1739         if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1740                 goto out;
1741         if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
1742                 goto out;
1743         if (mmget_not_zero(ctx->mm)) {
1744                 ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
1745                                    uffdio_copy.len, &ctx->mmap_changing,
1746                                    uffdio_copy.mode);
1747                 mmput(ctx->mm);
1748         } else {
1749                 return -ESRCH;
1750         }
1751         if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1752                 return -EFAULT;
1753         if (ret < 0)
1754                 goto out;
1755         BUG_ON(!ret);
1756         /* len == 0 would wake all */
1757         range.len = ret;
1758         if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1759                 range.start = uffdio_copy.dst;
1760                 wake_userfault(ctx, &range);
1761         }
1762         ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1763 out:
1764         return ret;
1765 }
1766
1767 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1768                                 unsigned long arg)
1769 {
1770         __s64 ret;
1771         struct uffdio_zeropage uffdio_zeropage;
1772         struct uffdio_zeropage __user *user_uffdio_zeropage;
1773         struct userfaultfd_wake_range range;
1774
1775         user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1776
1777         ret = -EAGAIN;
1778         if (atomic_read(&ctx->mmap_changing))
1779                 goto out;
1780
1781         ret = -EFAULT;
1782         if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1783                            /* don't copy "zeropage" last field */
1784                            sizeof(uffdio_zeropage)-sizeof(__s64)))
1785                 goto out;
1786
1787         ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
1788                              uffdio_zeropage.range.len);
1789         if (ret)
1790                 goto out;
1791         ret = -EINVAL;
1792         if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1793                 goto out;
1794
1795         if (mmget_not_zero(ctx->mm)) {
1796                 ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
1797                                      uffdio_zeropage.range.len,
1798                                      &ctx->mmap_changing);
1799                 mmput(ctx->mm);
1800         } else {
1801                 return -ESRCH;
1802         }
1803         if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1804                 return -EFAULT;
1805         if (ret < 0)
1806                 goto out;
1807         /* len == 0 would wake all */
1808         BUG_ON(!ret);
1809         range.len = ret;
1810         if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1811                 range.start = uffdio_zeropage.range.start;
1812                 wake_userfault(ctx, &range);
1813         }
1814         ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1815 out:
1816         return ret;
1817 }
1818
1819 static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
1820                                     unsigned long arg)
1821 {
1822         int ret;
1823         struct uffdio_writeprotect uffdio_wp;
1824         struct uffdio_writeprotect __user *user_uffdio_wp;
1825         struct userfaultfd_wake_range range;
1826         bool mode_wp, mode_dontwake;
1827
1828         if (atomic_read(&ctx->mmap_changing))
1829                 return -EAGAIN;
1830
1831         user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
1832
1833         if (copy_from_user(&uffdio_wp, user_uffdio_wp,
1834                            sizeof(struct uffdio_writeprotect)))
1835                 return -EFAULT;
1836
1837         ret = validate_range(ctx->mm, uffdio_wp.range.start,
1838                              uffdio_wp.range.len);
1839         if (ret)
1840                 return ret;
1841
1842         if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
1843                                UFFDIO_WRITEPROTECT_MODE_WP))
1844                 return -EINVAL;
1845
1846         mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
1847         mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
1848
1849         if (mode_wp && mode_dontwake)
1850                 return -EINVAL;
1851
1852         if (mmget_not_zero(ctx->mm)) {
1853                 ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start,
1854                                           uffdio_wp.range.len, mode_wp,
1855                                           &ctx->mmap_changing);
1856                 mmput(ctx->mm);
1857         } else {
1858                 return -ESRCH;
1859         }
1860
1861         if (ret)
1862                 return ret;
1863
1864         if (!mode_wp && !mode_dontwake) {
1865                 range.start = uffdio_wp.range.start;
1866                 range.len = uffdio_wp.range.len;
1867                 wake_userfault(ctx, &range);
1868         }
1869         return ret;
1870 }
1871
1872 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
1873 {
1874         __s64 ret;
1875         struct uffdio_continue uffdio_continue;
1876         struct uffdio_continue __user *user_uffdio_continue;
1877         struct userfaultfd_wake_range range;
1878
1879         user_uffdio_continue = (struct uffdio_continue __user *)arg;
1880
1881         ret = -EAGAIN;
1882         if (atomic_read(&ctx->mmap_changing))
1883                 goto out;
1884
1885         ret = -EFAULT;
1886         if (copy_from_user(&uffdio_continue, user_uffdio_continue,
1887                            /* don't copy the output fields */
1888                            sizeof(uffdio_continue) - (sizeof(__s64))))
1889                 goto out;
1890
1891         ret = validate_range(ctx->mm, uffdio_continue.range.start,
1892                              uffdio_continue.range.len);
1893         if (ret)
1894                 goto out;
1895
1896         ret = -EINVAL;
1897         /* double check for wraparound just in case. */
1898         if (uffdio_continue.range.start + uffdio_continue.range.len <=
1899             uffdio_continue.range.start) {
1900                 goto out;
1901         }
1902         if (uffdio_continue.mode & ~UFFDIO_CONTINUE_MODE_DONTWAKE)
1903                 goto out;
1904
1905         if (mmget_not_zero(ctx->mm)) {
1906                 ret = mcopy_continue(ctx->mm, uffdio_continue.range.start,
1907                                      uffdio_continue.range.len,
1908                                      &ctx->mmap_changing);
1909                 mmput(ctx->mm);
1910         } else {
1911                 return -ESRCH;
1912         }
1913
1914         if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
1915                 return -EFAULT;
1916         if (ret < 0)
1917                 goto out;
1918
1919         /* len == 0 would wake all */
1920         BUG_ON(!ret);
1921         range.len = ret;
1922         if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
1923                 range.start = uffdio_continue.range.start;
1924                 wake_userfault(ctx, &range);
1925         }
1926         ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
1927
1928 out:
1929         return ret;
1930 }
1931
1932 static inline unsigned int uffd_ctx_features(__u64 user_features)
1933 {
1934         /*
1935          * For the current set of features the bits just coincide. Set
1936          * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
1937          */
1938         return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
1939 }
1940
1941 /*
1942  * userland asks for a certain API version and we return which bits
1943  * and ioctl commands are implemented in this kernel for such API
1944  * version or -EINVAL if unknown.
1945  */
1946 static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1947                            unsigned long arg)
1948 {
1949         struct uffdio_api uffdio_api;
1950         void __user *buf = (void __user *)arg;
1951         unsigned int ctx_features;
1952         int ret;
1953         __u64 features;
1954
1955         ret = -EFAULT;
1956         if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
1957                 goto out;
1958         /* Ignore unsupported features (userspace built against newer kernel) */
1959         features = uffdio_api.features & UFFD_API_FEATURES;
1960         ret = -EPERM;
1961         if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
1962                 goto err_out;
1963         /* report all available features and ioctls to userland */
1964         uffdio_api.features = UFFD_API_FEATURES;
1965 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1966         uffdio_api.features &=
1967                 ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
1968 #endif
1969 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1970         uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
1971 #endif
1972 #ifndef CONFIG_PTE_MARKER_UFFD_WP
1973         uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
1974 #endif
1975         uffdio_api.ioctls = UFFD_API_IOCTLS;
1976         ret = -EFAULT;
1977         if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1978                 goto out;
1979
1980         /* only enable the requested features for this uffd context */
1981         ctx_features = uffd_ctx_features(features);
1982         ret = -EINVAL;
1983         if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
1984                 goto err_out;
1985
1986         ret = 0;
1987 out:
1988         return ret;
1989 err_out:
1990         memset(&uffdio_api, 0, sizeof(uffdio_api));
1991         if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1992                 ret = -EFAULT;
1993         goto out;
1994 }
1995
1996 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1997                               unsigned long arg)
1998 {
1999         int ret = -EINVAL;
2000         struct userfaultfd_ctx *ctx = file->private_data;
2001
2002         if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
2003                 return -EINVAL;
2004
2005         switch(cmd) {
2006         case UFFDIO_API:
2007                 ret = userfaultfd_api(ctx, arg);
2008                 break;
2009         case UFFDIO_REGISTER:
2010                 ret = userfaultfd_register(ctx, arg);
2011                 break;
2012         case UFFDIO_UNREGISTER:
2013                 ret = userfaultfd_unregister(ctx, arg);
2014                 break;
2015         case UFFDIO_WAKE:
2016                 ret = userfaultfd_wake(ctx, arg);
2017                 break;
2018         case UFFDIO_COPY:
2019                 ret = userfaultfd_copy(ctx, arg);
2020                 break;
2021         case UFFDIO_ZEROPAGE:
2022                 ret = userfaultfd_zeropage(ctx, arg);
2023                 break;
2024         case UFFDIO_WRITEPROTECT:
2025                 ret = userfaultfd_writeprotect(ctx, arg);
2026                 break;
2027         case UFFDIO_CONTINUE:
2028                 ret = userfaultfd_continue(ctx, arg);
2029                 break;
2030         }
2031         return ret;
2032 }
2033
2034 #ifdef CONFIG_PROC_FS
2035 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
2036 {
2037         struct userfaultfd_ctx *ctx = f->private_data;
2038         wait_queue_entry_t *wq;
2039         unsigned long pending = 0, total = 0;
2040
2041         spin_lock_irq(&ctx->fault_pending_wqh.lock);
2042         list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
2043                 pending++;
2044                 total++;
2045         }
2046         list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
2047                 total++;
2048         }
2049         spin_unlock_irq(&ctx->fault_pending_wqh.lock);
2050
2051         /*
2052          * If more protocols will be added, there will be all shown
2053          * separated by a space. Like this:
2054          *      protocols: aa:... bb:...
2055          */
2056         seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
2057                    pending, total, UFFD_API, ctx->features,
2058                    UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
2059 }
2060 #endif
2061
2062 static const struct file_operations userfaultfd_fops = {
2063 #ifdef CONFIG_PROC_FS
2064         .show_fdinfo    = userfaultfd_show_fdinfo,
2065 #endif
2066         .release        = userfaultfd_release,
2067         .poll           = userfaultfd_poll,
2068         .read           = userfaultfd_read,
2069         .unlocked_ioctl = userfaultfd_ioctl,
2070         .compat_ioctl   = compat_ptr_ioctl,
2071         .llseek         = noop_llseek,
2072 };
2073
2074 static void init_once_userfaultfd_ctx(void *mem)
2075 {
2076         struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
2077
2078         init_waitqueue_head(&ctx->fault_pending_wqh);
2079         init_waitqueue_head(&ctx->fault_wqh);
2080         init_waitqueue_head(&ctx->event_wqh);
2081         init_waitqueue_head(&ctx->fd_wqh);
2082         seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
2083 }
2084
2085 static int new_userfaultfd(int flags)
2086 {
2087         struct userfaultfd_ctx *ctx;
2088         int fd;
2089
2090         BUG_ON(!current->mm);
2091
2092         /* Check the UFFD_* constants for consistency.  */
2093         BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
2094         BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
2095         BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
2096
2097         if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
2098                 return -EINVAL;
2099
2100         ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
2101         if (!ctx)
2102                 return -ENOMEM;
2103
2104         refcount_set(&ctx->refcount, 1);
2105         ctx->flags = flags;
2106         ctx->features = 0;
2107         ctx->released = false;
2108         atomic_set(&ctx->mmap_changing, 0);
2109         ctx->mm = current->mm;
2110         /* prevent the mm struct to be freed */
2111         mmgrab(ctx->mm);
2112
2113         fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
2114                         O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
2115         if (fd < 0) {
2116                 mmdrop(ctx->mm);
2117                 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
2118         }
2119         return fd;
2120 }
2121
2122 static inline bool userfaultfd_syscall_allowed(int flags)
2123 {
2124         /* Userspace-only page faults are always allowed */
2125         if (flags & UFFD_USER_MODE_ONLY)
2126                 return true;
2127
2128         /*
2129          * The user is requesting a userfaultfd which can handle kernel faults.
2130          * Privileged users are always allowed to do this.
2131          */
2132         if (capable(CAP_SYS_PTRACE))
2133                 return true;
2134
2135         /* Otherwise, access to kernel fault handling is sysctl controlled. */
2136         return sysctl_unprivileged_userfaultfd;
2137 }
2138
2139 SYSCALL_DEFINE1(userfaultfd, int, flags)
2140 {
2141         if (!userfaultfd_syscall_allowed(flags))
2142                 return -EPERM;
2143
2144         return new_userfaultfd(flags);
2145 }
2146
2147 static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)
2148 {
2149         if (cmd != USERFAULTFD_IOC_NEW)
2150                 return -EINVAL;
2151
2152         return new_userfaultfd(flags);
2153 }
2154
2155 static const struct file_operations userfaultfd_dev_fops = {
2156         .unlocked_ioctl = userfaultfd_dev_ioctl,
2157         .compat_ioctl = userfaultfd_dev_ioctl,
2158         .owner = THIS_MODULE,
2159         .llseek = noop_llseek,
2160 };
2161
2162 static struct miscdevice userfaultfd_misc = {
2163         .minor = MISC_DYNAMIC_MINOR,
2164         .name = "userfaultfd",
2165         .fops = &userfaultfd_dev_fops
2166 };
2167
2168 static int __init userfaultfd_init(void)
2169 {
2170         int ret;
2171
2172         ret = misc_register(&userfaultfd_misc);
2173         if (ret)
2174                 return ret;
2175
2176         userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
2177                                                 sizeof(struct userfaultfd_ctx),
2178                                                 0,
2179                                                 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2180                                                 init_once_userfaultfd_ctx);
2181         return 0;
2182 }
2183 __initcall(userfaultfd_init);