mm/uffd: enable write protection for shmem & hugetlbfs

We've had all the necessary changes ready for both shmem and hugetlbfs.
Turn on all the shmem/hugetlbfs switches for userfaultfd-wp.

We can expand UFFD_API_RANGE_IOCTLS_BASIC with _UFFDIO_WRITEPROTECT too
because all existing types now support write protection mode.

Since vma_can_userfault() will be used elsewhere, move into userfaultfd_k.h.

Link: https://lkml.kernel.org/r/20220405014926.15101-1-peterx@redhat.com
Signed-off-by: Peter Xu <peterx@redhat.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: "Kirill A . Shutemov" <kirill@shutemov.name>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Mike Rapoport <rppt@linux.vnet.ibm.com>
Cc: Nadav Amit <nadav.amit@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 78b68e0f9774c53c1e27e0c2b83f372fb1c87a2d..e943370107d06afab21d37fc43e2aa3714fb816e 100644 (file)
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -1258,24 +1258,6 @@ static __always_inline int validate_range(struct mm_struct *mm,
        return 0;
 }
 
-static inline bool vma_can_userfault(struct vm_area_struct *vma,
-                                    unsigned long vm_flags)
-{
-       /* FIXME: add WP support to hugetlbfs and shmem */
-       if (vm_flags & VM_UFFD_WP) {
-               if (is_vm_hugetlb_page(vma) || vma_is_shmem(vma))
-                       return false;
-       }
-
-       if (vm_flags & VM_UFFD_MINOR) {
-               if (!(is_vm_hugetlb_page(vma) || vma_is_shmem(vma)))
-                       return false;
-       }
-
-       return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
-              vma_is_shmem(vma);
-}
-
 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
                                unsigned long arg)
 {
@@ -1956,6 +1938,9 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx,
 #endif
 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
        uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
+#endif
+#ifndef CONFIG_PTE_MARKER_UFFD_WP
+       uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
 #endif
        uffdio_api.ioctls = UFFD_API_IOCTLS;
        ret = -EFAULT;

diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index e7afcdfd4b46f53dfc57a173c6869b4a4e37812e..732b522bacb7e5c15a8d6e00f9f4d4e8c6daa4ea 100644 (file)
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -18,6 +18,7 @@
 #include <linux/swap.h>
 #include <linux/swapops.h>
 #include <asm-generic/pgtable_uffd.h>
+#include <linux/hugetlb_inline.h>
 
 /* The set of all possible UFFD-related VM flags. */
 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
@@ -140,6 +141,25 @@ static inline bool userfaultfd_armed(struct vm_area_struct *vma)
        return vma->vm_flags & __VM_UFFD_FLAGS;
 }
 
+static inline bool vma_can_userfault(struct vm_area_struct *vma,
+                                    unsigned long vm_flags)
+{
+       if (vm_flags & VM_UFFD_MINOR)
+               return is_vm_hugetlb_page(vma) || vma_is_shmem(vma);
+
+#ifndef CONFIG_PTE_MARKER_UFFD_WP
+       /*
+        * If user requested uffd-wp but not enabled pte markers for
+        * uffd-wp, then shmem & hugetlbfs are not supported but only
+        * anonymous.
+        */
+       if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma))
+               return false;
+#endif
+       return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
+           vma_is_shmem(vma);
+}
+
 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *);
 extern void dup_userfaultfd_complete(struct list_head *);
 

diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h
index ef739054cb1c35b560dd54e6b32cee275a56ef4f..7d32b1e797fb2f3deb67bebe097982256d69135a 100644 (file)
--- a/include/uapi/linux/userfaultfd.h
+++ b/include/uapi/linux/userfaultfd.h
@@ -33,7 +33,8 @@
                           UFFD_FEATURE_THREAD_ID |             \
                           UFFD_FEATURE_MINOR_HUGETLBFS |       \
                           UFFD_FEATURE_MINOR_SHMEM |           \
-                          UFFD_FEATURE_EXACT_ADDRESS)
+                          UFFD_FEATURE_EXACT_ADDRESS |         \
+                          UFFD_FEATURE_WP_HUGETLBFS_SHMEM)
 #define UFFD_API_IOCTLS                                \
        ((__u64)1 << _UFFDIO_REGISTER |         \
         (__u64)1 << _UFFDIO_UNREGISTER |       \
@@ -47,7 +48,8 @@
 #define UFFD_API_RANGE_IOCTLS_BASIC            \
        ((__u64)1 << _UFFDIO_WAKE |             \
         (__u64)1 << _UFFDIO_COPY |             \
-        (__u64)1 << _UFFDIO_CONTINUE)
+        (__u64)1 << _UFFDIO_CONTINUE |         \
+        (__u64)1 << _UFFDIO_WRITEPROTECT)
 
 /*
  * Valid ioctl command number range with this API is from 0x00 to
@@ -194,6 +196,9 @@ struct uffdio_api {
         * UFFD_FEATURE_EXACT_ADDRESS indicates that the exact address of page
         * faults would be provided and the offset within the page would not be
         * masked.
+        *
+        * UFFD_FEATURE_WP_HUGETLBFS_SHMEM indicates that userfaultfd
+        * write-protection mode is supported on both shmem and hugetlbfs.
         */
 #define UFFD_FEATURE_PAGEFAULT_FLAG_WP         (1<<0)
 #define UFFD_FEATURE_EVENT_FORK                        (1<<1)
@@ -207,6 +212,7 @@ struct uffdio_api {
 #define UFFD_FEATURE_MINOR_HUGETLBFS           (1<<9)
 #define UFFD_FEATURE_MINOR_SHMEM               (1<<10)
 #define UFFD_FEATURE_EXACT_ADDRESS             (1<<11)
+#define UFFD_FEATURE_WP_HUGETLBFS_SHMEM                (1<<12)
        __u64 features;
 
        __u64 ioctls;

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 01edc18902c543435ee821ea12b31374e2cb23b2..4f4892a5f767f9f36678226bf8c72d9aa733240b 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -732,15 +732,12 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start,
 
        err = -ENOENT;
        dst_vma = find_dst_vma(dst_mm, start, len);
-       /*
-        * Make sure the vma is not shared, that the dst range is
-        * both valid and fully within a single existing vma.
-        */
-       if (!dst_vma || (dst_vma->vm_flags & VM_SHARED))
+
+       if (!dst_vma)
                goto out_unlock;
        if (!userfaultfd_wp(dst_vma))
                goto out_unlock;
-       if (!vma_is_anonymous(dst_vma))
+       if (!vma_can_userfault(dst_vma, dst_vma->vm_flags))
                goto out_unlock;
 
        if (is_vm_hugetlb_page(dst_vma)) {