IB/umem: Add support to huge ODP

Add IB_ACCESS_HUGETLB ib_reg_mr flag.
Hugetlb region registered with this flag
will use single translation entry per huge page.

Signed-off-by: Artemy Kovalyov <artemyko@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 6b87c05..3dbf811 100644 (file)
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -133,7 +133,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
 
        if (access & IB_ACCESS_ON_DEMAND) {
                put_pid(umem->pid);
-               ret = ib_umem_odp_get(context, umem);
+               ret = ib_umem_odp_get(context, umem, access);
                if (ret) {
                        kfree(umem);
                        return ERR_PTR(ret);

diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 73053c8..0780b1a 100644 (file)
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -38,6 +38,7 @@
 #include <linux/slab.h>
 #include <linux/export.h>
 #include <linux/vmalloc.h>
+#include <linux/hugetlb.h>
 
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_umem.h>
@@ -306,7 +307,8 @@ out_umem:
 }
 EXPORT_SYMBOL(ib_alloc_odp_umem);
 
-int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem)
+int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
+                   int access)
 {
        int ret_val;
        struct pid *our_pid;
@@ -315,6 +317,20 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem)
        if (!mm)
                return -EINVAL;
 
+       if (access & IB_ACCESS_HUGETLB) {
+               struct vm_area_struct *vma;
+               struct hstate *h;
+
+               vma = find_vma(mm, ib_umem_start(umem));
+               if (!vma || !is_vm_hugetlb_page(vma))
+                       return -EINVAL;
+               h = hstate_vma(vma);
+               umem->page_shift = huge_page_shift(h);
+               umem->hugetlb = 1;
+       } else {
+               umem->hugetlb = 0;
+       }
+
        /* Prevent creating ODP MRs in child processes */
        rcu_read_lock();
        our_pid = get_task_pid(current->group_leader, PIDTYPE_PID);
@@ -325,7 +341,6 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem)
                goto out_mm;
        }
 
-       umem->hugetlb = 0;
        umem->odp_data = kzalloc(sizeof(*umem->odp_data), GFP_KERNEL);
        if (!umem->odp_data) {
                ret_val = -ENOMEM;

diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index 542cd8b..fb67554 100644 (file)
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -84,7 +84,8 @@ struct ib_umem_odp {
 
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 
-int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem);
+int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
+                   int access);
 struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,
                                  unsigned long addr,
                                  size_t size);
@@ -154,7 +155,8 @@ static inline int ib_umem_mmu_notifier_retry(struct ib_umem *item,
 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
 static inline int ib_umem_odp_get(struct ib_ucontext *context,
-                                 struct ib_umem *umem)
+                                 struct ib_umem *umem,
+                                 int access)
 {
        return -EINVAL;
 }

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 91686d2..ab2dc52 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1339,6 +1339,7 @@ enum ib_access_flags {
        IB_ACCESS_MW_BIND       = (1<<4),
        IB_ZERO_BASED           = (1<<5),
        IB_ACCESS_ON_DEMAND     = (1<<6),
+       IB_ACCESS_HUGETLB       = (1<<7),
 };
 
 /*