IB/mlx5: Introduce DEVX

Introduce DEVX to enable direct device commands in downstream patches
from this series.

In that mode of work the firmware manages the isolation between
processes' resources and as such a DEVX user id is created and assigned
to the given user context upon allocation request.

A capability check is done to make sure that this feature is really
supported by the firmware prior to creating the DEVX user id.

Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile
index d42b922..577e4c4 100644 (file)
--- a/drivers/infiniband/hw/mlx5/Makefile
+++ b/drivers/infiniband/hw/mlx5/Makefile
@@ -3,3 +3,4 @@ obj-$(CONFIG_MLX5_INFINIBAND)   += mlx5_ib.o
 mlx5_ib-y :=   main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o cong.o
 mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
 mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
+mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o

diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
new file mode 100644 (file)
index 0000000..7754489
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
+ */
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/uverbs_types.h>
+#include <rdma/uverbs_ioctl.h>
+#include <rdma/mlx5_user_ioctl_cmds.h>
+#include <rdma/ib_umem.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/fs.h>
+#include "mlx5_ib.h"
+
+int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
+{
+       u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
+       u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+       u64 general_obj_types;
+       void *uctx;
+       void *hdr;
+       int err;
+
+       uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
+       hdr = MLX5_ADDR_OF(create_uctx_in, in, hdr);
+
+       general_obj_types = MLX5_CAP_GEN_64(dev->mdev, general_obj_types);
+       if (!(general_obj_types & MLX5_GENERAL_OBJ_TYPES_CAP_UCTX) ||
+           !(general_obj_types & MLX5_GENERAL_OBJ_TYPES_CAP_UMEM))
+               return -EINVAL;
+
+       if (!capable(CAP_NET_RAW))
+               return -EPERM;
+
+       MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+       MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type, MLX5_OBJ_TYPE_UCTX);
+
+       err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+       if (err)
+               return err;
+
+       context->devx_uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+       return 0;
+}
+
+void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev,
+                         struct mlx5_ib_ucontext *context)
+{
+       u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
+       u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+
+       MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+       MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_UCTX);
+       MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, context->devx_uid);
+
+       mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+}

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index e46cda7..058a82a 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -1650,8 +1650,8 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
        if (err)
                return ERR_PTR(err);
 
-       if (req.flags)
-               return ERR_PTR(-EINVAL);
+       if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
+               return ERR_PTR(-EOPNOTSUPP);
 
        if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
                return ERR_PTR(-EOPNOTSUPP);
@@ -1735,6 +1735,18 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
                        goto out_uars;
        }
 
+       if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
+               /* Block DEVX on Infiniband as of SELinux */
+               if (mlx5_ib_port_link_layer(ibdev, 1) != IB_LINK_LAYER_ETHERNET) {
+                       err = -EPERM;
+                       goto out_td;
+               }
+
+               err = mlx5_ib_devx_create(dev, context);
+               if (err)
+                       goto out_td;
+       }
+
        INIT_LIST_HEAD(&context->vma_private_list);
        mutex_init(&context->vma_private_list_mutex);
        INIT_LIST_HEAD(&context->db_page_list);
@@ -1795,7 +1807,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 
        err = ib_copy_to_udata(udata, &resp, resp.response_length);
        if (err)
-               goto out_td;
+               goto out_mdev;
 
        bfregi->ver = ver;
        bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
@@ -1805,6 +1817,9 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 
        return &context->ibucontext;
 
+out_mdev:
+       if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
+               mlx5_ib_devx_destroy(dev, context);
 out_td:
        if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
                mlx5_ib_dealloc_transport_domain(dev, context->tdn);
@@ -1830,6 +1845,9 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
        struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
        struct mlx5_bfreg_info *bfregi;
 
+       if (context->devx_uid)
+               mlx5_ib_devx_destroy(dev, context);
+
        bfregi = &context->bfregi;
        if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
                mlx5_ib_dealloc_transport_domain(dev, context->tdn);

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 615bd6e..1c857dd 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -143,6 +143,7 @@ struct mlx5_ib_ucontext {
 
        u64                     lib_caps;
        DECLARE_BITMAP(dm_pages, MLX5_MAX_MEMIC_PAGES);
+       u16                     devx_uid;
 };
 
 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
@@ -1215,6 +1216,18 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev,
 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
                                  u8 port_num);
 
+#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
+int mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
+                       struct mlx5_ib_ucontext *context);
+void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev,
+                         struct mlx5_ib_ucontext *context);
+#else
+static inline int
+mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
+                   struct mlx5_ib_ucontext *context) { return -EOPNOTSUPP; };
+static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev,
+                                       struct mlx5_ib_ucontext *context) {}
+#endif
 static inline void init_query_mad(struct ib_smp *mad)
 {
        mad->base_version  = 1;

diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h
index 8daec1f..5d591ff 100644 (file)
--- a/include/uapi/rdma/mlx5-abi.h
+++ b/include/uapi/rdma/mlx5-abi.h
@@ -76,6 +76,9 @@ enum mlx5_lib_caps {
        MLX5_LIB_CAP_4K_UAR     = (__u64)1 << 0,
 };
 
+enum mlx5_ib_alloc_uctx_v2_flags {
+       MLX5_IB_ALLOC_UCTX_DEVX = 1 << 0,
+};
 struct mlx5_ib_alloc_ucontext_req_v2 {
        __u32   total_num_bfregs;
        __u32   num_low_latency_bfregs;