mlx5: Fix missing device local_dma_lkey

The mlx5 driver exposes device capability IB_DEVICE_LOCAL_DMA_LKEY
but does not set the the device local_dma_lkey. This breaks
rpcrdma drivers.

Query and set this lkey when creating the device resources.

Signed-off-by: Sagi Grimberg <sagig@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 085c24b..5ab5bb3 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -1256,10 +1256,18 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
        struct ib_srq_init_attr attr;
        struct mlx5_ib_dev *dev;
        struct ib_cq_init_attr cq_attr = {.cqe = 1};
+       u32 rsvd_lkey;
        int ret = 0;
 
        dev = container_of(devr, struct mlx5_ib_dev, devr);
 
+       ret = mlx5_core_query_special_context(dev->mdev, &rsvd_lkey);
+       if (ret) {
+               pr_err("Failed to query special context %d\n", ret);
+               return ret;
+       }
+       dev->ib_dev.local_dma_lkey = rsvd_lkey;
+
        devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
        if (IS_ERR(devr->p0)) {
                ret = PTR_ERR(devr->p0);
@@ -1421,7 +1429,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
        strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
        dev->ib_dev.owner               = THIS_MODULE;
        dev->ib_dev.node_type           = RDMA_NODE_IB_CA;
-       dev->ib_dev.local_dma_lkey      = 0 /* not supported for now */;
        dev->num_ports          = MLX5_CAP_GEN(mdev, num_ports);
        dev->ib_dev.phys_port_cnt     = dev->num_ports;
        dev->ib_dev.num_comp_vectors    =

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index 9335e5a..aa0d5ff 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -200,3 +200,25 @@ int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev)
 
        return err;
 }
+
+int mlx5_core_query_special_context(struct mlx5_core_dev *dev, u32 *rsvd_lkey)
+{
+       struct mlx5_cmd_query_special_contexts_mbox_in in;
+       struct mlx5_cmd_query_special_contexts_mbox_out out;
+       int err;
+
+       memset(&in, 0, sizeof(in));
+       memset(&out, 0, sizeof(out));
+       in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
+       err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+       if (err)
+               return err;
+
+       if (out.hdr.status)
+               err = mlx5_cmd_status_to_err(&out.hdr);
+
+       *rsvd_lkey = be32_to_cpu(out.resd_lkey);
+
+       return err;
+}
+EXPORT_SYMBOL(mlx5_core_query_special_context);

diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index b943cd9..6e4169c 100644 (file)
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -402,6 +402,17 @@ struct mlx5_cmd_teardown_hca_mbox_out {
        u8                      rsvd[8];
 };
 
+struct mlx5_cmd_query_special_contexts_mbox_in {
+       struct mlx5_inbox_hdr   hdr;
+       u8                      rsvd[8];
+};
+
+struct mlx5_cmd_query_special_contexts_mbox_out {
+       struct mlx5_outbox_hdr  hdr;
+       __be32                  dump_fill_mkey;
+       __be32                  resd_lkey;
+};
+
 struct mlx5_cmd_layout {
        u8              type;
        u8              rsvd0[3];

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 5722d88..1e2e48c 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -828,6 +828,7 @@ void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol);
 int mlx5_register_interface(struct mlx5_interface *intf);
 void mlx5_unregister_interface(struct mlx5_interface *intf);
 int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id);
+int mlx5_core_query_special_context(struct mlx5_core_dev *dev, u32 *rsvd_lkey);
 
 struct mlx5_profile {
        u64     mask;