net/mlx5: Expose mlx5_sriov_blocking_notifier_register / unregister APIs

Expose mlx5_sriov_blocking_notifier_register / unregister APIs to let a
VF register to be notified for its enablement / disablement by the PF.

Upon VF probe it will call mlx5_sriov_blocking_notifier_register() with
its notifier block and upon VF remove it will call
mlx5_sriov_blocking_notifier_unregister() to drop its registration.

This can give a VF the ability to clean some resources upon disable
before that the command interface goes down and on the other hand sets
some stuff before that it's enabled.

This may be used by a VF which is migration capable in few cases.(e.g.
PF load/unload upon an health recovery).

Link: https://lore.kernel.org/r/20220510090206.90374-2-yishaih@nvidia.com
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
index 887ee0f..2935614 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
@@ -87,6 +87,11 @@ static int mlx5_device_enable_sriov(struct mlx5_core_dev *dev, int num_vfs)
 enable_vfs_hca:
        num_msix_count = mlx5_get_default_msix_vec_count(dev, num_vfs);
        for (vf = 0; vf < num_vfs; vf++) {
+               /* Notify the VF before its enablement to let it set
+                * some stuff.
+                */
+               blocking_notifier_call_chain(&sriov->vfs_ctx[vf].notifier,
+                                            MLX5_PF_NOTIFY_ENABLE_VF, dev);
                err = mlx5_core_enable_hca(dev, vf + 1);
                if (err) {
                        mlx5_core_warn(dev, "failed to enable VF %d (%d)\n", vf, err);
@@ -127,6 +132,11 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf)
        for (vf = num_vfs - 1; vf >= 0; vf--) {
                if (!sriov->vfs_ctx[vf].enabled)
                        continue;
+               /* Notify the VF before its disablement to let it clean
+                * some resources.
+                */
+               blocking_notifier_call_chain(&sriov->vfs_ctx[vf].notifier,
+                                            MLX5_PF_NOTIFY_DISABLE_VF, dev);
                err = mlx5_core_disable_hca(dev, vf + 1);
                if (err) {
                        mlx5_core_warn(dev, "failed to disable VF %d\n", vf);
@@ -257,7 +267,7 @@ int mlx5_sriov_init(struct mlx5_core_dev *dev)
 {
        struct mlx5_core_sriov *sriov = &dev->priv.sriov;
        struct pci_dev *pdev = dev->pdev;
-       int total_vfs;
+       int total_vfs, i;
 
        if (!mlx5_core_is_pf(dev))
                return 0;
@@ -269,6 +279,9 @@ int mlx5_sriov_init(struct mlx5_core_dev *dev)
        if (!sriov->vfs_ctx)
                return -ENOMEM;
 
+       for (i = 0; i < total_vfs; i++)
+               BLOCKING_INIT_NOTIFIER_HEAD(&sriov->vfs_ctx[i].notifier);
+
        return 0;
 }
 
@@ -281,3 +294,53 @@ void mlx5_sriov_cleanup(struct mlx5_core_dev *dev)
 
        kfree(sriov->vfs_ctx);
 }
+
+/**
+ * mlx5_sriov_blocking_notifier_unregister - Unregister a VF from
+ * a notification block chain.
+ *
+ * @mdev: The mlx5 core device.
+ * @vf_id: The VF id.
+ * @nb: The notifier block to be unregistered.
+ */
+void mlx5_sriov_blocking_notifier_unregister(struct mlx5_core_dev *mdev,
+                                            int vf_id,
+                                            struct notifier_block *nb)
+{
+       struct mlx5_vf_context *vfs_ctx;
+       struct mlx5_core_sriov *sriov;
+
+       sriov = &mdev->priv.sriov;
+       if (WARN_ON(vf_id < 0 || vf_id >= sriov->num_vfs))
+               return;
+
+       vfs_ctx = &sriov->vfs_ctx[vf_id];
+       blocking_notifier_chain_unregister(&vfs_ctx->notifier, nb);
+}
+EXPORT_SYMBOL(mlx5_sriov_blocking_notifier_unregister);
+
+/**
+ * mlx5_sriov_blocking_notifier_register - Register a VF notification
+ * block chain.
+ *
+ * @mdev: The mlx5 core device.
+ * @vf_id: The VF id.
+ * @nb: The notifier block to be called upon the VF events.
+ *
+ * Returns 0 on success or an error code.
+ */
+int mlx5_sriov_blocking_notifier_register(struct mlx5_core_dev *mdev,
+                                         int vf_id,
+                                         struct notifier_block *nb)
+{
+       struct mlx5_vf_context *vfs_ctx;
+       struct mlx5_core_sriov *sriov;
+
+       sriov = &mdev->priv.sriov;
+       if (vf_id < 0 || vf_id >= sriov->num_vfs)
+               return -EINVAL;
+
+       vfs_ctx = &sriov->vfs_ctx[vf_id];
+       return blocking_notifier_chain_register(&vfs_ctx->notifier, nb);
+}
+EXPORT_SYMBOL(mlx5_sriov_blocking_notifier_register);

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index ff47d49..6fac542 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -445,6 +445,11 @@ struct mlx5_qp_table {
        struct radix_tree_root  tree;
 };
 
+enum {
+       MLX5_PF_NOTIFY_DISABLE_VF,
+       MLX5_PF_NOTIFY_ENABLE_VF,
+};
+
 struct mlx5_vf_context {
        int     enabled;
        u64     port_guid;
@@ -455,6 +460,7 @@ struct mlx5_vf_context {
        u8      port_guid_valid:1;
        u8      node_guid_valid:1;
        enum port_state_policy  policy;
+       struct blocking_notifier_head notifier;
 };
 
 struct mlx5_core_sriov {
@@ -1152,6 +1158,12 @@ int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type
 struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev);
 void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev);
 
+int mlx5_sriov_blocking_notifier_register(struct mlx5_core_dev *mdev,
+                                         int vf_id,
+                                         struct notifier_block *nb);
+void mlx5_sriov_blocking_notifier_unregister(struct mlx5_core_dev *mdev,
+                                            int vf_id,
+                                            struct notifier_block *nb);
 #ifdef CONFIG_MLX5_CORE_IPOIB
 struct net_device *mlx5_rdma_netdev_alloc(struct mlx5_core_dev *mdev,
                                          struct ib_device *ibdev,