hisi_acc_vfio_pci: Use its own PCI reset_done error handler

Register private handler for pci_error_handlers.reset_done and update
state accordingly.

Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Longfang Liu <liulongfang@huawei.com>
Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Link: https://lore.kernel.org/r/20220308184902.2242-10-shameerali.kolothum.thodi@huawei.com
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>

diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index 51b814f..767b5d4 100644 (file)
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -626,6 +626,27 @@ static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vde
        }
 }
 
+/*
+ * This function is called in all state_mutex unlock cases to
+ * handle a 'deferred_reset' if exists.
+ */
+static void
+hisi_acc_vf_state_mutex_unlock(struct hisi_acc_vf_core_device *hisi_acc_vdev)
+{
+again:
+       spin_lock(&hisi_acc_vdev->reset_lock);
+       if (hisi_acc_vdev->deferred_reset) {
+               hisi_acc_vdev->deferred_reset = false;
+               spin_unlock(&hisi_acc_vdev->reset_lock);
+               hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
+               hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
+               hisi_acc_vf_disable_fds(hisi_acc_vdev);
+               goto again;
+       }
+       mutex_unlock(&hisi_acc_vdev->state_mutex);
+       spin_unlock(&hisi_acc_vdev->reset_lock);
+}
+
 static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 {
        struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
@@ -922,7 +943,7 @@ hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,
                        break;
                }
        }
-       mutex_unlock(&hisi_acc_vdev->state_mutex);
+       hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
        return res;
 }
 
@@ -935,10 +956,35 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
 
        mutex_lock(&hisi_acc_vdev->state_mutex);
        *curr_state = hisi_acc_vdev->mig_state;
-       mutex_unlock(&hisi_acc_vdev->state_mutex);
+       hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
        return 0;
 }
 
+static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
+{
+       struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(&pdev->dev);
+
+       if (hisi_acc_vdev->core_device.vdev.migration_flags !=
+                               VFIO_MIGRATION_STOP_COPY)
+               return;
+
+       /*
+        * As the higher VFIO layers are holding locks across reset and using
+        * those same locks with the mm_lock we need to prevent ABBA deadlock
+        * with the state_mutex and mm_lock.
+        * In case the state_mutex was taken already we defer the cleanup work
+        * to the unlock flow of the other running context.
+        */
+       spin_lock(&hisi_acc_vdev->reset_lock);
+       hisi_acc_vdev->deferred_reset = true;
+       if (!mutex_trylock(&hisi_acc_vdev->state_mutex)) {
+               spin_unlock(&hisi_acc_vdev->reset_lock);
+               return;
+       }
+       spin_unlock(&hisi_acc_vdev->reset_lock);
+       hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
+}
+
 static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 {
        struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
@@ -1259,12 +1305,17 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
 
 MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
 
+static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
+       .reset_done = hisi_acc_vf_pci_aer_reset_done,
+       .error_detected = vfio_pci_core_aer_err_detected,
+};
+
 static struct pci_driver hisi_acc_vfio_pci_driver = {
        .name = KBUILD_MODNAME,
        .id_table = hisi_acc_vfio_pci_table,
        .probe = hisi_acc_vfio_pci_probe,
        .remove = hisi_acc_vfio_pci_remove,
-       .err_handler = &vfio_pci_core_err_handlers,
+       .err_handler = &hisi_acc_vf_err_handlers,
 };
 
 module_pci_driver(hisi_acc_vfio_pci_driver);

diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
index 1c7d754..5494f49 100644 (file)
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
@@ -98,6 +98,7 @@ struct hisi_acc_vf_migration_file {
 
 struct hisi_acc_vf_core_device {
        struct vfio_pci_core_device core_device;
+       u8 deferred_reset:1;
        /* for migration state */
        struct mutex state_mutex;
        enum vfio_device_mig_state mig_state;
@@ -107,7 +108,8 @@ struct hisi_acc_vf_core_device {
        struct hisi_qm vf_qm;
        u32 vf_qm_state;
        int vf_id;
-
+       /* for reset handler */
+       spinlock_t reset_lock;
        struct hisi_acc_vf_migration_file *resuming_migf;
        struct hisi_acc_vf_migration_file *saving_migf;
 };