vfio/ccw: Convert to use vfio_register_emulated_iommu_dev()

This is a more complicated conversion because vfio_ccw is sharing the
vfio_device between both the mdev_device, its vfio_device and the
css_driver.

The mdev is a singleton, and the reason for this sharing is so the extra
css_driver function callbacks to be delivered to the vfio_device
implementation.

This keeps things as they are, with the css_driver allocating the
singleton, not the mdev_driver.

Embed the vfio_device in the vfio_ccw_private and instantiate it as a
vfio_device when the mdev probes. The drvdata of both the css_device and
the mdev_device point at the private, and container_of is used to get it
back from the vfio_device.

Reviewed-by: Eric Farman <farman@linux.ibm.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/4-v4-cea4f5bd2c00+b52-ccw_mdev_jgg@nvidia.com
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>

diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c
index e32678a..0407427 100644 (file)
--- a/drivers/s390/cio/vfio_ccw_drv.c
+++ b/drivers/s390/cio/vfio_ccw_drv.c
@@ -468,7 +468,7 @@ static int __init vfio_ccw_sch_init(void)
        vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw");
        if (!vfio_ccw_work_q) {
                ret = -ENOMEM;
-               goto out_err;
+               goto out_regions;
        }
 
        vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region",
@@ -477,7 +477,7 @@ static int __init vfio_ccw_sch_init(void)
                                        sizeof(struct ccw_io_region), NULL);
        if (!vfio_ccw_io_region) {
                ret = -ENOMEM;
-               goto out_err;
+               goto out_regions;
        }
 
        vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region",
@@ -486,7 +486,7 @@ static int __init vfio_ccw_sch_init(void)
                                        sizeof(struct ccw_cmd_region), NULL);
        if (!vfio_ccw_cmd_region) {
                ret = -ENOMEM;
-               goto out_err;
+               goto out_regions;
        }
 
        vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region",
@@ -496,7 +496,7 @@ static int __init vfio_ccw_sch_init(void)
 
        if (!vfio_ccw_schib_region) {
                ret = -ENOMEM;
-               goto out_err;
+               goto out_regions;
        }
 
        vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region",
@@ -506,19 +506,25 @@ static int __init vfio_ccw_sch_init(void)
 
        if (!vfio_ccw_crw_region) {
                ret = -ENOMEM;
-               goto out_err;
+               goto out_regions;
        }
 
+       ret = mdev_register_driver(&vfio_ccw_mdev_driver);
+       if (ret)
+               goto out_regions;
+
        isc_register(VFIO_CCW_ISC);
        ret = css_driver_register(&vfio_ccw_sch_driver);
        if (ret) {
                isc_unregister(VFIO_CCW_ISC);
-               goto out_err;
+               goto out_driver;
        }
 
        return ret;
 
-out_err:
+out_driver:
+       mdev_unregister_driver(&vfio_ccw_mdev_driver);
+out_regions:
        vfio_ccw_destroy_regions();
        destroy_workqueue(vfio_ccw_work_q);
        vfio_ccw_debug_exit();
@@ -528,6 +534,7 @@ out_err:
 static void __exit vfio_ccw_sch_exit(void)
 {
        css_driver_unregister(&vfio_ccw_sch_driver);
+       mdev_unregister_driver(&vfio_ccw_mdev_driver);
        isc_unregister(VFIO_CCW_ISC);
        vfio_ccw_destroy_regions();
        destroy_workqueue(vfio_ccw_work_q);

diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c
index 1edbea9..d8589af 100644 (file)
--- a/drivers/s390/cio/vfio_ccw_ops.c
+++ b/drivers/s390/cio/vfio_ccw_ops.c
@@ -17,6 +17,8 @@
 
 #include "vfio_ccw_private.h"
 
+static const struct vfio_device_ops vfio_ccw_dev_ops;
+
 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private)
 {
        struct subchannel *sch;
@@ -111,10 +113,10 @@ static struct attribute_group *mdev_type_groups[] = {
        NULL,
 };
 
-static int vfio_ccw_mdev_create(struct mdev_device *mdev)
+static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
 {
-       struct vfio_ccw_private *private =
-               dev_get_drvdata(mdev_parent_dev(mdev));
+       struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent);
+       int ret;
 
        if (private->state == VFIO_CCW_STATE_NOT_OPER)
                return -ENODEV;
@@ -122,6 +124,10 @@ static int vfio_ccw_mdev_create(struct mdev_device *mdev)
        if (atomic_dec_if_positive(&private->avail) < 0)
                return -EPERM;
 
+       memset(&private->vdev, 0, sizeof(private->vdev));
+       vfio_init_group_dev(&private->vdev, &mdev->dev,
+                           &vfio_ccw_dev_ops);
+
        private->mdev = mdev;
        private->state = VFIO_CCW_STATE_IDLE;
 
@@ -130,19 +136,31 @@ static int vfio_ccw_mdev_create(struct mdev_device *mdev)
                           private->sch->schid.ssid,
                           private->sch->schid.sch_no);
 
+       ret = vfio_register_emulated_iommu_dev(&private->vdev);
+       if (ret)
+               goto err_atomic;
+       dev_set_drvdata(&mdev->dev, private);
        return 0;
+
+err_atomic:
+       vfio_uninit_group_dev(&private->vdev);
+       atomic_inc(&private->avail);
+       private->mdev = NULL;
+       private->state = VFIO_CCW_STATE_IDLE;
+       return ret;
 }
 
-static int vfio_ccw_mdev_remove(struct mdev_device *mdev)
+static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
 {
-       struct vfio_ccw_private *private =
-               dev_get_drvdata(mdev_parent_dev(mdev));
+       struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent);
 
        VFIO_CCW_MSG_EVENT(2, "mdev %pUl, sch %x.%x.%04x: remove\n",
                           mdev_uuid(mdev), private->sch->schid.cssid,
                           private->sch->schid.ssid,
                           private->sch->schid.sch_no);
 
+       vfio_unregister_group_dev(&private->vdev);
+
        if ((private->state != VFIO_CCW_STATE_NOT_OPER) &&
            (private->state != VFIO_CCW_STATE_STANDBY)) {
                if (!vfio_ccw_sch_quiesce(private->sch))
@@ -150,23 +168,22 @@ static int vfio_ccw_mdev_remove(struct mdev_device *mdev)
                /* The state will be NOT_OPER on error. */
        }
 
+       vfio_uninit_group_dev(&private->vdev);
        cp_free(&private->cp);
        private->mdev = NULL;
        atomic_inc(&private->avail);
-
-       return 0;
 }
 
-static int vfio_ccw_mdev_open_device(struct mdev_device *mdev)
+static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
 {
        struct vfio_ccw_private *private =
-               dev_get_drvdata(mdev_parent_dev(mdev));
+               container_of(vdev, struct vfio_ccw_private, vdev);
        unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
        int ret;
 
        private->nb.notifier_call = vfio_ccw_mdev_notifier;
 
-       ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
+       ret = vfio_register_notifier(vdev->dev, VFIO_IOMMU_NOTIFY,
                                     &events, &private->nb);
        if (ret)
                return ret;
@@ -187,15 +204,15 @@ static int vfio_ccw_mdev_open_device(struct mdev_device *mdev)
 
 out_unregister:
        vfio_ccw_unregister_dev_regions(private);
-       vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
+       vfio_unregister_notifier(vdev->dev, VFIO_IOMMU_NOTIFY,
                                 &private->nb);
        return ret;
 }
 
-static void vfio_ccw_mdev_close_device(struct mdev_device *mdev)
+static void vfio_ccw_mdev_close_device(struct vfio_device *vdev)
 {
        struct vfio_ccw_private *private =
-               dev_get_drvdata(mdev_parent_dev(mdev));
+               container_of(vdev, struct vfio_ccw_private, vdev);
 
        if ((private->state != VFIO_CCW_STATE_NOT_OPER) &&
            (private->state != VFIO_CCW_STATE_STANDBY)) {
@@ -206,8 +223,7 @@ static void vfio_ccw_mdev_close_device(struct mdev_device *mdev)
 
        cp_free(&private->cp);
        vfio_ccw_unregister_dev_regions(private);
-       vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
-                                &private->nb);
+       vfio_unregister_notifier(vdev->dev, VFIO_IOMMU_NOTIFY, &private->nb);
 }
 
 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,
@@ -231,15 +247,14 @@ static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,
        return ret;
 }
 
-static ssize_t vfio_ccw_mdev_read(struct mdev_device *mdev,
+static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev,
                                  char __user *buf,
                                  size_t count,
                                  loff_t *ppos)
 {
+       struct vfio_ccw_private *private =
+               container_of(vdev, struct vfio_ccw_private, vdev);
        unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
-       struct vfio_ccw_private *private;
-
-       private = dev_get_drvdata(mdev_parent_dev(mdev));
 
        if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
                return -EINVAL;
@@ -284,15 +299,14 @@ out_unlock:
        return ret;
 }
 
-static ssize_t vfio_ccw_mdev_write(struct mdev_device *mdev,
+static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev,
                                   const char __user *buf,
                                   size_t count,
                                   loff_t *ppos)
 {
+       struct vfio_ccw_private *private =
+               container_of(vdev, struct vfio_ccw_private, vdev);
        unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
-       struct vfio_ccw_private *private;
-
-       private = dev_get_drvdata(mdev_parent_dev(mdev));
 
        if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
                return -EINVAL;
@@ -510,12 +524,12 @@ void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private)
        private->region = NULL;
 }
 
-static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev,
+static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev,
                                   unsigned int cmd,
                                   unsigned long arg)
 {
        struct vfio_ccw_private *private =
-               dev_get_drvdata(mdev_parent_dev(mdev));
+               container_of(vdev, struct vfio_ccw_private, vdev);
        int ret = 0;
        unsigned long minsz;
 
@@ -606,37 +620,48 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev,
 }
 
 /* Request removal of the device*/
-static void vfio_ccw_mdev_request(struct mdev_device *mdev, unsigned int count)
+static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)
 {
-       struct vfio_ccw_private *private = dev_get_drvdata(mdev_parent_dev(mdev));
-
-       if (!private)
-               return;
+       struct vfio_ccw_private *private =
+               container_of(vdev, struct vfio_ccw_private, vdev);
+       struct device *dev = vdev->dev;
 
        if (private->req_trigger) {
                if (!(count % 10))
-                       dev_notice_ratelimited(mdev_dev(private->mdev),
+                       dev_notice_ratelimited(dev,
                                               "Relaying device request to user (#%u)\n",
                                               count);
 
                eventfd_signal(private->req_trigger, 1);
        } else if (count == 0) {
-               dev_notice(mdev_dev(private->mdev),
+               dev_notice(dev,
                           "No device request channel registered, blocked until released by user\n");
        }
 }
 
+static const struct vfio_device_ops vfio_ccw_dev_ops = {
+       .open_device = vfio_ccw_mdev_open_device,
+       .close_device = vfio_ccw_mdev_close_device,
+       .read = vfio_ccw_mdev_read,
+       .write = vfio_ccw_mdev_write,
+       .ioctl = vfio_ccw_mdev_ioctl,
+       .request = vfio_ccw_mdev_request,
+};
+
+struct mdev_driver vfio_ccw_mdev_driver = {
+       .driver = {
+               .name = "vfio_ccw_mdev",
+               .owner = THIS_MODULE,
+               .mod_name = KBUILD_MODNAME,
+       },
+       .probe = vfio_ccw_mdev_probe,
+       .remove = vfio_ccw_mdev_remove,
+};
+
 static const struct mdev_parent_ops vfio_ccw_mdev_ops = {
        .owner                  = THIS_MODULE,
+       .device_driver          = &vfio_ccw_mdev_driver,
        .supported_type_groups  = mdev_type_groups,
-       .create                 = vfio_ccw_mdev_create,
-       .remove                 = vfio_ccw_mdev_remove,
-       .open_device            = vfio_ccw_mdev_open_device,
-       .close_device           = vfio_ccw_mdev_close_device,
-       .read                   = vfio_ccw_mdev_read,
-       .write                  = vfio_ccw_mdev_write,
-       .ioctl                  = vfio_ccw_mdev_ioctl,
-       .request                = vfio_ccw_mdev_request,
 };
 
 int vfio_ccw_mdev_reg(struct subchannel *sch)

diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h
index b2c762e..7272eb7 100644 (file)
--- a/drivers/s390/cio/vfio_ccw_private.h
+++ b/drivers/s390/cio/vfio_ccw_private.h
@@ -17,6 +17,7 @@
 #include <linux/eventfd.h>
 #include <linux/workqueue.h>
 #include <linux/vfio_ccw.h>
+#include <linux/vfio.h>
 #include <asm/crw.h>
 #include <asm/debug.h>
 
@@ -67,6 +68,7 @@ struct vfio_ccw_crw {
 
 /**
  * struct vfio_ccw_private
+ * @vdev: Embedded VFIO device
  * @sch: pointer to the subchannel
  * @state: internal state of the device
  * @completion: synchronization helper of the I/O completion
@@ -90,6 +92,7 @@ struct vfio_ccw_crw {
  * @crw_work: work for deferral process of CRW handling
  */
 struct vfio_ccw_private {
+       struct vfio_device vdev;
        struct subchannel       *sch;
        int                     state;
        struct completion       *completion;
@@ -121,6 +124,8 @@ extern void vfio_ccw_mdev_unreg(struct subchannel *sch);
 
 extern int vfio_ccw_sch_quiesce(struct subchannel *sch);
 
+extern struct mdev_driver vfio_ccw_mdev_driver;
+
 /*
  * States of the device statemachine.
  */