1 // SPDX-License-Identifier: GPL-2.0-only
3 * vDPA bridge driver for modern virtio-pci device
5 * Copyright (c) 2020, Red Hat Inc. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
8 * Based on virtio_pci_modern.c.
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/vdpa.h>
15 #include <linux/virtio.h>
16 #include <linux/virtio_config.h>
17 #include <linux/virtio_ring.h>
18 #include <linux/virtio_pci.h>
19 #include <linux/virtio_pci_modern.h>
21 #define VP_VDPA_QUEUE_MAX 256
22 #define VP_VDPA_DRIVER_NAME "vp_vdpa"
23 #define VP_VDPA_NAME_SIZE 256
27 char msix_name[VP_VDPA_NAME_SIZE];
28 struct vdpa_callback cb;
33 struct vdpa_device vdpa;
34 struct virtio_pci_modern_device mdev;
35 struct vp_vring *vring;
36 struct vdpa_callback config_cb;
37 char msix_name[VP_VDPA_NAME_SIZE];
43 static struct vp_vdpa *vdpa_to_vp(struct vdpa_device *vdpa)
45 return container_of(vdpa, struct vp_vdpa, vdpa);
48 static struct virtio_pci_modern_device *vdpa_to_mdev(struct vdpa_device *vdpa)
50 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
52 return &vp_vdpa->mdev;
55 static u64 vp_vdpa_get_features(struct vdpa_device *vdpa)
57 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
59 return vp_modern_get_features(mdev);
62 static int vp_vdpa_set_features(struct vdpa_device *vdpa, u64 features)
64 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
66 vp_modern_set_features(mdev, features);
71 static u8 vp_vdpa_get_status(struct vdpa_device *vdpa)
73 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
75 return vp_modern_get_status(mdev);
78 static void vp_vdpa_free_irq(struct vp_vdpa *vp_vdpa)
80 struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;
81 struct pci_dev *pdev = mdev->pci_dev;
84 for (i = 0; i < vp_vdpa->queues; i++) {
85 if (vp_vdpa->vring[i].irq != VIRTIO_MSI_NO_VECTOR) {
86 vp_modern_queue_vector(mdev, i, VIRTIO_MSI_NO_VECTOR);
87 devm_free_irq(&pdev->dev, vp_vdpa->vring[i].irq,
89 vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR;
93 if (vp_vdpa->config_irq != VIRTIO_MSI_NO_VECTOR) {
94 vp_modern_config_vector(mdev, VIRTIO_MSI_NO_VECTOR);
95 devm_free_irq(&pdev->dev, vp_vdpa->config_irq, vp_vdpa);
96 vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR;
99 if (vp_vdpa->vectors) {
100 pci_free_irq_vectors(pdev);
101 vp_vdpa->vectors = 0;
105 static irqreturn_t vp_vdpa_vq_handler(int irq, void *arg)
107 struct vp_vring *vring = arg;
109 if (vring->cb.callback)
110 return vring->cb.callback(vring->cb.private);
115 static irqreturn_t vp_vdpa_config_handler(int irq, void *arg)
117 struct vp_vdpa *vp_vdpa = arg;
119 if (vp_vdpa->config_cb.callback)
120 return vp_vdpa->config_cb.callback(vp_vdpa->config_cb.private);
125 static int vp_vdpa_request_irq(struct vp_vdpa *vp_vdpa)
127 struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;
128 struct pci_dev *pdev = mdev->pci_dev;
130 int queues = vp_vdpa->queues;
131 int vectors = queues + 1;
133 ret = pci_alloc_irq_vectors(pdev, vectors, vectors, PCI_IRQ_MSIX);
134 if (ret != vectors) {
136 "vp_vdpa: fail to allocate irq vectors want %d but %d\n",
141 vp_vdpa->vectors = vectors;
143 for (i = 0; i < queues; i++) {
144 snprintf(vp_vdpa->vring[i].msix_name, VP_VDPA_NAME_SIZE,
145 "vp-vdpa[%s]-%d\n", pci_name(pdev), i);
146 irq = pci_irq_vector(pdev, i);
147 ret = devm_request_irq(&pdev->dev, irq,
149 0, vp_vdpa->vring[i].msix_name,
153 "vp_vdpa: fail to request irq for vq %d\n", i);
156 vp_modern_queue_vector(mdev, i, i);
157 vp_vdpa->vring[i].irq = irq;
160 snprintf(vp_vdpa->msix_name, VP_VDPA_NAME_SIZE, "vp-vdpa[%s]-config\n",
162 irq = pci_irq_vector(pdev, queues);
163 ret = devm_request_irq(&pdev->dev, irq, vp_vdpa_config_handler, 0,
164 vp_vdpa->msix_name, vp_vdpa);
167 "vp_vdpa: fail to request irq for vq %d\n", i);
170 vp_modern_config_vector(mdev, queues);
171 vp_vdpa->config_irq = irq;
175 vp_vdpa_free_irq(vp_vdpa);
179 static void vp_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
181 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
182 struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;
183 u8 s = vp_vdpa_get_status(vdpa);
185 if (status & VIRTIO_CONFIG_S_DRIVER_OK &&
186 !(s & VIRTIO_CONFIG_S_DRIVER_OK)) {
187 vp_vdpa_request_irq(vp_vdpa);
190 vp_modern_set_status(mdev, status);
192 if (!(status & VIRTIO_CONFIG_S_DRIVER_OK) &&
193 (s & VIRTIO_CONFIG_S_DRIVER_OK))
194 vp_vdpa_free_irq(vp_vdpa);
197 static u16 vp_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
199 return VP_VDPA_QUEUE_MAX;
202 static int vp_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid,
203 struct vdpa_vq_state *state)
205 /* Note that this is not supported by virtio specification, so
206 * we return -EOPNOTSUPP here. This means we can't support live
207 * migration, vhost device start/stop.
212 static int vp_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 qid,
213 const struct vdpa_vq_state *state)
215 /* Note that this is not supported by virtio specification, so
216 * we return -ENOPOTSUPP here. This means we can't support live
217 * migration, vhost device start/stop.
222 static void vp_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 qid,
223 struct vdpa_callback *cb)
225 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
227 vp_vdpa->vring[qid].cb = *cb;
230 static void vp_vdpa_set_vq_ready(struct vdpa_device *vdpa,
233 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
235 vp_modern_set_queue_enable(mdev, qid, ready);
238 static bool vp_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 qid)
240 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
242 return vp_modern_get_queue_enable(mdev, qid);
245 static void vp_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 qid,
248 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
250 vp_modern_set_queue_size(mdev, qid, num);
253 static int vp_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 qid,
254 u64 desc_area, u64 driver_area,
257 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
259 vp_modern_queue_address(mdev, qid, desc_area,
260 driver_area, device_area);
265 static void vp_vdpa_kick_vq(struct vdpa_device *vdpa, u16 qid)
267 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
269 vp_iowrite16(qid, vp_vdpa->vring[qid].notify);
272 static u32 vp_vdpa_get_generation(struct vdpa_device *vdpa)
274 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
276 return vp_modern_generation(mdev);
279 static u32 vp_vdpa_get_device_id(struct vdpa_device *vdpa)
281 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
283 return mdev->id.device;
286 static u32 vp_vdpa_get_vendor_id(struct vdpa_device *vdpa)
288 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
290 return mdev->id.vendor;
293 static u32 vp_vdpa_get_vq_align(struct vdpa_device *vdpa)
298 static void vp_vdpa_get_config(struct vdpa_device *vdpa,
300 void *buf, unsigned int len)
302 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
303 struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;
309 old = vp_ioread8(&mdev->common->config_generation);
311 for (i = 0; i < len; i++)
312 *p++ = vp_ioread8(mdev->device + offset + i);
314 new = vp_ioread8(&mdev->common->config_generation);
315 } while (old != new);
318 static void vp_vdpa_set_config(struct vdpa_device *vdpa,
319 unsigned int offset, const void *buf,
322 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
323 struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;
327 for (i = 0; i < len; i++)
328 vp_iowrite8(*p++, mdev->device + offset + i);
331 static void vp_vdpa_set_config_cb(struct vdpa_device *vdpa,
332 struct vdpa_callback *cb)
334 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
336 vp_vdpa->config_cb = *cb;
339 static const struct vdpa_config_ops vp_vdpa_ops = {
340 .get_features = vp_vdpa_get_features,
341 .set_features = vp_vdpa_set_features,
342 .get_status = vp_vdpa_get_status,
343 .set_status = vp_vdpa_set_status,
344 .get_vq_num_max = vp_vdpa_get_vq_num_max,
345 .get_vq_state = vp_vdpa_get_vq_state,
346 .set_vq_state = vp_vdpa_set_vq_state,
347 .set_vq_cb = vp_vdpa_set_vq_cb,
348 .set_vq_ready = vp_vdpa_set_vq_ready,
349 .get_vq_ready = vp_vdpa_get_vq_ready,
350 .set_vq_num = vp_vdpa_set_vq_num,
351 .set_vq_address = vp_vdpa_set_vq_address,
352 .kick_vq = vp_vdpa_kick_vq,
353 .get_generation = vp_vdpa_get_generation,
354 .get_device_id = vp_vdpa_get_device_id,
355 .get_vendor_id = vp_vdpa_get_vendor_id,
356 .get_vq_align = vp_vdpa_get_vq_align,
357 .get_config = vp_vdpa_get_config,
358 .set_config = vp_vdpa_set_config,
359 .set_config_cb = vp_vdpa_set_config_cb,
362 static void vp_vdpa_free_irq_vectors(void *data)
364 pci_free_irq_vectors(data);
367 static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
369 struct virtio_pci_modern_device *mdev;
370 struct device *dev = &pdev->dev;
371 struct vp_vdpa *vp_vdpa;
374 ret = pcim_enable_device(pdev);
378 vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa,
379 dev, &vp_vdpa_ops, NULL);
380 if (vp_vdpa == NULL) {
381 dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n");
385 mdev = &vp_vdpa->mdev;
386 mdev->pci_dev = pdev;
388 ret = vp_modern_probe(mdev);
390 dev_err(&pdev->dev, "Failed to probe modern PCI device\n");
394 pci_set_master(pdev);
395 pci_set_drvdata(pdev, vp_vdpa);
397 vp_vdpa->vdpa.dma_dev = &pdev->dev;
398 vp_vdpa->queues = vp_modern_get_num_queues(mdev);
400 ret = devm_add_action_or_reset(dev, vp_vdpa_free_irq_vectors, pdev);
403 "Failed for adding devres for freeing irq vectors\n");
407 vp_vdpa->vring = devm_kcalloc(&pdev->dev, vp_vdpa->queues,
408 sizeof(*vp_vdpa->vring),
410 if (!vp_vdpa->vring) {
412 dev_err(&pdev->dev, "Fail to allocate virtqueues\n");
416 for (i = 0; i < vp_vdpa->queues; i++) {
417 vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR;
418 vp_vdpa->vring[i].notify = vp_modern_map_vq_notify(mdev, i);
419 if (!vp_vdpa->vring[i].notify) {
420 dev_warn(&pdev->dev, "Fail to map vq notify %d\n", i);
424 vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR;
426 ret = vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues);
428 dev_err(&pdev->dev, "Failed to register to vdpa bus\n");
435 put_device(&vp_vdpa->vdpa.dev);
439 static void vp_vdpa_remove(struct pci_dev *pdev)
441 struct vp_vdpa *vp_vdpa = pci_get_drvdata(pdev);
443 vdpa_unregister_device(&vp_vdpa->vdpa);
444 vp_modern_remove(&vp_vdpa->mdev);
447 static struct pci_driver vp_vdpa_driver = {
449 .id_table = NULL, /* only dynamic ids */
450 .probe = vp_vdpa_probe,
451 .remove = vp_vdpa_remove,
454 module_pci_driver(vp_vdpa_driver);
456 MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");
457 MODULE_DESCRIPTION("vp-vdpa");
458 MODULE_LICENSE("GPL");