1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2018-2020 Intel Corporation.
4 * Copyright (C) 2020 Red Hat, Inc.
6 * Author: Tiwei Bie <tiwei.bie@intel.com>
7 * Jason Wang <jasowang@redhat.com>
9 * Thanks Michael S. Tsirkin for the valuable comments and
10 * suggestions. And thanks to Cunming Liang and Zhihong Wang for all
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
19 #include <linux/slab.h>
20 #include <linux/iommu.h>
21 #include <linux/uuid.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vhost.h>
29 VHOST_VDPA_BACKEND_FEATURES =
30 (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
31 (1ULL << VHOST_BACKEND_F_IOTLB_BATCH),
34 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
37 struct vhost_dev vdev;
38 struct iommu_domain *domain;
39 struct vhost_virtqueue *vqs;
40 struct completion completion;
41 struct vdpa_device *vdpa;
48 struct eventfd_ctx *config_ctx;
50 struct vdpa_iova_range range;
53 static DEFINE_IDA(vhost_vdpa_ida);
55 static dev_t vhost_vdpa_major;
57 static void handle_vq_kick(struct vhost_work *work)
59 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
61 struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
62 const struct vdpa_config_ops *ops = v->vdpa->config;
64 ops->kick_vq(v->vdpa, vq - v->vqs);
67 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
69 struct vhost_virtqueue *vq = private;
70 struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
73 eventfd_signal(call_ctx, 1);
78 static irqreturn_t vhost_vdpa_config_cb(void *private)
80 struct vhost_vdpa *v = private;
81 struct eventfd_ctx *config_ctx = v->config_ctx;
84 eventfd_signal(config_ctx, 1);
89 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
91 struct vhost_virtqueue *vq = &v->vqs[qid];
92 const struct vdpa_config_ops *ops = v->vdpa->config;
93 struct vdpa_device *vdpa = v->vdpa;
99 irq = ops->get_vq_irq(vdpa, qid);
103 irq_bypass_unregister_producer(&vq->call_ctx.producer);
104 if (!vq->call_ctx.ctx)
107 vq->call_ctx.producer.token = vq->call_ctx.ctx;
108 vq->call_ctx.producer.irq = irq;
109 ret = irq_bypass_register_producer(&vq->call_ctx.producer);
111 dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret = %d\n",
112 qid, vq->call_ctx.producer.token, ret);
115 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
117 struct vhost_virtqueue *vq = &v->vqs[qid];
119 irq_bypass_unregister_producer(&vq->call_ctx.producer);
122 static int vhost_vdpa_reset(struct vhost_vdpa *v)
124 struct vdpa_device *vdpa = v->vdpa;
128 return vdpa_reset(vdpa);
131 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
133 struct vdpa_device *vdpa = v->vdpa;
134 const struct vdpa_config_ops *ops = vdpa->config;
137 device_id = ops->get_device_id(vdpa);
139 if (copy_to_user(argp, &device_id, sizeof(device_id)))
145 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
147 struct vdpa_device *vdpa = v->vdpa;
148 const struct vdpa_config_ops *ops = vdpa->config;
151 status = ops->get_status(vdpa);
153 if (copy_to_user(statusp, &status, sizeof(status)))
159 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
161 struct vdpa_device *vdpa = v->vdpa;
162 const struct vdpa_config_ops *ops = vdpa->config;
163 u8 status, status_old;
164 int ret, nvqs = v->nvqs;
167 if (copy_from_user(&status, statusp, sizeof(status)))
170 status_old = ops->get_status(vdpa);
173 * Userspace shouldn't remove status bits unless reset the
176 if (status != 0 && (ops->get_status(vdpa) & ~status) != 0)
179 if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
180 for (i = 0; i < nvqs; i++)
181 vhost_vdpa_unsetup_vq_irq(v, i);
184 ret = ops->reset(vdpa);
188 ops->set_status(vdpa, status);
190 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
191 for (i = 0; i < nvqs; i++)
192 vhost_vdpa_setup_vq_irq(v, i);
197 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
198 struct vhost_vdpa_config *c)
200 struct vdpa_device *vdpa = v->vdpa;
201 long size = vdpa->config->get_config_size(vdpa);
203 if (c->len == 0 || c->off > size)
206 if (c->len > size - c->off)
212 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
213 struct vhost_vdpa_config __user *c)
215 struct vdpa_device *vdpa = v->vdpa;
216 struct vhost_vdpa_config config;
217 unsigned long size = offsetof(struct vhost_vdpa_config, buf);
220 if (copy_from_user(&config, c, size))
222 if (vhost_vdpa_config_validate(v, &config))
224 buf = kvzalloc(config.len, GFP_KERNEL);
228 vdpa_get_config(vdpa, config.off, buf, config.len);
230 if (copy_to_user(c->buf, buf, config.len)) {
239 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
240 struct vhost_vdpa_config __user *c)
242 struct vdpa_device *vdpa = v->vdpa;
243 const struct vdpa_config_ops *ops = vdpa->config;
244 struct vhost_vdpa_config config;
245 unsigned long size = offsetof(struct vhost_vdpa_config, buf);
248 if (copy_from_user(&config, c, size))
250 if (vhost_vdpa_config_validate(v, &config))
253 buf = vmemdup_user(c->buf, config.len);
257 ops->set_config(vdpa, config.off, buf, config.len);
263 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
265 struct vdpa_device *vdpa = v->vdpa;
266 const struct vdpa_config_ops *ops = vdpa->config;
269 features = ops->get_features(vdpa);
271 if (copy_to_user(featurep, &features, sizeof(features)))
277 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
279 struct vdpa_device *vdpa = v->vdpa;
280 const struct vdpa_config_ops *ops = vdpa->config;
284 * It's not allowed to change the features after they have
287 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
290 if (copy_from_user(&features, featurep, sizeof(features)))
293 if (vdpa_set_features(vdpa, features))
299 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
301 struct vdpa_device *vdpa = v->vdpa;
302 const struct vdpa_config_ops *ops = vdpa->config;
305 num = ops->get_vq_num_max(vdpa);
307 if (copy_to_user(argp, &num, sizeof(num)))
313 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
316 eventfd_ctx_put(v->config_ctx);
317 v->config_ctx = NULL;
321 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
323 struct vdpa_callback cb;
325 struct eventfd_ctx *ctx;
327 cb.callback = vhost_vdpa_config_cb;
329 if (copy_from_user(&fd, argp, sizeof(fd)))
332 ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
333 swap(ctx, v->config_ctx);
335 if (!IS_ERR_OR_NULL(ctx))
336 eventfd_ctx_put(ctx);
338 if (IS_ERR(v->config_ctx)) {
339 long ret = PTR_ERR(v->config_ctx);
341 v->config_ctx = NULL;
345 v->vdpa->config->set_config_cb(v->vdpa, &cb);
350 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
352 struct vhost_vdpa_iova_range range = {
353 .first = v->range.first,
354 .last = v->range.last,
357 if (copy_to_user(argp, &range, sizeof(range)))
362 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
365 struct vdpa_device *vdpa = v->vdpa;
366 const struct vdpa_config_ops *ops = vdpa->config;
367 struct vdpa_vq_state vq_state;
368 struct vdpa_callback cb;
369 struct vhost_virtqueue *vq;
370 struct vhost_vring_state s;
374 r = get_user(idx, (u32 __user *)argp);
381 idx = array_index_nospec(idx, v->nvqs);
385 case VHOST_VDPA_SET_VRING_ENABLE:
386 if (copy_from_user(&s, argp, sizeof(s)))
388 ops->set_vq_ready(vdpa, idx, s.num);
390 case VHOST_GET_VRING_BASE:
391 r = ops->get_vq_state(v->vdpa, idx, &vq_state);
395 vq->last_avail_idx = vq_state.split.avail_index;
399 r = vhost_vring_ioctl(&v->vdev, cmd, argp);
404 case VHOST_SET_VRING_ADDR:
405 if (ops->set_vq_address(vdpa, idx,
406 (u64)(uintptr_t)vq->desc,
407 (u64)(uintptr_t)vq->avail,
408 (u64)(uintptr_t)vq->used))
412 case VHOST_SET_VRING_BASE:
413 vq_state.split.avail_index = vq->last_avail_idx;
414 if (ops->set_vq_state(vdpa, idx, &vq_state))
418 case VHOST_SET_VRING_CALL:
419 if (vq->call_ctx.ctx) {
420 cb.callback = vhost_vdpa_virtqueue_cb;
426 ops->set_vq_cb(vdpa, idx, &cb);
427 vhost_vdpa_setup_vq_irq(v, idx);
430 case VHOST_SET_VRING_NUM:
431 ops->set_vq_num(vdpa, idx, vq->num);
438 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
439 unsigned int cmd, unsigned long arg)
441 struct vhost_vdpa *v = filep->private_data;
442 struct vhost_dev *d = &v->vdev;
443 void __user *argp = (void __user *)arg;
444 u64 __user *featurep = argp;
448 if (cmd == VHOST_SET_BACKEND_FEATURES) {
449 if (copy_from_user(&features, featurep, sizeof(features)))
451 if (features & ~VHOST_VDPA_BACKEND_FEATURES)
453 vhost_set_backend_features(&v->vdev, features);
457 mutex_lock(&d->mutex);
460 case VHOST_VDPA_GET_DEVICE_ID:
461 r = vhost_vdpa_get_device_id(v, argp);
463 case VHOST_VDPA_GET_STATUS:
464 r = vhost_vdpa_get_status(v, argp);
466 case VHOST_VDPA_SET_STATUS:
467 r = vhost_vdpa_set_status(v, argp);
469 case VHOST_VDPA_GET_CONFIG:
470 r = vhost_vdpa_get_config(v, argp);
472 case VHOST_VDPA_SET_CONFIG:
473 r = vhost_vdpa_set_config(v, argp);
475 case VHOST_GET_FEATURES:
476 r = vhost_vdpa_get_features(v, argp);
478 case VHOST_SET_FEATURES:
479 r = vhost_vdpa_set_features(v, argp);
481 case VHOST_VDPA_GET_VRING_NUM:
482 r = vhost_vdpa_get_vring_num(v, argp);
484 case VHOST_SET_LOG_BASE:
485 case VHOST_SET_LOG_FD:
488 case VHOST_VDPA_SET_CONFIG_CALL:
489 r = vhost_vdpa_set_config_call(v, argp);
491 case VHOST_GET_BACKEND_FEATURES:
492 features = VHOST_VDPA_BACKEND_FEATURES;
493 if (copy_to_user(featurep, &features, sizeof(features)))
496 case VHOST_VDPA_GET_IOVA_RANGE:
497 r = vhost_vdpa_get_iova_range(v, argp);
500 r = vhost_dev_ioctl(&v->vdev, cmd, argp);
501 if (r == -ENOIOCTLCMD)
502 r = vhost_vdpa_vring_ioctl(v, cmd, argp);
506 mutex_unlock(&d->mutex);
510 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, u64 start, u64 last)
512 struct vhost_dev *dev = &v->vdev;
513 struct vhost_iotlb *iotlb = dev->iotlb;
514 struct vhost_iotlb_map *map;
516 unsigned long pfn, pinned;
518 while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
519 pinned = PFN_DOWN(map->size);
520 for (pfn = PFN_DOWN(map->addr);
521 pinned > 0; pfn++, pinned--) {
522 page = pfn_to_page(pfn);
523 if (map->perm & VHOST_ACCESS_WO)
524 set_page_dirty_lock(page);
525 unpin_user_page(page);
527 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
528 vhost_iotlb_map_free(iotlb, map);
532 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, u64 start, u64 last)
534 struct vhost_dev *dev = &v->vdev;
535 struct vhost_iotlb *iotlb = dev->iotlb;
536 struct vhost_iotlb_map *map;
537 struct vdpa_map_file *map_file;
539 while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
540 map_file = (struct vdpa_map_file *)map->opaque;
541 fput(map_file->file);
543 vhost_iotlb_map_free(iotlb, map);
547 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last)
549 struct vdpa_device *vdpa = v->vdpa;
552 return vhost_vdpa_va_unmap(v, start, last);
554 return vhost_vdpa_pa_unmap(v, start, last);
557 static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v)
559 struct vhost_dev *dev = &v->vdev;
561 vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1);
566 static int perm_to_iommu_flags(u32 perm)
571 case VHOST_ACCESS_WO:
572 flags |= IOMMU_WRITE;
574 case VHOST_ACCESS_RO:
577 case VHOST_ACCESS_RW:
578 flags |= (IOMMU_WRITE | IOMMU_READ);
581 WARN(1, "invalidate vhost IOTLB permission\n");
585 return flags | IOMMU_CACHE;
588 static int vhost_vdpa_map(struct vhost_vdpa *v, u64 iova,
589 u64 size, u64 pa, u32 perm, void *opaque)
591 struct vhost_dev *dev = &v->vdev;
592 struct vdpa_device *vdpa = v->vdpa;
593 const struct vdpa_config_ops *ops = vdpa->config;
596 r = vhost_iotlb_add_range_ctx(dev->iotlb, iova, iova + size - 1,
602 r = ops->dma_map(vdpa, iova, size, pa, perm, opaque);
603 } else if (ops->set_map) {
605 r = ops->set_map(vdpa, dev->iotlb);
607 r = iommu_map(v->domain, iova, pa, size,
608 perm_to_iommu_flags(perm));
611 vhost_iotlb_del_range(dev->iotlb, iova, iova + size - 1);
616 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
621 static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size)
623 struct vhost_dev *dev = &v->vdev;
624 struct vdpa_device *vdpa = v->vdpa;
625 const struct vdpa_config_ops *ops = vdpa->config;
627 vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1);
630 ops->dma_unmap(vdpa, iova, size);
631 } else if (ops->set_map) {
633 ops->set_map(vdpa, dev->iotlb);
635 iommu_unmap(v->domain, iova, size);
639 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
640 u64 iova, u64 size, u64 uaddr, u32 perm)
642 struct vhost_dev *dev = &v->vdev;
643 u64 offset, map_size, map_iova = iova;
644 struct vdpa_map_file *map_file;
645 struct vm_area_struct *vma;
648 mmap_read_lock(dev->mm);
651 vma = find_vma(dev->mm, uaddr);
656 map_size = min(size, vma->vm_end - uaddr);
657 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
658 !(vma->vm_flags & (VM_IO | VM_PFNMAP))))
661 map_file = kzalloc(sizeof(*map_file), GFP_KERNEL);
666 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
667 map_file->offset = offset;
668 map_file->file = get_file(vma->vm_file);
669 ret = vhost_vdpa_map(v, map_iova, map_size, uaddr,
672 fput(map_file->file);
679 map_iova += map_size;
682 vhost_vdpa_unmap(v, iova, map_iova - iova);
684 mmap_read_unlock(dev->mm);
689 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
690 u64 iova, u64 size, u64 uaddr, u32 perm)
692 struct vhost_dev *dev = &v->vdev;
693 struct page **page_list;
694 unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
695 unsigned int gup_flags = FOLL_LONGTERM;
696 unsigned long npages, cur_base, map_pfn, last_pfn = 0;
697 unsigned long lock_limit, sz2pin, nchunks, i;
702 /* Limit the use of memory for bookkeeping */
703 page_list = (struct page **) __get_free_page(GFP_KERNEL);
707 if (perm & VHOST_ACCESS_WO)
708 gup_flags |= FOLL_WRITE;
710 npages = PFN_UP(size + (iova & ~PAGE_MASK));
716 mmap_read_lock(dev->mm);
718 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
719 if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
724 cur_base = uaddr & PAGE_MASK;
729 sz2pin = min_t(unsigned long, npages, list_size);
730 pinned = pin_user_pages(cur_base, sz2pin,
731 gup_flags, page_list, NULL);
732 if (sz2pin != pinned) {
736 unpin_user_pages(page_list, pinned);
744 map_pfn = page_to_pfn(page_list[0]);
746 for (i = 0; i < pinned; i++) {
747 unsigned long this_pfn = page_to_pfn(page_list[i]);
750 if (last_pfn && (this_pfn != last_pfn + 1)) {
751 /* Pin a contiguous chunk of memory */
752 csize = PFN_PHYS(last_pfn - map_pfn + 1);
753 ret = vhost_vdpa_map(v, iova, csize,
758 * Unpin the pages that are left unmapped
759 * from this point on in the current
760 * page_list. The remaining outstanding
761 * ones which may stride across several
762 * chunks will be covered in the common
763 * error path subsequently.
765 unpin_user_pages(&page_list[i],
778 cur_base += PFN_PHYS(pinned);
782 /* Pin the rest chunk */
783 ret = vhost_vdpa_map(v, iova, PFN_PHYS(last_pfn - map_pfn + 1),
784 PFN_PHYS(map_pfn), perm, NULL);
791 * Unpin the outstanding pages which are yet to be
792 * mapped but haven't due to vdpa_map() or
793 * pin_user_pages() failure.
795 * Mapped pages are accounted in vdpa_map(), hence
796 * the corresponding unpinning will be handled by
800 for (pfn = map_pfn; pfn <= last_pfn; pfn++)
801 unpin_user_page(pfn_to_page(pfn));
803 vhost_vdpa_unmap(v, start, size);
806 mmap_read_unlock(dev->mm);
808 free_page((unsigned long)page_list);
813 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
814 struct vhost_iotlb_msg *msg)
816 struct vhost_dev *dev = &v->vdev;
817 struct vdpa_device *vdpa = v->vdpa;
818 struct vhost_iotlb *iotlb = dev->iotlb;
820 if (msg->iova < v->range.first || !msg->size ||
821 msg->iova > U64_MAX - msg->size + 1 ||
822 msg->iova + msg->size - 1 > v->range.last)
825 if (vhost_iotlb_itree_first(iotlb, msg->iova,
826 msg->iova + msg->size - 1))
830 return vhost_vdpa_va_map(v, msg->iova, msg->size,
831 msg->uaddr, msg->perm);
833 return vhost_vdpa_pa_map(v, msg->iova, msg->size, msg->uaddr,
837 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev,
838 struct vhost_iotlb_msg *msg)
840 struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
841 struct vdpa_device *vdpa = v->vdpa;
842 const struct vdpa_config_ops *ops = vdpa->config;
845 mutex_lock(&dev->mutex);
847 r = vhost_dev_check_owner(dev);
852 case VHOST_IOTLB_UPDATE:
853 r = vhost_vdpa_process_iotlb_update(v, msg);
855 case VHOST_IOTLB_INVALIDATE:
856 vhost_vdpa_unmap(v, msg->iova, msg->size);
858 case VHOST_IOTLB_BATCH_BEGIN:
861 case VHOST_IOTLB_BATCH_END:
862 if (v->in_batch && ops->set_map)
863 ops->set_map(vdpa, dev->iotlb);
871 mutex_unlock(&dev->mutex);
876 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
877 struct iov_iter *from)
879 struct file *file = iocb->ki_filp;
880 struct vhost_vdpa *v = file->private_data;
881 struct vhost_dev *dev = &v->vdev;
883 return vhost_chr_write_iter(dev, from);
886 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
888 struct vdpa_device *vdpa = v->vdpa;
889 const struct vdpa_config_ops *ops = vdpa->config;
890 struct device *dma_dev = vdpa_get_dma_dev(vdpa);
891 struct bus_type *bus;
894 /* Device want to do DMA by itself */
895 if (ops->set_map || ops->dma_map)
902 if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY))
905 v->domain = iommu_domain_alloc(bus);
909 ret = iommu_attach_device(v->domain, dma_dev);
916 iommu_domain_free(v->domain);
920 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
922 struct vdpa_device *vdpa = v->vdpa;
923 struct device *dma_dev = vdpa_get_dma_dev(vdpa);
926 iommu_detach_device(v->domain, dma_dev);
927 iommu_domain_free(v->domain);
933 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
935 struct vdpa_iova_range *range = &v->range;
936 struct vdpa_device *vdpa = v->vdpa;
937 const struct vdpa_config_ops *ops = vdpa->config;
939 if (ops->get_iova_range) {
940 *range = ops->get_iova_range(vdpa);
941 } else if (v->domain && v->domain->geometry.force_aperture) {
942 range->first = v->domain->geometry.aperture_start;
943 range->last = v->domain->geometry.aperture_end;
946 range->last = ULLONG_MAX;
950 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
952 struct vhost_vdpa *v;
953 struct vhost_dev *dev;
954 struct vhost_virtqueue **vqs;
955 int nvqs, i, r, opened;
957 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
959 opened = atomic_cmpxchg(&v->opened, 0, 1);
964 r = vhost_vdpa_reset(v);
968 vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
975 for (i = 0; i < nvqs; i++) {
977 vqs[i]->handle_kick = handle_vq_kick;
979 vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
980 vhost_vdpa_process_iotlb_msg);
982 dev->iotlb = vhost_iotlb_alloc(0, 0);
988 r = vhost_vdpa_alloc_domain(v);
992 vhost_vdpa_set_iova_range(v);
994 filep->private_data = v;
999 vhost_dev_cleanup(&v->vdev);
1002 atomic_dec(&v->opened);
1006 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1010 for (i = 0; i < v->nvqs; i++)
1011 vhost_vdpa_unsetup_vq_irq(v, i);
1014 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1016 struct vhost_vdpa *v = filep->private_data;
1017 struct vhost_dev *d = &v->vdev;
1019 mutex_lock(&d->mutex);
1020 filep->private_data = NULL;
1021 vhost_vdpa_reset(v);
1022 vhost_dev_stop(&v->vdev);
1023 vhost_vdpa_iotlb_free(v);
1024 vhost_vdpa_free_domain(v);
1025 vhost_vdpa_config_put(v);
1026 vhost_vdpa_clean_irq(v);
1027 vhost_dev_cleanup(&v->vdev);
1029 mutex_unlock(&d->mutex);
1031 atomic_dec(&v->opened);
1032 complete(&v->completion);
1038 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1040 struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1041 struct vdpa_device *vdpa = v->vdpa;
1042 const struct vdpa_config_ops *ops = vdpa->config;
1043 struct vdpa_notification_area notify;
1044 struct vm_area_struct *vma = vmf->vma;
1045 u16 index = vma->vm_pgoff;
1047 notify = ops->get_vq_notification(vdpa, index);
1049 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1050 if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
1051 PFN_DOWN(notify.addr), PAGE_SIZE,
1053 return VM_FAULT_SIGBUS;
1055 return VM_FAULT_NOPAGE;
1058 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1059 .fault = vhost_vdpa_fault,
1062 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1064 struct vhost_vdpa *v = vma->vm_file->private_data;
1065 struct vdpa_device *vdpa = v->vdpa;
1066 const struct vdpa_config_ops *ops = vdpa->config;
1067 struct vdpa_notification_area notify;
1068 unsigned long index = vma->vm_pgoff;
1070 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1072 if ((vma->vm_flags & VM_SHARED) == 0)
1074 if (vma->vm_flags & VM_READ)
1078 if (!ops->get_vq_notification)
1081 /* To be safe and easily modelled by userspace, We only
1082 * support the doorbell which sits on the page boundary and
1083 * does not share the page with other registers.
1085 notify = ops->get_vq_notification(vdpa, index);
1086 if (notify.addr & (PAGE_SIZE - 1))
1088 if (vma->vm_end - vma->vm_start != notify.size)
1091 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1092 vma->vm_ops = &vhost_vdpa_vm_ops;
1095 #endif /* CONFIG_MMU */
1097 static const struct file_operations vhost_vdpa_fops = {
1098 .owner = THIS_MODULE,
1099 .open = vhost_vdpa_open,
1100 .release = vhost_vdpa_release,
1101 .write_iter = vhost_vdpa_chr_write_iter,
1102 .unlocked_ioctl = vhost_vdpa_unlocked_ioctl,
1104 .mmap = vhost_vdpa_mmap,
1105 #endif /* CONFIG_MMU */
1106 .compat_ioctl = compat_ptr_ioctl,
1109 static void vhost_vdpa_release_dev(struct device *device)
1111 struct vhost_vdpa *v =
1112 container_of(device, struct vhost_vdpa, dev);
1114 ida_simple_remove(&vhost_vdpa_ida, v->minor);
1119 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1121 const struct vdpa_config_ops *ops = vdpa->config;
1122 struct vhost_vdpa *v;
1126 v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1130 minor = ida_simple_get(&vhost_vdpa_ida, 0,
1131 VHOST_VDPA_DEV_MAX, GFP_KERNEL);
1137 atomic_set(&v->opened, 0);
1140 v->nvqs = vdpa->nvqs;
1141 v->virtio_id = ops->get_device_id(vdpa);
1143 device_initialize(&v->dev);
1144 v->dev.release = vhost_vdpa_release_dev;
1145 v->dev.parent = &vdpa->dev;
1146 v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1147 v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),
1154 r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1158 cdev_init(&v->cdev, &vhost_vdpa_fops);
1159 v->cdev.owner = THIS_MODULE;
1161 r = cdev_device_add(&v->cdev, &v->dev);
1165 init_completion(&v->completion);
1166 vdpa_set_drvdata(vdpa, v);
1171 put_device(&v->dev);
1175 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1177 struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1180 cdev_device_del(&v->cdev, &v->dev);
1183 opened = atomic_cmpxchg(&v->opened, 0, 1);
1186 wait_for_completion(&v->completion);
1189 put_device(&v->dev);
1192 static struct vdpa_driver vhost_vdpa_driver = {
1194 .name = "vhost_vdpa",
1196 .probe = vhost_vdpa_probe,
1197 .remove = vhost_vdpa_remove,
1200 static int __init vhost_vdpa_init(void)
1204 r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1207 goto err_alloc_chrdev;
1209 r = vdpa_register_driver(&vhost_vdpa_driver);
1211 goto err_vdpa_register_driver;
1215 err_vdpa_register_driver:
1216 unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1220 module_init(vhost_vdpa_init);
1222 static void __exit vhost_vdpa_exit(void)
1224 vdpa_unregister_driver(&vhost_vdpa_driver);
1225 unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1227 module_exit(vhost_vdpa_exit);
1229 MODULE_VERSION("0.0.1");
1230 MODULE_LICENSE("GPL v2");
1231 MODULE_AUTHOR("Intel Corporation");
1232 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");