1 // SPDX-License-Identifier: GPL-2.0-only
3 * VDUSE: vDPA Device in Userspace
5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
7 * Author: Xie Yongji <xieyongji@bytedance.com>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/slab.h>
17 #include <linux/wait.h>
18 #include <linux/dma-map-ops.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/uio.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <uapi/linux/vduse.h>
25 #include <uapi/linux/vdpa.h>
26 #include <uapi/linux/virtio_config.h>
27 #include <uapi/linux/virtio_ids.h>
28 #include <uapi/linux/virtio_blk.h>
29 #include <linux/mod_devicetable.h>
31 #include "iova_domain.h"
33 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>"
34 #define DRV_DESC "vDPA Device in Userspace"
35 #define DRV_LICENSE "GPL v2"
37 #define VDUSE_DEV_MAX (1U << MINORBITS)
38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
42 struct vduse_virtqueue {
49 struct vdpa_vq_state state;
54 struct eventfd_ctx *kickfd;
55 struct vdpa_callback cb;
56 struct work_struct inject;
57 struct work_struct kick;
63 struct vdpa_device vdpa;
64 struct vduse_dev *dev;
68 struct vduse_vdpa *vdev;
70 struct vduse_virtqueue *vqs;
71 struct vduse_iova_domain *domain;
77 wait_queue_head_t waitq;
78 struct list_head send_list;
79 struct list_head recv_list;
80 struct vdpa_callback config_cb;
81 struct work_struct inject;
99 struct vduse_dev_msg {
100 struct vduse_dev_request req;
101 struct vduse_dev_response resp;
102 struct list_head list;
103 wait_queue_head_t waitq;
107 struct vduse_control {
111 static DEFINE_MUTEX(vduse_lock);
112 static DEFINE_IDR(vduse_idr);
114 static dev_t vduse_major;
115 static struct class *vduse_class;
116 static struct cdev vduse_ctrl_cdev;
117 static struct cdev vduse_cdev;
118 static struct workqueue_struct *vduse_irq_wq;
120 static u32 allowed_device_id[] = {
124 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
126 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
131 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
133 struct vdpa_device *vdpa = dev_to_vdpa(dev);
135 return vdpa_to_vduse(vdpa);
138 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
141 struct vduse_dev_msg *msg;
143 list_for_each_entry(msg, head, list) {
144 if (msg->req.request_id == request_id) {
145 list_del(&msg->list);
153 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
155 struct vduse_dev_msg *msg = NULL;
157 if (!list_empty(head)) {
158 msg = list_first_entry(head, struct vduse_dev_msg, list);
159 list_del(&msg->list);
165 static void vduse_enqueue_msg(struct list_head *head,
166 struct vduse_dev_msg *msg)
168 list_add_tail(&msg->list, head);
171 static void vduse_dev_broken(struct vduse_dev *dev)
173 struct vduse_dev_msg *msg, *tmp;
175 if (unlikely(dev->broken))
178 list_splice_init(&dev->recv_list, &dev->send_list);
179 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
180 list_del(&msg->list);
182 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
183 wake_up(&msg->waitq);
186 wake_up(&dev->waitq);
189 static int vduse_dev_msg_sync(struct vduse_dev *dev,
190 struct vduse_dev_msg *msg)
194 if (unlikely(dev->broken))
197 init_waitqueue_head(&msg->waitq);
198 spin_lock(&dev->msg_lock);
199 if (unlikely(dev->broken)) {
200 spin_unlock(&dev->msg_lock);
203 msg->req.request_id = dev->msg_unique++;
204 vduse_enqueue_msg(&dev->send_list, msg);
205 wake_up(&dev->waitq);
206 spin_unlock(&dev->msg_lock);
207 if (dev->msg_timeout)
208 ret = wait_event_killable_timeout(msg->waitq, msg->completed,
209 (long)dev->msg_timeout * HZ);
211 ret = wait_event_killable(msg->waitq, msg->completed);
213 spin_lock(&dev->msg_lock);
214 if (!msg->completed) {
215 list_del(&msg->list);
216 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
217 /* Mark the device as malfunction when there is a timeout */
219 vduse_dev_broken(dev);
221 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
222 spin_unlock(&dev->msg_lock);
227 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
228 struct vduse_virtqueue *vq,
229 struct vdpa_vq_state_packed *packed)
231 struct vduse_dev_msg msg = { 0 };
234 msg.req.type = VDUSE_GET_VQ_STATE;
235 msg.req.vq_state.index = vq->index;
237 ret = vduse_dev_msg_sync(dev, &msg);
241 packed->last_avail_counter =
242 msg.resp.vq_state.packed.last_avail_counter & 0x0001;
243 packed->last_avail_idx =
244 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
245 packed->last_used_counter =
246 msg.resp.vq_state.packed.last_used_counter & 0x0001;
247 packed->last_used_idx =
248 msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
253 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
254 struct vduse_virtqueue *vq,
255 struct vdpa_vq_state_split *split)
257 struct vduse_dev_msg msg = { 0 };
260 msg.req.type = VDUSE_GET_VQ_STATE;
261 msg.req.vq_state.index = vq->index;
263 ret = vduse_dev_msg_sync(dev, &msg);
267 split->avail_index = msg.resp.vq_state.split.avail_index;
272 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
274 struct vduse_dev_msg msg = { 0 };
276 msg.req.type = VDUSE_SET_STATUS;
277 msg.req.s.status = status;
279 return vduse_dev_msg_sync(dev, &msg);
282 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
285 struct vduse_dev_msg msg = { 0 };
290 msg.req.type = VDUSE_UPDATE_IOTLB;
291 msg.req.iova.start = start;
292 msg.req.iova.last = last;
294 return vduse_dev_msg_sync(dev, &msg);
297 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
299 struct file *file = iocb->ki_filp;
300 struct vduse_dev *dev = file->private_data;
301 struct vduse_dev_msg *msg;
302 int size = sizeof(struct vduse_dev_request);
305 if (iov_iter_count(to) < size)
308 spin_lock(&dev->msg_lock);
310 msg = vduse_dequeue_msg(&dev->send_list);
315 if (file->f_flags & O_NONBLOCK)
318 spin_unlock(&dev->msg_lock);
319 ret = wait_event_interruptible_exclusive(dev->waitq,
320 !list_empty(&dev->send_list));
324 spin_lock(&dev->msg_lock);
326 spin_unlock(&dev->msg_lock);
327 ret = copy_to_iter(&msg->req, size, to);
328 spin_lock(&dev->msg_lock);
331 vduse_enqueue_msg(&dev->send_list, msg);
334 vduse_enqueue_msg(&dev->recv_list, msg);
336 spin_unlock(&dev->msg_lock);
341 static bool is_mem_zero(const char *ptr, int size)
345 for (i = 0; i < size; i++) {
352 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
354 struct file *file = iocb->ki_filp;
355 struct vduse_dev *dev = file->private_data;
356 struct vduse_dev_response resp;
357 struct vduse_dev_msg *msg;
360 ret = copy_from_iter(&resp, sizeof(resp), from);
361 if (ret != sizeof(resp))
364 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
367 spin_lock(&dev->msg_lock);
368 msg = vduse_find_msg(&dev->recv_list, resp.request_id);
374 memcpy(&msg->resp, &resp, sizeof(resp));
376 wake_up(&msg->waitq);
378 spin_unlock(&dev->msg_lock);
383 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
385 struct vduse_dev *dev = file->private_data;
388 poll_wait(file, &dev->waitq, wait);
390 spin_lock(&dev->msg_lock);
392 if (unlikely(dev->broken))
394 if (!list_empty(&dev->send_list))
395 mask |= EPOLLIN | EPOLLRDNORM;
396 if (!list_empty(&dev->recv_list))
397 mask |= EPOLLOUT | EPOLLWRNORM;
399 spin_unlock(&dev->msg_lock);
404 static void vduse_dev_reset(struct vduse_dev *dev)
407 struct vduse_iova_domain *domain = dev->domain;
409 /* The coherent mappings are handled in vduse_dev_free_coherent() */
410 if (domain->bounce_map)
411 vduse_domain_reset_bounce_map(domain);
414 dev->driver_features = 0;
416 spin_lock(&dev->irq_lock);
417 dev->config_cb.callback = NULL;
418 dev->config_cb.private = NULL;
419 spin_unlock(&dev->irq_lock);
420 flush_work(&dev->inject);
422 for (i = 0; i < dev->vq_num; i++) {
423 struct vduse_virtqueue *vq = &dev->vqs[i];
430 memset(&vq->state, 0, sizeof(vq->state));
432 spin_lock(&vq->kick_lock);
435 eventfd_ctx_put(vq->kickfd);
437 spin_unlock(&vq->kick_lock);
439 spin_lock(&vq->irq_lock);
440 vq->cb.callback = NULL;
441 vq->cb.private = NULL;
442 spin_unlock(&vq->irq_lock);
443 flush_work(&vq->inject);
444 flush_work(&vq->kick);
448 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
449 u64 desc_area, u64 driver_area,
452 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
453 struct vduse_virtqueue *vq = &dev->vqs[idx];
455 vq->desc_addr = desc_area;
456 vq->driver_addr = driver_area;
457 vq->device_addr = device_area;
462 static void vduse_vq_kick(struct vduse_virtqueue *vq)
464 spin_lock(&vq->kick_lock);
469 eventfd_signal(vq->kickfd, 1);
473 spin_unlock(&vq->kick_lock);
476 static void vduse_vq_kick_work(struct work_struct *work)
478 struct vduse_virtqueue *vq = container_of(work,
479 struct vduse_virtqueue, kick);
484 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
486 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
487 struct vduse_virtqueue *vq = &dev->vqs[idx];
489 if (!eventfd_signal_allowed()) {
490 schedule_work(&vq->kick);
496 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
497 struct vdpa_callback *cb)
499 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
500 struct vduse_virtqueue *vq = &dev->vqs[idx];
502 spin_lock(&vq->irq_lock);
503 vq->cb.callback = cb->callback;
504 vq->cb.private = cb->private;
505 spin_unlock(&vq->irq_lock);
508 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
510 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
511 struct vduse_virtqueue *vq = &dev->vqs[idx];
516 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
519 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
520 struct vduse_virtqueue *vq = &dev->vqs[idx];
525 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
527 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
528 struct vduse_virtqueue *vq = &dev->vqs[idx];
533 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
534 const struct vdpa_vq_state *state)
536 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
537 struct vduse_virtqueue *vq = &dev->vqs[idx];
539 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
540 vq->state.packed.last_avail_counter =
541 state->packed.last_avail_counter;
542 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
543 vq->state.packed.last_used_counter =
544 state->packed.last_used_counter;
545 vq->state.packed.last_used_idx = state->packed.last_used_idx;
547 vq->state.split.avail_index = state->split.avail_index;
552 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
553 struct vdpa_vq_state *state)
555 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
556 struct vduse_virtqueue *vq = &dev->vqs[idx];
558 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
559 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
561 return vduse_dev_get_vq_state_split(dev, vq, &state->split);
564 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
566 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
568 return dev->vq_align;
571 static u64 vduse_vdpa_get_features(struct vdpa_device *vdpa)
573 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
575 return dev->device_features;
578 static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features)
580 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
582 dev->driver_features = features;
586 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
587 struct vdpa_callback *cb)
589 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
591 spin_lock(&dev->irq_lock);
592 dev->config_cb.callback = cb->callback;
593 dev->config_cb.private = cb->private;
594 spin_unlock(&dev->irq_lock);
597 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
599 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
603 for (i = 0; i < dev->vq_num; i++)
604 if (num_max < dev->vqs[i].num_max)
605 num_max = dev->vqs[i].num_max;
610 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
612 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
614 return dev->device_id;
617 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
619 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
621 return dev->vendor_id;
624 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
626 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
631 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
633 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
635 if (vduse_dev_set_status(dev, status))
638 dev->status = status;
641 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
643 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
645 return dev->config_size;
648 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
649 void *buf, unsigned int len)
651 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
653 if (len > dev->config_size - offset)
656 memcpy(buf, dev->config + offset, len);
659 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
660 const void *buf, unsigned int len)
662 /* Now we only support read-only configuration space */
665 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
667 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
668 int ret = vduse_dev_set_status(dev, 0);
670 vduse_dev_reset(dev);
675 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
677 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
679 return dev->generation;
682 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
683 struct vhost_iotlb *iotlb)
685 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
688 ret = vduse_domain_set_map(dev->domain, iotlb);
692 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
694 vduse_domain_clear_map(dev->domain, iotlb);
701 static void vduse_vdpa_free(struct vdpa_device *vdpa)
703 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
708 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
709 .set_vq_address = vduse_vdpa_set_vq_address,
710 .kick_vq = vduse_vdpa_kick_vq,
711 .set_vq_cb = vduse_vdpa_set_vq_cb,
712 .set_vq_num = vduse_vdpa_set_vq_num,
713 .set_vq_ready = vduse_vdpa_set_vq_ready,
714 .get_vq_ready = vduse_vdpa_get_vq_ready,
715 .set_vq_state = vduse_vdpa_set_vq_state,
716 .get_vq_state = vduse_vdpa_get_vq_state,
717 .get_vq_align = vduse_vdpa_get_vq_align,
718 .get_features = vduse_vdpa_get_features,
719 .set_features = vduse_vdpa_set_features,
720 .set_config_cb = vduse_vdpa_set_config_cb,
721 .get_vq_num_max = vduse_vdpa_get_vq_num_max,
722 .get_device_id = vduse_vdpa_get_device_id,
723 .get_vendor_id = vduse_vdpa_get_vendor_id,
724 .get_status = vduse_vdpa_get_status,
725 .set_status = vduse_vdpa_set_status,
726 .get_config_size = vduse_vdpa_get_config_size,
727 .get_config = vduse_vdpa_get_config,
728 .set_config = vduse_vdpa_set_config,
729 .get_generation = vduse_vdpa_get_generation,
730 .reset = vduse_vdpa_reset,
731 .set_map = vduse_vdpa_set_map,
732 .free = vduse_vdpa_free,
735 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
736 unsigned long offset, size_t size,
737 enum dma_data_direction dir,
740 struct vduse_dev *vdev = dev_to_vduse(dev);
741 struct vduse_iova_domain *domain = vdev->domain;
743 return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
746 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
747 size_t size, enum dma_data_direction dir,
750 struct vduse_dev *vdev = dev_to_vduse(dev);
751 struct vduse_iova_domain *domain = vdev->domain;
753 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
756 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
757 dma_addr_t *dma_addr, gfp_t flag,
760 struct vduse_dev *vdev = dev_to_vduse(dev);
761 struct vduse_iova_domain *domain = vdev->domain;
765 *dma_addr = DMA_MAPPING_ERROR;
766 addr = vduse_domain_alloc_coherent(domain, size,
767 (dma_addr_t *)&iova, flag, attrs);
771 *dma_addr = (dma_addr_t)iova;
776 static void vduse_dev_free_coherent(struct device *dev, size_t size,
777 void *vaddr, dma_addr_t dma_addr,
780 struct vduse_dev *vdev = dev_to_vduse(dev);
781 struct vduse_iova_domain *domain = vdev->domain;
783 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
786 static size_t vduse_dev_max_mapping_size(struct device *dev)
788 struct vduse_dev *vdev = dev_to_vduse(dev);
789 struct vduse_iova_domain *domain = vdev->domain;
791 return domain->bounce_size;
794 static const struct dma_map_ops vduse_dev_dma_ops = {
795 .map_page = vduse_dev_map_page,
796 .unmap_page = vduse_dev_unmap_page,
797 .alloc = vduse_dev_alloc_coherent,
798 .free = vduse_dev_free_coherent,
799 .max_mapping_size = vduse_dev_max_mapping_size,
802 static unsigned int perm_to_file_flags(u8 perm)
804 unsigned int flags = 0;
807 case VDUSE_ACCESS_WO:
810 case VDUSE_ACCESS_RO:
813 case VDUSE_ACCESS_RW:
817 WARN(1, "invalidate vhost IOTLB permission\n");
824 static int vduse_kickfd_setup(struct vduse_dev *dev,
825 struct vduse_vq_eventfd *eventfd)
827 struct eventfd_ctx *ctx = NULL;
828 struct vduse_virtqueue *vq;
831 if (eventfd->index >= dev->vq_num)
834 index = array_index_nospec(eventfd->index, dev->vq_num);
835 vq = &dev->vqs[index];
836 if (eventfd->fd >= 0) {
837 ctx = eventfd_ctx_fdget(eventfd->fd);
840 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
843 spin_lock(&vq->kick_lock);
845 eventfd_ctx_put(vq->kickfd);
847 if (vq->ready && vq->kicked && vq->kickfd) {
848 eventfd_signal(vq->kickfd, 1);
851 spin_unlock(&vq->kick_lock);
856 static bool vduse_dev_is_ready(struct vduse_dev *dev)
860 for (i = 0; i < dev->vq_num; i++)
861 if (!dev->vqs[i].num_max)
867 static void vduse_dev_irq_inject(struct work_struct *work)
869 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
871 spin_lock_irq(&dev->irq_lock);
872 if (dev->config_cb.callback)
873 dev->config_cb.callback(dev->config_cb.private);
874 spin_unlock_irq(&dev->irq_lock);
877 static void vduse_vq_irq_inject(struct work_struct *work)
879 struct vduse_virtqueue *vq = container_of(work,
880 struct vduse_virtqueue, inject);
882 spin_lock_irq(&vq->irq_lock);
883 if (vq->ready && vq->cb.callback)
884 vq->cb.callback(vq->cb.private);
885 spin_unlock_irq(&vq->irq_lock);
888 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
891 struct vduse_dev *dev = file->private_data;
892 void __user *argp = (void __user *)arg;
895 if (unlikely(dev->broken))
899 case VDUSE_IOTLB_GET_FD: {
900 struct vduse_iotlb_entry entry;
901 struct vhost_iotlb_map *map;
902 struct vdpa_map_file *map_file;
903 struct vduse_iova_domain *domain = dev->domain;
904 struct file *f = NULL;
907 if (copy_from_user(&entry, argp, sizeof(entry)))
911 if (entry.start > entry.last)
914 spin_lock(&domain->iotlb_lock);
915 map = vhost_iotlb_itree_first(domain->iotlb,
916 entry.start, entry.last);
918 map_file = (struct vdpa_map_file *)map->opaque;
919 f = get_file(map_file->file);
920 entry.offset = map_file->offset;
921 entry.start = map->start;
922 entry.last = map->last;
923 entry.perm = map->perm;
925 spin_unlock(&domain->iotlb_lock);
931 if (copy_to_user(argp, &entry, sizeof(entry))) {
935 ret = receive_fd(f, perm_to_file_flags(entry.perm));
939 case VDUSE_DEV_GET_FEATURES:
941 * Just mirror what driver wrote here.
942 * The driver is expected to check FEATURE_OK later.
944 ret = put_user(dev->driver_features, (u64 __user *)argp);
946 case VDUSE_DEV_SET_CONFIG: {
947 struct vduse_config_data config;
948 unsigned long size = offsetof(struct vduse_config_data,
952 if (copy_from_user(&config, argp, size))
956 if (config.length == 0 ||
957 config.length > dev->config_size - config.offset)
961 if (copy_from_user(dev->config + config.offset, argp + size,
968 case VDUSE_DEV_INJECT_CONFIG_IRQ:
970 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
974 queue_work(vduse_irq_wq, &dev->inject);
976 case VDUSE_VQ_SETUP: {
977 struct vduse_vq_config config;
981 if (copy_from_user(&config, argp, sizeof(config)))
985 if (config.index >= dev->vq_num)
988 if (!is_mem_zero((const char *)config.reserved,
989 sizeof(config.reserved)))
992 index = array_index_nospec(config.index, dev->vq_num);
993 dev->vqs[index].num_max = config.max_size;
997 case VDUSE_VQ_GET_INFO: {
998 struct vduse_vq_info vq_info;
999 struct vduse_virtqueue *vq;
1003 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1007 if (vq_info.index >= dev->vq_num)
1010 index = array_index_nospec(vq_info.index, dev->vq_num);
1011 vq = &dev->vqs[index];
1012 vq_info.desc_addr = vq->desc_addr;
1013 vq_info.driver_addr = vq->driver_addr;
1014 vq_info.device_addr = vq->device_addr;
1015 vq_info.num = vq->num;
1017 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1018 vq_info.packed.last_avail_counter =
1019 vq->state.packed.last_avail_counter;
1020 vq_info.packed.last_avail_idx =
1021 vq->state.packed.last_avail_idx;
1022 vq_info.packed.last_used_counter =
1023 vq->state.packed.last_used_counter;
1024 vq_info.packed.last_used_idx =
1025 vq->state.packed.last_used_idx;
1027 vq_info.split.avail_index =
1028 vq->state.split.avail_index;
1030 vq_info.ready = vq->ready;
1033 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1039 case VDUSE_VQ_SETUP_KICKFD: {
1040 struct vduse_vq_eventfd eventfd;
1043 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1046 ret = vduse_kickfd_setup(dev, &eventfd);
1049 case VDUSE_VQ_INJECT_IRQ: {
1053 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1057 if (get_user(index, (u32 __user *)argp))
1061 if (index >= dev->vq_num)
1065 index = array_index_nospec(index, dev->vq_num);
1066 queue_work(vduse_irq_wq, &dev->vqs[index].inject);
1077 static int vduse_dev_release(struct inode *inode, struct file *file)
1079 struct vduse_dev *dev = file->private_data;
1081 spin_lock(&dev->msg_lock);
1082 /* Make sure the inflight messages can processed after reconncection */
1083 list_splice_init(&dev->recv_list, &dev->send_list);
1084 spin_unlock(&dev->msg_lock);
1085 dev->connected = false;
1090 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1092 struct vduse_dev *dev;
1094 mutex_lock(&vduse_lock);
1095 dev = idr_find(&vduse_idr, minor);
1096 mutex_unlock(&vduse_lock);
1101 static int vduse_dev_open(struct inode *inode, struct file *file)
1104 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1110 mutex_lock(&dev->lock);
1115 dev->connected = true;
1116 file->private_data = dev;
1118 mutex_unlock(&dev->lock);
1123 static const struct file_operations vduse_dev_fops = {
1124 .owner = THIS_MODULE,
1125 .open = vduse_dev_open,
1126 .release = vduse_dev_release,
1127 .read_iter = vduse_dev_read_iter,
1128 .write_iter = vduse_dev_write_iter,
1129 .poll = vduse_dev_poll,
1130 .unlocked_ioctl = vduse_dev_ioctl,
1131 .compat_ioctl = compat_ptr_ioctl,
1132 .llseek = noop_llseek,
1135 static struct vduse_dev *vduse_dev_create(void)
1137 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1142 mutex_init(&dev->lock);
1143 spin_lock_init(&dev->msg_lock);
1144 INIT_LIST_HEAD(&dev->send_list);
1145 INIT_LIST_HEAD(&dev->recv_list);
1146 spin_lock_init(&dev->irq_lock);
1148 INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1149 init_waitqueue_head(&dev->waitq);
1154 static void vduse_dev_destroy(struct vduse_dev *dev)
1159 static struct vduse_dev *vduse_find_dev(const char *name)
1161 struct vduse_dev *dev;
1164 idr_for_each_entry(&vduse_idr, dev, id)
1165 if (!strcmp(dev->name, name))
1171 static int vduse_destroy_dev(char *name)
1173 struct vduse_dev *dev = vduse_find_dev(name);
1178 mutex_lock(&dev->lock);
1179 if (dev->vdev || dev->connected) {
1180 mutex_unlock(&dev->lock);
1183 dev->connected = true;
1184 mutex_unlock(&dev->lock);
1186 vduse_dev_reset(dev);
1187 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1188 idr_remove(&vduse_idr, dev->minor);
1189 kvfree(dev->config);
1191 vduse_domain_destroy(dev->domain);
1193 vduse_dev_destroy(dev);
1194 module_put(THIS_MODULE);
1199 static bool device_is_allowed(u32 device_id)
1203 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1204 if (allowed_device_id[i] == device_id)
1210 static bool features_is_valid(u64 features)
1212 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1215 /* Now we only support read-only configuration space */
1216 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1222 static bool vduse_validate_config(struct vduse_dev_config *config)
1224 if (!is_mem_zero((const char *)config->reserved,
1225 sizeof(config->reserved)))
1228 if (config->vq_align > PAGE_SIZE)
1231 if (config->config_size > PAGE_SIZE)
1234 if (!device_is_allowed(config->device_id))
1237 if (!features_is_valid(config->features))
1243 static ssize_t msg_timeout_show(struct device *device,
1244 struct device_attribute *attr, char *buf)
1246 struct vduse_dev *dev = dev_get_drvdata(device);
1248 return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1251 static ssize_t msg_timeout_store(struct device *device,
1252 struct device_attribute *attr,
1253 const char *buf, size_t count)
1255 struct vduse_dev *dev = dev_get_drvdata(device);
1258 ret = kstrtouint(buf, 10, &dev->msg_timeout);
1265 static DEVICE_ATTR_RW(msg_timeout);
1267 static struct attribute *vduse_dev_attrs[] = {
1268 &dev_attr_msg_timeout.attr,
1272 ATTRIBUTE_GROUPS(vduse_dev);
1274 static int vduse_create_dev(struct vduse_dev_config *config,
1275 void *config_buf, u64 api_version)
1278 struct vduse_dev *dev;
1281 if (vduse_find_dev(config->name))
1285 dev = vduse_dev_create();
1289 dev->api_version = api_version;
1290 dev->device_features = config->features;
1291 dev->device_id = config->device_id;
1292 dev->vendor_id = config->vendor_id;
1293 dev->name = kstrdup(config->name, GFP_KERNEL);
1297 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1302 dev->config = config_buf;
1303 dev->config_size = config->config_size;
1304 dev->vq_align = config->vq_align;
1305 dev->vq_num = config->vq_num;
1306 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1310 for (i = 0; i < dev->vq_num; i++) {
1311 dev->vqs[i].index = i;
1312 INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
1313 INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
1314 spin_lock_init(&dev->vqs[i].kick_lock);
1315 spin_lock_init(&dev->vqs[i].irq_lock);
1318 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1323 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1324 dev->dev = device_create(vduse_class, NULL,
1325 MKDEV(MAJOR(vduse_major), dev->minor),
1326 dev, "%s", config->name);
1327 if (IS_ERR(dev->dev)) {
1328 ret = PTR_ERR(dev->dev);
1331 __module_get(THIS_MODULE);
1335 idr_remove(&vduse_idr, dev->minor);
1339 vduse_domain_destroy(dev->domain);
1343 vduse_dev_destroy(dev);
1349 static long vduse_ioctl(struct file *file, unsigned int cmd,
1353 void __user *argp = (void __user *)arg;
1354 struct vduse_control *control = file->private_data;
1356 mutex_lock(&vduse_lock);
1358 case VDUSE_GET_API_VERSION:
1359 ret = put_user(control->api_version, (u64 __user *)argp);
1361 case VDUSE_SET_API_VERSION: {
1365 if (get_user(api_version, (u64 __user *)argp))
1369 if (api_version > VDUSE_API_VERSION)
1373 control->api_version = api_version;
1376 case VDUSE_CREATE_DEV: {
1377 struct vduse_dev_config config;
1378 unsigned long size = offsetof(struct vduse_dev_config, config);
1382 if (copy_from_user(&config, argp, size))
1386 if (vduse_validate_config(&config) == false)
1389 buf = vmemdup_user(argp + size, config.config_size);
1394 config.name[VDUSE_NAME_MAX - 1] = '\0';
1395 ret = vduse_create_dev(&config, buf, control->api_version);
1398 case VDUSE_DESTROY_DEV: {
1399 char name[VDUSE_NAME_MAX];
1402 if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1405 name[VDUSE_NAME_MAX - 1] = '\0';
1406 ret = vduse_destroy_dev(name);
1413 mutex_unlock(&vduse_lock);
1418 static int vduse_release(struct inode *inode, struct file *file)
1420 struct vduse_control *control = file->private_data;
1426 static int vduse_open(struct inode *inode, struct file *file)
1428 struct vduse_control *control;
1430 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1434 control->api_version = VDUSE_API_VERSION;
1435 file->private_data = control;
1440 static const struct file_operations vduse_ctrl_fops = {
1441 .owner = THIS_MODULE,
1443 .release = vduse_release,
1444 .unlocked_ioctl = vduse_ioctl,
1445 .compat_ioctl = compat_ptr_ioctl,
1446 .llseek = noop_llseek,
1449 static char *vduse_devnode(struct device *dev, umode_t *mode)
1451 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1454 static void vduse_mgmtdev_release(struct device *dev)
1458 static struct device vduse_mgmtdev = {
1459 .init_name = "vduse",
1460 .release = vduse_mgmtdev_release,
1463 static struct vdpa_mgmt_dev mgmt_dev;
1465 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1467 struct vduse_vdpa *vdev;
1473 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1474 &vduse_vdpa_config_ops, name, true);
1476 return PTR_ERR(vdev);
1480 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1481 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1483 put_device(&vdev->vdpa.dev);
1486 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1487 vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1488 vdev->vdpa.mdev = &mgmt_dev;
1493 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
1495 struct vduse_dev *dev;
1498 mutex_lock(&vduse_lock);
1499 dev = vduse_find_dev(name);
1500 if (!dev || !vduse_dev_is_ready(dev)) {
1501 mutex_unlock(&vduse_lock);
1504 ret = vduse_dev_init_vdpa(dev, name);
1505 mutex_unlock(&vduse_lock);
1509 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1511 put_device(&dev->vdev->vdpa.dev);
1518 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
1520 _vdpa_unregister_device(dev);
1523 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
1524 .dev_add = vdpa_dev_add,
1525 .dev_del = vdpa_dev_del,
1528 static struct virtio_device_id id_table[] = {
1529 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1533 static struct vdpa_mgmt_dev mgmt_dev = {
1534 .device = &vduse_mgmtdev,
1535 .id_table = id_table,
1536 .ops = &vdpa_dev_mgmtdev_ops,
1539 static int vduse_mgmtdev_init(void)
1543 ret = device_register(&vduse_mgmtdev);
1547 ret = vdpa_mgmtdev_register(&mgmt_dev);
1553 device_unregister(&vduse_mgmtdev);
1557 static void vduse_mgmtdev_exit(void)
1559 vdpa_mgmtdev_unregister(&mgmt_dev);
1560 device_unregister(&vduse_mgmtdev);
1563 static int vduse_init(void)
1568 vduse_class = class_create(THIS_MODULE, "vduse");
1569 if (IS_ERR(vduse_class))
1570 return PTR_ERR(vduse_class);
1572 vduse_class->devnode = vduse_devnode;
1573 vduse_class->dev_groups = vduse_dev_groups;
1575 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
1577 goto err_chardev_region;
1579 /* /dev/vduse/control */
1580 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
1581 vduse_ctrl_cdev.owner = THIS_MODULE;
1582 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
1586 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
1592 /* /dev/vduse/$DEVICE */
1593 cdev_init(&vduse_cdev, &vduse_dev_fops);
1594 vduse_cdev.owner = THIS_MODULE;
1595 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
1600 vduse_irq_wq = alloc_workqueue("vduse-irq",
1601 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
1602 if (!vduse_irq_wq) {
1607 ret = vduse_domain_init();
1611 ret = vduse_mgmtdev_init();
1617 vduse_domain_exit();
1619 destroy_workqueue(vduse_irq_wq);
1621 cdev_del(&vduse_cdev);
1623 device_destroy(vduse_class, vduse_major);
1625 cdev_del(&vduse_ctrl_cdev);
1627 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1629 class_destroy(vduse_class);
1632 module_init(vduse_init);
1634 static void vduse_exit(void)
1636 vduse_mgmtdev_exit();
1637 vduse_domain_exit();
1638 destroy_workqueue(vduse_irq_wq);
1639 cdev_del(&vduse_cdev);
1640 device_destroy(vduse_class, vduse_major);
1641 cdev_del(&vduse_ctrl_cdev);
1642 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1643 class_destroy(vduse_class);
1645 module_exit(vduse_exit);
1647 MODULE_LICENSE(DRV_LICENSE);
1648 MODULE_AUTHOR(DRV_AUTHOR);
1649 MODULE_DESCRIPTION(DRV_DESC);