drivers/vdpa/vdpa_user/vduse_dev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VDUSE: vDPA Device in Userspace
   4  *
   5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
   6  *
   7  * Author: Xie Yongji <xieyongji@bytedance.com>
   8  *
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/cdev.h>
  14 #include <linux/device.h>
  15 #include <linux/eventfd.h>
  16 #include <linux/slab.h>
  17 #include <linux/wait.h>
  18 #include <linux/dma-map-ops.h>
  19 #include <linux/poll.h>
  20 #include <linux/file.h>
  21 #include <linux/uio.h>
  22 #include <linux/vdpa.h>
  23 #include <linux/nospec.h>
  24 #include <uapi/linux/vduse.h>
  25 #include <uapi/linux/vdpa.h>
  26 #include <uapi/linux/virtio_config.h>
  27 #include <uapi/linux/virtio_ids.h>
  28 #include <uapi/linux/virtio_blk.h>
  29 #include <linux/mod_devicetable.h>
  30
  31 #include "iova_domain.h"
  32
  33 #define DRV_AUTHOR   "Yongji Xie <xieyongji@bytedance.com>"
  34 #define DRV_DESC     "vDPA Device in Userspace"
  35 #define DRV_LICENSE  "GPL v2"
  36
  37 #define VDUSE_DEV_MAX (1U << MINORBITS)
  38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
  39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
  40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
  41
  42 struct vduse_virtqueue {
  43         u16 index;
  44         u16 num_max;
  45         u32 num;
  46         u64 desc_addr;
  47         u64 driver_addr;
  48         u64 device_addr;
  49         struct vdpa_vq_state state;
  50         bool ready;
  51         bool kicked;
  52         spinlock_t kick_lock;
  53         spinlock_t irq_lock;
  54         struct eventfd_ctx *kickfd;
  55         struct vdpa_callback cb;
  56         struct work_struct inject;
  57         struct work_struct kick;
  58 };
  59
  60 struct vduse_dev;
  61
  62 struct vduse_vdpa {
  63         struct vdpa_device vdpa;
  64         struct vduse_dev *dev;
  65 };
  66
  67 struct vduse_dev {
  68         struct vduse_vdpa *vdev;
  69         struct device *dev;
  70         struct vduse_virtqueue *vqs;
  71         struct vduse_iova_domain *domain;
  72         char *name;
  73         struct mutex lock;
  74         spinlock_t msg_lock;
  75         u64 msg_unique;
  76         u32 msg_timeout;
  77         wait_queue_head_t waitq;
  78         struct list_head send_list;
  79         struct list_head recv_list;
  80         struct vdpa_callback config_cb;
  81         struct work_struct inject;
  82         spinlock_t irq_lock;
  83         int minor;
  84         bool broken;
  85         bool connected;
  86         u64 api_version;
  87         u64 device_features;
  88         u64 driver_features;
  89         u32 device_id;
  90         u32 vendor_id;
  91         u32 generation;
  92         u32 config_size;
  93         void *config;
  94         u8 status;
  95         u32 vq_num;
  96         u32 vq_align;
  97 };
  98
  99 struct vduse_dev_msg {
 100         struct vduse_dev_request req;
 101         struct vduse_dev_response resp;
 102         struct list_head list;
 103         wait_queue_head_t waitq;
 104         bool completed;
 105 };
 106
 107 struct vduse_control {
 108         u64 api_version;
 109 };
 110
 111 static DEFINE_MUTEX(vduse_lock);
 112 static DEFINE_IDR(vduse_idr);
 113
 114 static dev_t vduse_major;
 115 static struct class *vduse_class;
 116 static struct cdev vduse_ctrl_cdev;
 117 static struct cdev vduse_cdev;
 118 static struct workqueue_struct *vduse_irq_wq;
 119
 120 static u32 allowed_device_id[] = {
 121         VIRTIO_ID_BLOCK,
 122 };
 123
 124 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
 125 {
 126         struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
 127
 128         return vdev->dev;
 129 }
 130
 131 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
 132 {
 133         struct vdpa_device *vdpa = dev_to_vdpa(dev);
 134
 135         return vdpa_to_vduse(vdpa);
 136 }
 137
 138 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
 139                                             uint32_t request_id)
 140 {
 141         struct vduse_dev_msg *msg;
 142
 143         list_for_each_entry(msg, head, list) {
 144                 if (msg->req.request_id == request_id) {
 145                         list_del(&msg->list);
 146                         return msg;
 147                 }
 148         }
 149
 150         return NULL;
 151 }
 152
 153 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
 154 {
 155         struct vduse_dev_msg *msg = NULL;
 156
 157         if (!list_empty(head)) {
 158                 msg = list_first_entry(head, struct vduse_dev_msg, list);
 159                 list_del(&msg->list);
 160         }
 161
 162         return msg;
 163 }
 164
 165 static void vduse_enqueue_msg(struct list_head *head,
 166                               struct vduse_dev_msg *msg)
 167 {
 168         list_add_tail(&msg->list, head);
 169 }
 170
 171 static void vduse_dev_broken(struct vduse_dev *dev)
 172 {
 173         struct vduse_dev_msg *msg, *tmp;
 174
 175         if (unlikely(dev->broken))
 176                 return;
 177
 178         list_splice_init(&dev->recv_list, &dev->send_list);
 179         list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
 180                 list_del(&msg->list);
 181                 msg->completed = 1;
 182                 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
 183                 wake_up(&msg->waitq);
 184         }
 185         dev->broken = true;
 186         wake_up(&dev->waitq);
 187 }
 188
 189 static int vduse_dev_msg_sync(struct vduse_dev *dev,
 190                               struct vduse_dev_msg *msg)
 191 {
 192         int ret;
 193
 194         if (unlikely(dev->broken))
 195                 return -EIO;
 196
 197         init_waitqueue_head(&msg->waitq);
 198         spin_lock(&dev->msg_lock);
 199         if (unlikely(dev->broken)) {
 200                 spin_unlock(&dev->msg_lock);
 201                 return -EIO;
 202         }
 203         msg->req.request_id = dev->msg_unique++;
 204         vduse_enqueue_msg(&dev->send_list, msg);
 205         wake_up(&dev->waitq);
 206         spin_unlock(&dev->msg_lock);
 207         if (dev->msg_timeout)
 208                 ret = wait_event_killable_timeout(msg->waitq, msg->completed,
 209                                                   (long)dev->msg_timeout * HZ);
 210         else
 211                 ret = wait_event_killable(msg->waitq, msg->completed);
 212
 213         spin_lock(&dev->msg_lock);
 214         if (!msg->completed) {
 215                 list_del(&msg->list);
 216                 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
 217                 /* Mark the device as malfunction when there is a timeout */
 218                 if (!ret)
 219                         vduse_dev_broken(dev);
 220         }
 221         ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
 222         spin_unlock(&dev->msg_lock);
 223
 224         return ret;
 225 }
 226
 227 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
 228                                          struct vduse_virtqueue *vq,
 229                                          struct vdpa_vq_state_packed *packed)
 230 {
 231         struct vduse_dev_msg msg = { 0 };
 232         int ret;
 233
 234         msg.req.type = VDUSE_GET_VQ_STATE;
 235         msg.req.vq_state.index = vq->index;
 236
 237         ret = vduse_dev_msg_sync(dev, &msg);
 238         if (ret)
 239                 return ret;
 240
 241         packed->last_avail_counter =
 242                         msg.resp.vq_state.packed.last_avail_counter & 0x0001;
 243         packed->last_avail_idx =
 244                         msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
 245         packed->last_used_counter =
 246                         msg.resp.vq_state.packed.last_used_counter & 0x0001;
 247         packed->last_used_idx =
 248                         msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
 249
 250         return 0;
 251 }
 252
 253 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
 254                                         struct vduse_virtqueue *vq,
 255                                         struct vdpa_vq_state_split *split)
 256 {
 257         struct vduse_dev_msg msg = { 0 };
 258         int ret;
 259
 260         msg.req.type = VDUSE_GET_VQ_STATE;
 261         msg.req.vq_state.index = vq->index;
 262
 263         ret = vduse_dev_msg_sync(dev, &msg);
 264         if (ret)
 265                 return ret;
 266
 267         split->avail_index = msg.resp.vq_state.split.avail_index;
 268
 269         return 0;
 270 }
 271
 272 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
 273 {
 274         struct vduse_dev_msg msg = { 0 };
 275
 276         msg.req.type = VDUSE_SET_STATUS;
 277         msg.req.s.status = status;
 278
 279         return vduse_dev_msg_sync(dev, &msg);
 280 }
 281
 282 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
 283                                   u64 start, u64 last)
 284 {
 285         struct vduse_dev_msg msg = { 0 };
 286
 287         if (last < start)
 288                 return -EINVAL;
 289
 290         msg.req.type = VDUSE_UPDATE_IOTLB;
 291         msg.req.iova.start = start;
 292         msg.req.iova.last = last;
 293
 294         return vduse_dev_msg_sync(dev, &msg);
 295 }
 296
 297 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
 298 {
 299         struct file *file = iocb->ki_filp;
 300         struct vduse_dev *dev = file->private_data;
 301         struct vduse_dev_msg *msg;
 302         int size = sizeof(struct vduse_dev_request);
 303         ssize_t ret;
 304
 305         if (iov_iter_count(to) < size)
 306                 return -EINVAL;
 307
 308         spin_lock(&dev->msg_lock);
 309         while (1) {
 310                 msg = vduse_dequeue_msg(&dev->send_list);
 311                 if (msg)
 312                         break;
 313
 314                 ret = -EAGAIN;
 315                 if (file->f_flags & O_NONBLOCK)
 316                         goto unlock;
 317
 318                 spin_unlock(&dev->msg_lock);
 319                 ret = wait_event_interruptible_exclusive(dev->waitq,
 320                                         !list_empty(&dev->send_list));
 321                 if (ret)
 322                         return ret;
 323
 324                 spin_lock(&dev->msg_lock);
 325         }
 326         spin_unlock(&dev->msg_lock);
 327         ret = copy_to_iter(&msg->req, size, to);
 328         spin_lock(&dev->msg_lock);
 329         if (ret != size) {
 330                 ret = -EFAULT;
 331                 vduse_enqueue_msg(&dev->send_list, msg);
 332                 goto unlock;
 333         }
 334         vduse_enqueue_msg(&dev->recv_list, msg);
 335 unlock:
 336         spin_unlock(&dev->msg_lock);
 337
 338         return ret;
 339 }
 340
 341 static bool is_mem_zero(const char *ptr, int size)
 342 {
 343         int i;
 344
 345         for (i = 0; i < size; i++) {
 346                 if (ptr[i])
 347                         return false;
 348         }
 349         return true;
 350 }
 351
 352 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 353 {
 354         struct file *file = iocb->ki_filp;
 355         struct vduse_dev *dev = file->private_data;
 356         struct vduse_dev_response resp;
 357         struct vduse_dev_msg *msg;
 358         size_t ret;
 359
 360         ret = copy_from_iter(&resp, sizeof(resp), from);
 361         if (ret != sizeof(resp))
 362                 return -EINVAL;
 363
 364         if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
 365                 return -EINVAL;
 366
 367         spin_lock(&dev->msg_lock);
 368         msg = vduse_find_msg(&dev->recv_list, resp.request_id);
 369         if (!msg) {
 370                 ret = -ENOENT;
 371                 goto unlock;
 372         }
 373
 374         memcpy(&msg->resp, &resp, sizeof(resp));
 375         msg->completed = 1;
 376         wake_up(&msg->waitq);
 377 unlock:
 378         spin_unlock(&dev->msg_lock);
 379
 380         return ret;
 381 }
 382
 383 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
 384 {
 385         struct vduse_dev *dev = file->private_data;
 386         __poll_t mask = 0;
 387
 388         poll_wait(file, &dev->waitq, wait);
 389
 390         spin_lock(&dev->msg_lock);
 391
 392         if (unlikely(dev->broken))
 393                 mask |= EPOLLERR;
 394         if (!list_empty(&dev->send_list))
 395                 mask |= EPOLLIN | EPOLLRDNORM;
 396         if (!list_empty(&dev->recv_list))
 397                 mask |= EPOLLOUT | EPOLLWRNORM;
 398
 399         spin_unlock(&dev->msg_lock);
 400
 401         return mask;
 402 }
 403
 404 static void vduse_dev_reset(struct vduse_dev *dev)
 405 {
 406         int i;
 407         struct vduse_iova_domain *domain = dev->domain;
 408
 409         /* The coherent mappings are handled in vduse_dev_free_coherent() */
 410         if (domain->bounce_map)
 411                 vduse_domain_reset_bounce_map(domain);
 412
 413         dev->status = 0;
 414         dev->driver_features = 0;
 415         dev->generation++;
 416         spin_lock(&dev->irq_lock);
 417         dev->config_cb.callback = NULL;
 418         dev->config_cb.private = NULL;
 419         spin_unlock(&dev->irq_lock);
 420         flush_work(&dev->inject);
 421
 422         for (i = 0; i < dev->vq_num; i++) {
 423                 struct vduse_virtqueue *vq = &dev->vqs[i];
 424
 425                 vq->ready = false;
 426                 vq->desc_addr = 0;
 427                 vq->driver_addr = 0;
 428                 vq->device_addr = 0;
 429                 vq->num = 0;
 430                 memset(&vq->state, 0, sizeof(vq->state));
 431
 432                 spin_lock(&vq->kick_lock);
 433                 vq->kicked = false;
 434                 if (vq->kickfd)
 435                         eventfd_ctx_put(vq->kickfd);
 436                 vq->kickfd = NULL;
 437                 spin_unlock(&vq->kick_lock);
 438
 439                 spin_lock(&vq->irq_lock);
 440                 vq->cb.callback = NULL;
 441                 vq->cb.private = NULL;
 442                 spin_unlock(&vq->irq_lock);
 443                 flush_work(&vq->inject);
 444                 flush_work(&vq->kick);
 445         }
 446 }
 447
 448 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
 449                                 u64 desc_area, u64 driver_area,
 450                                 u64 device_area)
 451 {
 452         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 453         struct vduse_virtqueue *vq = &dev->vqs[idx];
 454
 455         vq->desc_addr = desc_area;
 456         vq->driver_addr = driver_area;
 457         vq->device_addr = device_area;
 458
 459         return 0;
 460 }
 461
 462 static void vduse_vq_kick(struct vduse_virtqueue *vq)
 463 {
 464         spin_lock(&vq->kick_lock);
 465         if (!vq->ready)
 466                 goto unlock;
 467
 468         if (vq->kickfd)
 469                 eventfd_signal(vq->kickfd, 1);
 470         else
 471                 vq->kicked = true;
 472 unlock:
 473         spin_unlock(&vq->kick_lock);
 474 }
 475
 476 static void vduse_vq_kick_work(struct work_struct *work)
 477 {
 478         struct vduse_virtqueue *vq = container_of(work,
 479                                         struct vduse_virtqueue, kick);
 480
 481         vduse_vq_kick(vq);
 482 }
 483
 484 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
 485 {
 486         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 487         struct vduse_virtqueue *vq = &dev->vqs[idx];
 488
 489         if (!eventfd_signal_allowed()) {
 490                 schedule_work(&vq->kick);
 491                 return;
 492         }
 493         vduse_vq_kick(vq);
 494 }
 495
 496 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
 497                               struct vdpa_callback *cb)
 498 {
 499         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 500         struct vduse_virtqueue *vq = &dev->vqs[idx];
 501
 502         spin_lock(&vq->irq_lock);
 503         vq->cb.callback = cb->callback;
 504         vq->cb.private = cb->private;
 505         spin_unlock(&vq->irq_lock);
 506 }
 507
 508 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
 509 {
 510         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 511         struct vduse_virtqueue *vq = &dev->vqs[idx];
 512
 513         vq->num = num;
 514 }
 515
 516 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 517                                         u16 idx, bool ready)
 518 {
 519         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 520         struct vduse_virtqueue *vq = &dev->vqs[idx];
 521
 522         vq->ready = ready;
 523 }
 524
 525 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
 526 {
 527         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 528         struct vduse_virtqueue *vq = &dev->vqs[idx];
 529
 530         return vq->ready;
 531 }
 532
 533 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
 534                                 const struct vdpa_vq_state *state)
 535 {
 536         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 537         struct vduse_virtqueue *vq = &dev->vqs[idx];
 538
 539         if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
 540                 vq->state.packed.last_avail_counter =
 541                                 state->packed.last_avail_counter;
 542                 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
 543                 vq->state.packed.last_used_counter =
 544                                 state->packed.last_used_counter;
 545                 vq->state.packed.last_used_idx = state->packed.last_used_idx;
 546         } else
 547                 vq->state.split.avail_index = state->split.avail_index;
 548
 549         return 0;
 550 }
 551
 552 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
 553                                 struct vdpa_vq_state *state)
 554 {
 555         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 556         struct vduse_virtqueue *vq = &dev->vqs[idx];
 557
 558         if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
 559                 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
 560
 561         return vduse_dev_get_vq_state_split(dev, vq, &state->split);
 562 }
 563
 564 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
 565 {
 566         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 567
 568         return dev->vq_align;
 569 }
 570
 571 static u64 vduse_vdpa_get_features(struct vdpa_device *vdpa)
 572 {
 573         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 574
 575         return dev->device_features;
 576 }
 577
 578 static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features)
 579 {
 580         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 581
 582         dev->driver_features = features;
 583         return 0;
 584 }
 585
 586 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
 587                                   struct vdpa_callback *cb)
 588 {
 589         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 590
 591         spin_lock(&dev->irq_lock);
 592         dev->config_cb.callback = cb->callback;
 593         dev->config_cb.private = cb->private;
 594         spin_unlock(&dev->irq_lock);
 595 }
 596
 597 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
 598 {
 599         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 600         u16 num_max = 0;
 601         int i;
 602
 603         for (i = 0; i < dev->vq_num; i++)
 604                 if (num_max < dev->vqs[i].num_max)
 605                         num_max = dev->vqs[i].num_max;
 606
 607         return num_max;
 608 }
 609
 610 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
 611 {
 612         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 613
 614         return dev->device_id;
 615 }
 616
 617 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
 618 {
 619         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 620
 621         return dev->vendor_id;
 622 }
 623
 624 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
 625 {
 626         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 627
 628         return dev->status;
 629 }
 630
 631 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
 632 {
 633         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 634
 635         if (vduse_dev_set_status(dev, status))
 636                 return;
 637
 638         dev->status = status;
 639 }
 640
 641 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
 642 {
 643         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 644
 645         return dev->config_size;
 646 }
 647
 648 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
 649                                   void *buf, unsigned int len)
 650 {
 651         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 652
 653         if (len > dev->config_size - offset)
 654                 return;
 655
 656         memcpy(buf, dev->config + offset, len);
 657 }
 658
 659 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
 660                         const void *buf, unsigned int len)
 661 {
 662         /* Now we only support read-only configuration space */
 663 }
 664
 665 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
 666 {
 667         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 668         int ret = vduse_dev_set_status(dev, 0);
 669
 670         vduse_dev_reset(dev);
 671
 672         return ret;
 673 }
 674
 675 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
 676 {
 677         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 678
 679         return dev->generation;
 680 }
 681
 682 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
 683                                 struct vhost_iotlb *iotlb)
 684 {
 685         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 686         int ret;
 687
 688         ret = vduse_domain_set_map(dev->domain, iotlb);
 689         if (ret)
 690                 return ret;
 691
 692         ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
 693         if (ret) {
 694                 vduse_domain_clear_map(dev->domain, iotlb);
 695                 return ret;
 696         }
 697
 698         return 0;
 699 }
 700
 701 static void vduse_vdpa_free(struct vdpa_device *vdpa)
 702 {
 703         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 704
 705         dev->vdev = NULL;
 706 }
 707
 708 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
 709         .set_vq_address         = vduse_vdpa_set_vq_address,
 710         .kick_vq                = vduse_vdpa_kick_vq,
 711         .set_vq_cb              = vduse_vdpa_set_vq_cb,
 712         .set_vq_num             = vduse_vdpa_set_vq_num,
 713         .set_vq_ready           = vduse_vdpa_set_vq_ready,
 714         .get_vq_ready           = vduse_vdpa_get_vq_ready,
 715         .set_vq_state           = vduse_vdpa_set_vq_state,
 716         .get_vq_state           = vduse_vdpa_get_vq_state,
 717         .get_vq_align           = vduse_vdpa_get_vq_align,
 718         .get_features           = vduse_vdpa_get_features,
 719         .set_features           = vduse_vdpa_set_features,
 720         .set_config_cb          = vduse_vdpa_set_config_cb,
 721         .get_vq_num_max         = vduse_vdpa_get_vq_num_max,
 722         .get_device_id          = vduse_vdpa_get_device_id,
 723         .get_vendor_id          = vduse_vdpa_get_vendor_id,
 724         .get_status             = vduse_vdpa_get_status,
 725         .set_status             = vduse_vdpa_set_status,
 726         .get_config_size        = vduse_vdpa_get_config_size,
 727         .get_config             = vduse_vdpa_get_config,
 728         .set_config             = vduse_vdpa_set_config,
 729         .get_generation         = vduse_vdpa_get_generation,
 730         .reset                  = vduse_vdpa_reset,
 731         .set_map                = vduse_vdpa_set_map,
 732         .free                   = vduse_vdpa_free,
 733 };
 734
 735 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
 736                                      unsigned long offset, size_t size,
 737                                      enum dma_data_direction dir,
 738                                      unsigned long attrs)
 739 {
 740         struct vduse_dev *vdev = dev_to_vduse(dev);
 741         struct vduse_iova_domain *domain = vdev->domain;
 742
 743         return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
 744 }
 745
 746 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
 747                                 size_t size, enum dma_data_direction dir,
 748                                 unsigned long attrs)
 749 {
 750         struct vduse_dev *vdev = dev_to_vduse(dev);
 751         struct vduse_iova_domain *domain = vdev->domain;
 752
 753         return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
 754 }
 755
 756 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
 757                                         dma_addr_t *dma_addr, gfp_t flag,
 758                                         unsigned long attrs)
 759 {
 760         struct vduse_dev *vdev = dev_to_vduse(dev);
 761         struct vduse_iova_domain *domain = vdev->domain;
 762         unsigned long iova;
 763         void *addr;
 764
 765         *dma_addr = DMA_MAPPING_ERROR;
 766         addr = vduse_domain_alloc_coherent(domain, size,
 767                                 (dma_addr_t *)&iova, flag, attrs);
 768         if (!addr)
 769                 return NULL;
 770
 771         *dma_addr = (dma_addr_t)iova;
 772
 773         return addr;
 774 }
 775
 776 static void vduse_dev_free_coherent(struct device *dev, size_t size,
 777                                         void *vaddr, dma_addr_t dma_addr,
 778                                         unsigned long attrs)
 779 {
 780         struct vduse_dev *vdev = dev_to_vduse(dev);
 781         struct vduse_iova_domain *domain = vdev->domain;
 782
 783         vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
 784 }
 785
 786 static size_t vduse_dev_max_mapping_size(struct device *dev)
 787 {
 788         struct vduse_dev *vdev = dev_to_vduse(dev);
 789         struct vduse_iova_domain *domain = vdev->domain;
 790
 791         return domain->bounce_size;
 792 }
 793
 794 static const struct dma_map_ops vduse_dev_dma_ops = {
 795         .map_page = vduse_dev_map_page,
 796         .unmap_page = vduse_dev_unmap_page,
 797         .alloc = vduse_dev_alloc_coherent,
 798         .free = vduse_dev_free_coherent,
 799         .max_mapping_size = vduse_dev_max_mapping_size,
 800 };
 801
 802 static unsigned int perm_to_file_flags(u8 perm)
 803 {
 804         unsigned int flags = 0;
 805
 806         switch (perm) {
 807         case VDUSE_ACCESS_WO:
 808                 flags |= O_WRONLY;
 809                 break;
 810         case VDUSE_ACCESS_RO:
 811                 flags |= O_RDONLY;
 812                 break;
 813         case VDUSE_ACCESS_RW:
 814                 flags |= O_RDWR;
 815                 break;
 816         default:
 817                 WARN(1, "invalidate vhost IOTLB permission\n");
 818                 break;
 819         }
 820
 821         return flags;
 822 }
 823
 824 static int vduse_kickfd_setup(struct vduse_dev *dev,
 825                         struct vduse_vq_eventfd *eventfd)
 826 {
 827         struct eventfd_ctx *ctx = NULL;
 828         struct vduse_virtqueue *vq;
 829         u32 index;
 830
 831         if (eventfd->index >= dev->vq_num)
 832                 return -EINVAL;
 833
 834         index = array_index_nospec(eventfd->index, dev->vq_num);
 835         vq = &dev->vqs[index];
 836         if (eventfd->fd >= 0) {
 837                 ctx = eventfd_ctx_fdget(eventfd->fd);
 838                 if (IS_ERR(ctx))
 839                         return PTR_ERR(ctx);
 840         } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
 841                 return 0;
 842
 843         spin_lock(&vq->kick_lock);
 844         if (vq->kickfd)
 845                 eventfd_ctx_put(vq->kickfd);
 846         vq->kickfd = ctx;
 847         if (vq->ready && vq->kicked && vq->kickfd) {
 848                 eventfd_signal(vq->kickfd, 1);
 849                 vq->kicked = false;
 850         }
 851         spin_unlock(&vq->kick_lock);
 852
 853         return 0;
 854 }
 855
 856 static bool vduse_dev_is_ready(struct vduse_dev *dev)
 857 {
 858         int i;
 859
 860         for (i = 0; i < dev->vq_num; i++)
 861                 if (!dev->vqs[i].num_max)
 862                         return false;
 863
 864         return true;
 865 }
 866
 867 static void vduse_dev_irq_inject(struct work_struct *work)
 868 {
 869         struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
 870
 871         spin_lock_irq(&dev->irq_lock);
 872         if (dev->config_cb.callback)
 873                 dev->config_cb.callback(dev->config_cb.private);
 874         spin_unlock_irq(&dev->irq_lock);
 875 }
 876
 877 static void vduse_vq_irq_inject(struct work_struct *work)
 878 {
 879         struct vduse_virtqueue *vq = container_of(work,
 880                                         struct vduse_virtqueue, inject);
 881
 882         spin_lock_irq(&vq->irq_lock);
 883         if (vq->ready && vq->cb.callback)
 884                 vq->cb.callback(vq->cb.private);
 885         spin_unlock_irq(&vq->irq_lock);
 886 }
 887
 888 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 889                             unsigned long arg)
 890 {
 891         struct vduse_dev *dev = file->private_data;
 892         void __user *argp = (void __user *)arg;
 893         int ret;
 894
 895         if (unlikely(dev->broken))
 896                 return -EPERM;
 897
 898         switch (cmd) {
 899         case VDUSE_IOTLB_GET_FD: {
 900                 struct vduse_iotlb_entry entry;
 901                 struct vhost_iotlb_map *map;
 902                 struct vdpa_map_file *map_file;
 903                 struct vduse_iova_domain *domain = dev->domain;
 904                 struct file *f = NULL;
 905
 906                 ret = -EFAULT;
 907                 if (copy_from_user(&entry, argp, sizeof(entry)))
 908                         break;
 909
 910                 ret = -EINVAL;
 911                 if (entry.start > entry.last)
 912                         break;
 913
 914                 spin_lock(&domain->iotlb_lock);
 915                 map = vhost_iotlb_itree_first(domain->iotlb,
 916                                               entry.start, entry.last);
 917                 if (map) {
 918                         map_file = (struct vdpa_map_file *)map->opaque;
 919                         f = get_file(map_file->file);
 920                         entry.offset = map_file->offset;
 921                         entry.start = map->start;
 922                         entry.last = map->last;
 923                         entry.perm = map->perm;
 924                 }
 925                 spin_unlock(&domain->iotlb_lock);
 926                 ret = -EINVAL;
 927                 if (!f)
 928                         break;
 929
 930                 ret = -EFAULT;
 931                 if (copy_to_user(argp, &entry, sizeof(entry))) {
 932                         fput(f);
 933                         break;
 934                 }
 935                 ret = receive_fd(f, perm_to_file_flags(entry.perm));
 936                 fput(f);
 937                 break;
 938         }
 939         case VDUSE_DEV_GET_FEATURES:
 940                 /*
 941                  * Just mirror what driver wrote here.
 942                  * The driver is expected to check FEATURE_OK later.
 943                  */
 944                 ret = put_user(dev->driver_features, (u64 __user *)argp);
 945                 break;
 946         case VDUSE_DEV_SET_CONFIG: {
 947                 struct vduse_config_data config;
 948                 unsigned long size = offsetof(struct vduse_config_data,
 949                                               buffer);
 950
 951                 ret = -EFAULT;
 952                 if (copy_from_user(&config, argp, size))
 953                         break;
 954
 955                 ret = -EINVAL;
 956                 if (config.length == 0 ||
 957                     config.length > dev->config_size - config.offset)
 958                         break;
 959
 960                 ret = -EFAULT;
 961                 if (copy_from_user(dev->config + config.offset, argp + size,
 962                                    config.length))
 963                         break;
 964
 965                 ret = 0;
 966                 break;
 967         }
 968         case VDUSE_DEV_INJECT_CONFIG_IRQ:
 969                 ret = -EINVAL;
 970                 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
 971                         break;
 972
 973                 ret = 0;
 974                 queue_work(vduse_irq_wq, &dev->inject);
 975                 break;
 976         case VDUSE_VQ_SETUP: {
 977                 struct vduse_vq_config config;
 978                 u32 index;
 979
 980                 ret = -EFAULT;
 981                 if (copy_from_user(&config, argp, sizeof(config)))
 982                         break;
 983
 984                 ret = -EINVAL;
 985                 if (config.index >= dev->vq_num)
 986                         break;
 987
 988                 if (!is_mem_zero((const char *)config.reserved,
 989                                  sizeof(config.reserved)))
 990                         break;
 991
 992                 index = array_index_nospec(config.index, dev->vq_num);
 993                 dev->vqs[index].num_max = config.max_size;
 994                 ret = 0;
 995                 break;
 996         }
 997         case VDUSE_VQ_GET_INFO: {
 998                 struct vduse_vq_info vq_info;
 999                 struct vduse_virtqueue *vq;
1000                 u32 index;
1001
1002                 ret = -EFAULT;
1003                 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1004                         break;
1005
1006                 ret = -EINVAL;
1007                 if (vq_info.index >= dev->vq_num)
1008                         break;
1009
1010                 index = array_index_nospec(vq_info.index, dev->vq_num);
1011                 vq = &dev->vqs[index];
1012                 vq_info.desc_addr = vq->desc_addr;
1013                 vq_info.driver_addr = vq->driver_addr;
1014                 vq_info.device_addr = vq->device_addr;
1015                 vq_info.num = vq->num;
1016
1017                 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1018                         vq_info.packed.last_avail_counter =
1019                                 vq->state.packed.last_avail_counter;
1020                         vq_info.packed.last_avail_idx =
1021                                 vq->state.packed.last_avail_idx;
1022                         vq_info.packed.last_used_counter =
1023                                 vq->state.packed.last_used_counter;
1024                         vq_info.packed.last_used_idx =
1025                                 vq->state.packed.last_used_idx;
1026                 } else
1027                         vq_info.split.avail_index =
1028                                 vq->state.split.avail_index;
1029
1030                 vq_info.ready = vq->ready;
1031
1032                 ret = -EFAULT;
1033                 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1034                         break;
1035
1036                 ret = 0;
1037                 break;
1038         }
1039         case VDUSE_VQ_SETUP_KICKFD: {
1040                 struct vduse_vq_eventfd eventfd;
1041
1042                 ret = -EFAULT;
1043                 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1044                         break;
1045
1046                 ret = vduse_kickfd_setup(dev, &eventfd);
1047                 break;
1048         }
1049         case VDUSE_VQ_INJECT_IRQ: {
1050                 u32 index;
1051
1052                 ret = -EINVAL;
1053                 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1054                         break;
1055
1056                 ret = -EFAULT;
1057                 if (get_user(index, (u32 __user *)argp))
1058                         break;
1059
1060                 ret = -EINVAL;
1061                 if (index >= dev->vq_num)
1062                         break;
1063
1064                 ret = 0;
1065                 index = array_index_nospec(index, dev->vq_num);
1066                 queue_work(vduse_irq_wq, &dev->vqs[index].inject);
1067                 break;
1068         }
1069         default:
1070                 ret = -ENOIOCTLCMD;
1071                 break;
1072         }
1073
1074         return ret;
1075 }
1076
1077 static int vduse_dev_release(struct inode *inode, struct file *file)
1078 {
1079         struct vduse_dev *dev = file->private_data;
1080
1081         spin_lock(&dev->msg_lock);
1082         /* Make sure the inflight messages can processed after reconncection */
1083         list_splice_init(&dev->recv_list, &dev->send_list);
1084         spin_unlock(&dev->msg_lock);
1085         dev->connected = false;
1086
1087         return 0;
1088 }
1089
1090 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1091 {
1092         struct vduse_dev *dev;
1093
1094         mutex_lock(&vduse_lock);
1095         dev = idr_find(&vduse_idr, minor);
1096         mutex_unlock(&vduse_lock);
1097
1098         return dev;
1099 }
1100
1101 static int vduse_dev_open(struct inode *inode, struct file *file)
1102 {
1103         int ret;
1104         struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1105
1106         if (!dev)
1107                 return -ENODEV;
1108
1109         ret = -EBUSY;
1110         mutex_lock(&dev->lock);
1111         if (dev->connected)
1112                 goto unlock;
1113
1114         ret = 0;
1115         dev->connected = true;
1116         file->private_data = dev;
1117 unlock:
1118         mutex_unlock(&dev->lock);
1119
1120         return ret;
1121 }
1122
1123 static const struct file_operations vduse_dev_fops = {
1124         .owner          = THIS_MODULE,
1125         .open           = vduse_dev_open,
1126         .release        = vduse_dev_release,
1127         .read_iter      = vduse_dev_read_iter,
1128         .write_iter     = vduse_dev_write_iter,
1129         .poll           = vduse_dev_poll,
1130         .unlocked_ioctl = vduse_dev_ioctl,
1131         .compat_ioctl   = compat_ptr_ioctl,
1132         .llseek         = noop_llseek,
1133 };
1134
1135 static struct vduse_dev *vduse_dev_create(void)
1136 {
1137         struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1138
1139         if (!dev)
1140                 return NULL;
1141
1142         mutex_init(&dev->lock);
1143         spin_lock_init(&dev->msg_lock);
1144         INIT_LIST_HEAD(&dev->send_list);
1145         INIT_LIST_HEAD(&dev->recv_list);
1146         spin_lock_init(&dev->irq_lock);
1147
1148         INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1149         init_waitqueue_head(&dev->waitq);
1150
1151         return dev;
1152 }
1153
1154 static void vduse_dev_destroy(struct vduse_dev *dev)
1155 {
1156         kfree(dev);
1157 }
1158
1159 static struct vduse_dev *vduse_find_dev(const char *name)
1160 {
1161         struct vduse_dev *dev;
1162         int id;
1163
1164         idr_for_each_entry(&vduse_idr, dev, id)
1165                 if (!strcmp(dev->name, name))
1166                         return dev;
1167
1168         return NULL;
1169 }
1170
1171 static int vduse_destroy_dev(char *name)
1172 {
1173         struct vduse_dev *dev = vduse_find_dev(name);
1174
1175         if (!dev)
1176                 return -EINVAL;
1177
1178         mutex_lock(&dev->lock);
1179         if (dev->vdev || dev->connected) {
1180                 mutex_unlock(&dev->lock);
1181                 return -EBUSY;
1182         }
1183         dev->connected = true;
1184         mutex_unlock(&dev->lock);
1185
1186         vduse_dev_reset(dev);
1187         device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1188         idr_remove(&vduse_idr, dev->minor);
1189         kvfree(dev->config);
1190         kfree(dev->vqs);
1191         vduse_domain_destroy(dev->domain);
1192         kfree(dev->name);
1193         vduse_dev_destroy(dev);
1194         module_put(THIS_MODULE);
1195
1196         return 0;
1197 }
1198
1199 static bool device_is_allowed(u32 device_id)
1200 {
1201         int i;
1202
1203         for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1204                 if (allowed_device_id[i] == device_id)
1205                         return true;
1206
1207         return false;
1208 }
1209
1210 static bool features_is_valid(u64 features)
1211 {
1212         if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1213                 return false;
1214
1215         /* Now we only support read-only configuration space */
1216         if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1217                 return false;
1218
1219         return true;
1220 }
1221
1222 static bool vduse_validate_config(struct vduse_dev_config *config)
1223 {
1224         if (!is_mem_zero((const char *)config->reserved,
1225                          sizeof(config->reserved)))
1226                 return false;
1227
1228         if (config->vq_align > PAGE_SIZE)
1229                 return false;
1230
1231         if (config->config_size > PAGE_SIZE)
1232                 return false;
1233
1234         if (!device_is_allowed(config->device_id))
1235                 return false;
1236
1237         if (!features_is_valid(config->features))
1238                 return false;
1239
1240         return true;
1241 }
1242
1243 static ssize_t msg_timeout_show(struct device *device,
1244                                 struct device_attribute *attr, char *buf)
1245 {
1246         struct vduse_dev *dev = dev_get_drvdata(device);
1247
1248         return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1249 }
1250
1251 static ssize_t msg_timeout_store(struct device *device,
1252                                  struct device_attribute *attr,
1253                                  const char *buf, size_t count)
1254 {
1255         struct vduse_dev *dev = dev_get_drvdata(device);
1256         int ret;
1257
1258         ret = kstrtouint(buf, 10, &dev->msg_timeout);
1259         if (ret < 0)
1260                 return ret;
1261
1262         return count;
1263 }
1264
1265 static DEVICE_ATTR_RW(msg_timeout);
1266
1267 static struct attribute *vduse_dev_attrs[] = {
1268         &dev_attr_msg_timeout.attr,
1269         NULL
1270 };
1271
1272 ATTRIBUTE_GROUPS(vduse_dev);
1273
1274 static int vduse_create_dev(struct vduse_dev_config *config,
1275                             void *config_buf, u64 api_version)
1276 {
1277         int i, ret;
1278         struct vduse_dev *dev;
1279
1280         ret = -EEXIST;
1281         if (vduse_find_dev(config->name))
1282                 goto err;
1283
1284         ret = -ENOMEM;
1285         dev = vduse_dev_create();
1286         if (!dev)
1287                 goto err;
1288
1289         dev->api_version = api_version;
1290         dev->device_features = config->features;
1291         dev->device_id = config->device_id;
1292         dev->vendor_id = config->vendor_id;
1293         dev->name = kstrdup(config->name, GFP_KERNEL);
1294         if (!dev->name)
1295                 goto err_str;
1296
1297         dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1298                                           VDUSE_BOUNCE_SIZE);
1299         if (!dev->domain)
1300                 goto err_domain;
1301
1302         dev->config = config_buf;
1303         dev->config_size = config->config_size;
1304         dev->vq_align = config->vq_align;
1305         dev->vq_num = config->vq_num;
1306         dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1307         if (!dev->vqs)
1308                 goto err_vqs;
1309
1310         for (i = 0; i < dev->vq_num; i++) {
1311                 dev->vqs[i].index = i;
1312                 INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
1313                 INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
1314                 spin_lock_init(&dev->vqs[i].kick_lock);
1315                 spin_lock_init(&dev->vqs[i].irq_lock);
1316         }
1317
1318         ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1319         if (ret < 0)
1320                 goto err_idr;
1321
1322         dev->minor = ret;
1323         dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1324         dev->dev = device_create(vduse_class, NULL,
1325                                  MKDEV(MAJOR(vduse_major), dev->minor),
1326                                  dev, "%s", config->name);
1327         if (IS_ERR(dev->dev)) {
1328                 ret = PTR_ERR(dev->dev);
1329                 goto err_dev;
1330         }
1331         __module_get(THIS_MODULE);
1332
1333         return 0;
1334 err_dev:
1335         idr_remove(&vduse_idr, dev->minor);
1336 err_idr:
1337         kfree(dev->vqs);
1338 err_vqs:
1339         vduse_domain_destroy(dev->domain);
1340 err_domain:
1341         kfree(dev->name);
1342 err_str:
1343         vduse_dev_destroy(dev);
1344 err:
1345         kvfree(config_buf);
1346         return ret;
1347 }
1348
1349 static long vduse_ioctl(struct file *file, unsigned int cmd,
1350                         unsigned long arg)
1351 {
1352         int ret;
1353         void __user *argp = (void __user *)arg;
1354         struct vduse_control *control = file->private_data;
1355
1356         mutex_lock(&vduse_lock);
1357         switch (cmd) {
1358         case VDUSE_GET_API_VERSION:
1359                 ret = put_user(control->api_version, (u64 __user *)argp);
1360                 break;
1361         case VDUSE_SET_API_VERSION: {
1362                 u64 api_version;
1363
1364                 ret = -EFAULT;
1365                 if (get_user(api_version, (u64 __user *)argp))
1366                         break;
1367
1368                 ret = -EINVAL;
1369                 if (api_version > VDUSE_API_VERSION)
1370                         break;
1371
1372                 ret = 0;
1373                 control->api_version = api_version;
1374                 break;
1375         }
1376         case VDUSE_CREATE_DEV: {
1377                 struct vduse_dev_config config;
1378                 unsigned long size = offsetof(struct vduse_dev_config, config);
1379                 void *buf;
1380
1381                 ret = -EFAULT;
1382                 if (copy_from_user(&config, argp, size))
1383                         break;
1384
1385                 ret = -EINVAL;
1386                 if (vduse_validate_config(&config) == false)
1387                         break;
1388
1389                 buf = vmemdup_user(argp + size, config.config_size);
1390                 if (IS_ERR(buf)) {
1391                         ret = PTR_ERR(buf);
1392                         break;
1393                 }
1394                 config.name[VDUSE_NAME_MAX - 1] = '\0';
1395                 ret = vduse_create_dev(&config, buf, control->api_version);
1396                 break;
1397         }
1398         case VDUSE_DESTROY_DEV: {
1399                 char name[VDUSE_NAME_MAX];
1400
1401                 ret = -EFAULT;
1402                 if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1403                         break;
1404
1405                 name[VDUSE_NAME_MAX - 1] = '\0';
1406                 ret = vduse_destroy_dev(name);
1407                 break;
1408         }
1409         default:
1410                 ret = -EINVAL;
1411                 break;
1412         }
1413         mutex_unlock(&vduse_lock);
1414
1415         return ret;
1416 }
1417
1418 static int vduse_release(struct inode *inode, struct file *file)
1419 {
1420         struct vduse_control *control = file->private_data;
1421
1422         kfree(control);
1423         return 0;
1424 }
1425
1426 static int vduse_open(struct inode *inode, struct file *file)
1427 {
1428         struct vduse_control *control;
1429
1430         control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1431         if (!control)
1432                 return -ENOMEM;
1433
1434         control->api_version = VDUSE_API_VERSION;
1435         file->private_data = control;
1436
1437         return 0;
1438 }
1439
1440 static const struct file_operations vduse_ctrl_fops = {
1441         .owner          = THIS_MODULE,
1442         .open           = vduse_open,
1443         .release        = vduse_release,
1444         .unlocked_ioctl = vduse_ioctl,
1445         .compat_ioctl   = compat_ptr_ioctl,
1446         .llseek         = noop_llseek,
1447 };
1448
1449 static char *vduse_devnode(struct device *dev, umode_t *mode)
1450 {
1451         return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1452 }
1453
1454 static void vduse_mgmtdev_release(struct device *dev)
1455 {
1456 }
1457
1458 static struct device vduse_mgmtdev = {
1459         .init_name = "vduse",
1460         .release = vduse_mgmtdev_release,
1461 };
1462
1463 static struct vdpa_mgmt_dev mgmt_dev;
1464
1465 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1466 {
1467         struct vduse_vdpa *vdev;
1468         int ret;
1469
1470         if (dev->vdev)
1471                 return -EEXIST;
1472
1473         vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1474                                  &vduse_vdpa_config_ops, name, true);
1475         if (IS_ERR(vdev))
1476                 return PTR_ERR(vdev);
1477
1478         dev->vdev = vdev;
1479         vdev->dev = dev;
1480         vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1481         ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1482         if (ret) {
1483                 put_device(&vdev->vdpa.dev);
1484                 return ret;
1485         }
1486         set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1487         vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1488         vdev->vdpa.mdev = &mgmt_dev;
1489
1490         return 0;
1491 }
1492
1493 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
1494 {
1495         struct vduse_dev *dev;
1496         int ret;
1497
1498         mutex_lock(&vduse_lock);
1499         dev = vduse_find_dev(name);
1500         if (!dev || !vduse_dev_is_ready(dev)) {
1501                 mutex_unlock(&vduse_lock);
1502                 return -EINVAL;
1503         }
1504         ret = vduse_dev_init_vdpa(dev, name);
1505         mutex_unlock(&vduse_lock);
1506         if (ret)
1507                 return ret;
1508
1509         ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1510         if (ret) {
1511                 put_device(&dev->vdev->vdpa.dev);
1512                 return ret;
1513         }
1514
1515         return 0;
1516 }
1517
1518 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
1519 {
1520         _vdpa_unregister_device(dev);
1521 }
1522
1523 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
1524         .dev_add = vdpa_dev_add,
1525         .dev_del = vdpa_dev_del,
1526 };
1527
1528 static struct virtio_device_id id_table[] = {
1529         { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1530         { 0 },
1531 };
1532
1533 static struct vdpa_mgmt_dev mgmt_dev = {
1534         .device = &vduse_mgmtdev,
1535         .id_table = id_table,
1536         .ops = &vdpa_dev_mgmtdev_ops,
1537 };
1538
1539 static int vduse_mgmtdev_init(void)
1540 {
1541         int ret;
1542
1543         ret = device_register(&vduse_mgmtdev);
1544         if (ret)
1545                 return ret;
1546
1547         ret = vdpa_mgmtdev_register(&mgmt_dev);
1548         if (ret)
1549                 goto err;
1550
1551         return 0;
1552 err:
1553         device_unregister(&vduse_mgmtdev);
1554         return ret;
1555 }
1556
1557 static void vduse_mgmtdev_exit(void)
1558 {
1559         vdpa_mgmtdev_unregister(&mgmt_dev);
1560         device_unregister(&vduse_mgmtdev);
1561 }
1562
1563 static int vduse_init(void)
1564 {
1565         int ret;
1566         struct device *dev;
1567
1568         vduse_class = class_create(THIS_MODULE, "vduse");
1569         if (IS_ERR(vduse_class))
1570                 return PTR_ERR(vduse_class);
1571
1572         vduse_class->devnode = vduse_devnode;
1573         vduse_class->dev_groups = vduse_dev_groups;
1574
1575         ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
1576         if (ret)
1577                 goto err_chardev_region;
1578
1579         /* /dev/vduse/control */
1580         cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
1581         vduse_ctrl_cdev.owner = THIS_MODULE;
1582         ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
1583         if (ret)
1584                 goto err_ctrl_cdev;
1585
1586         dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
1587         if (IS_ERR(dev)) {
1588                 ret = PTR_ERR(dev);
1589                 goto err_device;
1590         }
1591
1592         /* /dev/vduse/$DEVICE */
1593         cdev_init(&vduse_cdev, &vduse_dev_fops);
1594         vduse_cdev.owner = THIS_MODULE;
1595         ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
1596                        VDUSE_DEV_MAX - 1);
1597         if (ret)
1598                 goto err_cdev;
1599
1600         vduse_irq_wq = alloc_workqueue("vduse-irq",
1601                                 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
1602         if (!vduse_irq_wq) {
1603                 ret = -ENOMEM;
1604                 goto err_wq;
1605         }
1606
1607         ret = vduse_domain_init();
1608         if (ret)
1609                 goto err_domain;
1610
1611         ret = vduse_mgmtdev_init();
1612         if (ret)
1613                 goto err_mgmtdev;
1614
1615         return 0;
1616 err_mgmtdev:
1617         vduse_domain_exit();
1618 err_domain:
1619         destroy_workqueue(vduse_irq_wq);
1620 err_wq:
1621         cdev_del(&vduse_cdev);
1622 err_cdev:
1623         device_destroy(vduse_class, vduse_major);
1624 err_device:
1625         cdev_del(&vduse_ctrl_cdev);
1626 err_ctrl_cdev:
1627         unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1628 err_chardev_region:
1629         class_destroy(vduse_class);
1630         return ret;
1631 }
1632 module_init(vduse_init);
1633
1634 static void vduse_exit(void)
1635 {
1636         vduse_mgmtdev_exit();
1637         vduse_domain_exit();
1638         destroy_workqueue(vduse_irq_wq);
1639         cdev_del(&vduse_cdev);
1640         device_destroy(vduse_class, vduse_major);
1641         cdev_del(&vduse_ctrl_cdev);
1642         unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1643         class_destroy(vduse_class);
1644 }
1645 module_exit(vduse_exit);
1646
1647 MODULE_LICENSE(DRV_LICENSE);
1648 MODULE_AUTHOR(DRV_AUTHOR);
1649 MODULE_DESCRIPTION(DRV_DESC);