drivers/vhost/vsock.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * vhost transport for vsock
   4  *
   5  * Copyright (C) 2013-2015 Red Hat, Inc.
   6  * Author: Asias He <asias@redhat.com>
   7  *         Stefan Hajnoczi <stefanha@redhat.com>
   8  */
   9 #include <linux/miscdevice.h>
  10 #include <linux/atomic.h>
  11 #include <linux/module.h>
  12 #include <linux/mutex.h>
  13 #include <linux/vmalloc.h>
  14 #include <net/sock.h>
  15 #include <linux/virtio_vsock.h>
  16 #include <linux/vhost.h>
  17 #include <linux/hashtable.h>
  18
  19 #include <net/af_vsock.h>
  20 #include "vhost.h"
  21
  22 #define VHOST_VSOCK_DEFAULT_HOST_CID    2
  23 /* Max number of bytes transferred before requeueing the job.
  24  * Using this limit prevents one virtqueue from starving others. */
  25 #define VHOST_VSOCK_WEIGHT 0x80000
  26 /* Max number of packets transferred before requeueing the job.
  27  * Using this limit prevents one virtqueue from starving others with
  28  * small pkts.
  29  */
  30 #define VHOST_VSOCK_PKT_WEIGHT 256
  31
  32 enum {
  33         VHOST_VSOCK_FEATURES = VHOST_FEATURES |
  34                                (1ULL << VIRTIO_F_ACCESS_PLATFORM) |
  35                                (1ULL << VIRTIO_VSOCK_F_SEQPACKET)
  36 };
  37
  38 enum {
  39         VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
  40 };
  41
  42 /* Used to track all the vhost_vsock instances on the system. */
  43 static DEFINE_MUTEX(vhost_vsock_mutex);
  44 static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8);
  45
  46 struct vhost_vsock {
  47         struct vhost_dev dev;
  48         struct vhost_virtqueue vqs[2];
  49
  50         /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */
  51         struct hlist_node hash;
  52
  53         struct vhost_work send_pkt_work;
  54         spinlock_t send_pkt_list_lock;
  55         struct list_head send_pkt_list; /* host->guest pending packets */
  56
  57         atomic_t queued_replies;
  58
  59         u32 guest_cid;
  60         bool seqpacket_allow;
  61 };
  62
  63 static u32 vhost_transport_get_local_cid(void)
  64 {
  65         return VHOST_VSOCK_DEFAULT_HOST_CID;
  66 }
  67
  68 /* Callers that dereference the return value must hold vhost_vsock_mutex or the
  69  * RCU read lock.
  70  */
  71 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
  72 {
  73         struct vhost_vsock *vsock;
  74
  75         hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) {
  76                 u32 other_cid = vsock->guest_cid;
  77
  78                 /* Skip instances that have no CID yet */
  79                 if (other_cid == 0)
  80                         continue;
  81
  82                 if (other_cid == guest_cid)
  83                         return vsock;
  84
  85         }
  86
  87         return NULL;
  88 }
  89
  90 static void
  91 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
  92                             struct vhost_virtqueue *vq)
  93 {
  94         struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
  95         int pkts = 0, total_len = 0;
  96         bool added = false;
  97         bool restart_tx = false;
  98
  99         mutex_lock(&vq->mutex);
 100
 101         if (!vhost_vq_get_backend(vq))
 102                 goto out;
 103
 104         if (!vq_meta_prefetch(vq))
 105                 goto out;
 106
 107         /* Avoid further vmexits, we're already processing the virtqueue */
 108         vhost_disable_notify(&vsock->dev, vq);
 109
 110         do {
 111                 struct virtio_vsock_pkt *pkt;
 112                 struct iov_iter iov_iter;
 113                 unsigned out, in;
 114                 size_t nbytes;
 115                 size_t iov_len, payload_len;
 116                 int head;
 117                 u32 flags_to_restore = 0;
 118
 119                 spin_lock_bh(&vsock->send_pkt_list_lock);
 120                 if (list_empty(&vsock->send_pkt_list)) {
 121                         spin_unlock_bh(&vsock->send_pkt_list_lock);
 122                         vhost_enable_notify(&vsock->dev, vq);
 123                         break;
 124                 }
 125
 126                 pkt = list_first_entry(&vsock->send_pkt_list,
 127                                        struct virtio_vsock_pkt, list);
 128                 list_del_init(&pkt->list);
 129                 spin_unlock_bh(&vsock->send_pkt_list_lock);
 130
 131                 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
 132                                          &out, &in, NULL, NULL);
 133                 if (head < 0) {
 134                         spin_lock_bh(&vsock->send_pkt_list_lock);
 135                         list_add(&pkt->list, &vsock->send_pkt_list);
 136                         spin_unlock_bh(&vsock->send_pkt_list_lock);
 137                         break;
 138                 }
 139
 140                 if (head == vq->num) {
 141                         spin_lock_bh(&vsock->send_pkt_list_lock);
 142                         list_add(&pkt->list, &vsock->send_pkt_list);
 143                         spin_unlock_bh(&vsock->send_pkt_list_lock);
 144
 145                         /* We cannot finish yet if more buffers snuck in while
 146                          * re-enabling notify.
 147                          */
 148                         if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
 149                                 vhost_disable_notify(&vsock->dev, vq);
 150                                 continue;
 151                         }
 152                         break;
 153                 }
 154
 155                 if (out) {
 156                         virtio_transport_free_pkt(pkt);
 157                         vq_err(vq, "Expected 0 output buffers, got %u\n", out);
 158                         break;
 159                 }
 160
 161                 iov_len = iov_length(&vq->iov[out], in);
 162                 if (iov_len < sizeof(pkt->hdr)) {
 163                         virtio_transport_free_pkt(pkt);
 164                         vq_err(vq, "Buffer len [%zu] too small\n", iov_len);
 165                         break;
 166                 }
 167
 168                 iov_iter_init(&iov_iter, READ, &vq->iov[out], in, iov_len);
 169                 payload_len = pkt->len - pkt->off;
 170
 171                 /* If the packet is greater than the space available in the
 172                  * buffer, we split it using multiple buffers.
 173                  */
 174                 if (payload_len > iov_len - sizeof(pkt->hdr)) {
 175                         payload_len = iov_len - sizeof(pkt->hdr);
 176
 177                         /* As we are copying pieces of large packet's buffer to
 178                          * small rx buffers, headers of packets in rx queue are
 179                          * created dynamically and are initialized with header
 180                          * of current packet(except length). But in case of
 181                          * SOCK_SEQPACKET, we also must clear message delimeter
 182                          * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit
 183                          * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise,
 184                          * there will be sequence of packets with these
 185                          * bits set. After initialized header will be copied to
 186                          * rx buffer, these required bits will be restored.
 187                          */
 188                         if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOM) {
 189                                 pkt->hdr.flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);
 190                                 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM;
 191
 192                                 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SEQ_EOR) {
 193                                         pkt->hdr.flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
 194                                         flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR;
 195                                 }
 196                         }
 197                 }
 198
 199                 /* Set the correct length in the header */
 200                 pkt->hdr.len = cpu_to_le32(payload_len);
 201
 202                 nbytes = copy_to_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter);
 203                 if (nbytes != sizeof(pkt->hdr)) {
 204                         virtio_transport_free_pkt(pkt);
 205                         vq_err(vq, "Faulted on copying pkt hdr\n");
 206                         break;
 207                 }
 208
 209                 nbytes = copy_to_iter(pkt->buf + pkt->off, payload_len,
 210                                       &iov_iter);
 211                 if (nbytes != payload_len) {
 212                         virtio_transport_free_pkt(pkt);
 213                         vq_err(vq, "Faulted on copying pkt buf\n");
 214                         break;
 215                 }
 216
 217                 /* Deliver to monitoring devices all packets that we
 218                  * will transmit.
 219                  */
 220                 virtio_transport_deliver_tap_pkt(pkt);
 221
 222                 vhost_add_used(vq, head, sizeof(pkt->hdr) + payload_len);
 223                 added = true;
 224
 225                 pkt->off += payload_len;
 226                 total_len += payload_len;
 227
 228                 /* If we didn't send all the payload we can requeue the packet
 229                  * to send it with the next available buffer.
 230                  */
 231                 if (pkt->off < pkt->len) {
 232                         pkt->hdr.flags |= cpu_to_le32(flags_to_restore);
 233
 234                         /* We are queueing the same virtio_vsock_pkt to handle
 235                          * the remaining bytes, and we want to deliver it
 236                          * to monitoring devices in the next iteration.
 237                          */
 238                         pkt->tap_delivered = false;
 239
 240                         spin_lock_bh(&vsock->send_pkt_list_lock);
 241                         list_add(&pkt->list, &vsock->send_pkt_list);
 242                         spin_unlock_bh(&vsock->send_pkt_list_lock);
 243                 } else {
 244                         if (pkt->reply) {
 245                                 int val;
 246
 247                                 val = atomic_dec_return(&vsock->queued_replies);
 248
 249                                 /* Do we have resources to resume tx
 250                                  * processing?
 251                                  */
 252                                 if (val + 1 == tx_vq->num)
 253                                         restart_tx = true;
 254                         }
 255
 256                         virtio_transport_free_pkt(pkt);
 257                 }
 258         } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
 259         if (added)
 260                 vhost_signal(&vsock->dev, vq);
 261
 262 out:
 263         mutex_unlock(&vq->mutex);
 264
 265         if (restart_tx)
 266                 vhost_poll_queue(&tx_vq->poll);
 267 }
 268
 269 static void vhost_transport_send_pkt_work(struct vhost_work *work)
 270 {
 271         struct vhost_virtqueue *vq;
 272         struct vhost_vsock *vsock;
 273
 274         vsock = container_of(work, struct vhost_vsock, send_pkt_work);
 275         vq = &vsock->vqs[VSOCK_VQ_RX];
 276
 277         vhost_transport_do_send_pkt(vsock, vq);
 278 }
 279
 280 static int
 281 vhost_transport_send_pkt(struct virtio_vsock_pkt *pkt)
 282 {
 283         struct vhost_vsock *vsock;
 284         int len = pkt->len;
 285
 286         rcu_read_lock();
 287
 288         /* Find the vhost_vsock according to guest context id  */
 289         vsock = vhost_vsock_get(le64_to_cpu(pkt->hdr.dst_cid));
 290         if (!vsock) {
 291                 rcu_read_unlock();
 292                 virtio_transport_free_pkt(pkt);
 293                 return -ENODEV;
 294         }
 295
 296         if (pkt->reply)
 297                 atomic_inc(&vsock->queued_replies);
 298
 299         spin_lock_bh(&vsock->send_pkt_list_lock);
 300         list_add_tail(&pkt->list, &vsock->send_pkt_list);
 301         spin_unlock_bh(&vsock->send_pkt_list_lock);
 302
 303         vhost_work_queue(&vsock->dev, &vsock->send_pkt_work);
 304
 305         rcu_read_unlock();
 306         return len;
 307 }
 308
 309 static int
 310 vhost_transport_cancel_pkt(struct vsock_sock *vsk)
 311 {
 312         struct vhost_vsock *vsock;
 313         struct virtio_vsock_pkt *pkt, *n;
 314         int cnt = 0;
 315         int ret = -ENODEV;
 316         LIST_HEAD(freeme);
 317
 318         rcu_read_lock();
 319
 320         /* Find the vhost_vsock according to guest context id  */
 321         vsock = vhost_vsock_get(vsk->remote_addr.svm_cid);
 322         if (!vsock)
 323                 goto out;
 324
 325         spin_lock_bh(&vsock->send_pkt_list_lock);
 326         list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {
 327                 if (pkt->vsk != vsk)
 328                         continue;
 329                 list_move(&pkt->list, &freeme);
 330         }
 331         spin_unlock_bh(&vsock->send_pkt_list_lock);
 332
 333         list_for_each_entry_safe(pkt, n, &freeme, list) {
 334                 if (pkt->reply)
 335                         cnt++;
 336                 list_del(&pkt->list);
 337                 virtio_transport_free_pkt(pkt);
 338         }
 339
 340         if (cnt) {
 341                 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
 342                 int new_cnt;
 343
 344                 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
 345                 if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num)
 346                         vhost_poll_queue(&tx_vq->poll);
 347         }
 348
 349         ret = 0;
 350 out:
 351         rcu_read_unlock();
 352         return ret;
 353 }
 354
 355 static struct virtio_vsock_pkt *
 356 vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,
 357                       unsigned int out, unsigned int in)
 358 {
 359         struct virtio_vsock_pkt *pkt;
 360         struct iov_iter iov_iter;
 361         size_t nbytes;
 362         size_t len;
 363
 364         if (in != 0) {
 365                 vq_err(vq, "Expected 0 input buffers, got %u\n", in);
 366                 return NULL;
 367         }
 368
 369         pkt = kzalloc(sizeof(*pkt), GFP_KERNEL);
 370         if (!pkt)
 371                 return NULL;
 372
 373         len = iov_length(vq->iov, out);
 374         iov_iter_init(&iov_iter, WRITE, vq->iov, out, len);
 375
 376         nbytes = copy_from_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter);
 377         if (nbytes != sizeof(pkt->hdr)) {
 378                 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
 379                        sizeof(pkt->hdr), nbytes);
 380                 kfree(pkt);
 381                 return NULL;
 382         }
 383
 384         pkt->len = le32_to_cpu(pkt->hdr.len);
 385
 386         /* No payload */
 387         if (!pkt->len)
 388                 return pkt;
 389
 390         /* The pkt is too big */
 391         if (pkt->len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) {
 392                 kfree(pkt);
 393                 return NULL;
 394         }
 395
 396         pkt->buf = kvmalloc(pkt->len, GFP_KERNEL);
 397         if (!pkt->buf) {
 398                 kfree(pkt);
 399                 return NULL;
 400         }
 401
 402         pkt->buf_len = pkt->len;
 403
 404         nbytes = copy_from_iter(pkt->buf, pkt->len, &iov_iter);
 405         if (nbytes != pkt->len) {
 406                 vq_err(vq, "Expected %u byte payload, got %zu bytes\n",
 407                        pkt->len, nbytes);
 408                 virtio_transport_free_pkt(pkt);
 409                 return NULL;
 410         }
 411
 412         return pkt;
 413 }
 414
 415 /* Is there space left for replies to rx packets? */
 416 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)
 417 {
 418         struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];
 419         int val;
 420
 421         smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
 422         val = atomic_read(&vsock->queued_replies);
 423
 424         return val < vq->num;
 425 }
 426
 427 static bool vhost_transport_seqpacket_allow(u32 remote_cid);
 428
 429 static struct virtio_transport vhost_transport = {
 430         .transport = {
 431                 .module                   = THIS_MODULE,
 432
 433                 .get_local_cid            = vhost_transport_get_local_cid,
 434
 435                 .init                     = virtio_transport_do_socket_init,
 436                 .destruct                 = virtio_transport_destruct,
 437                 .release                  = virtio_transport_release,
 438                 .connect                  = virtio_transport_connect,
 439                 .shutdown                 = virtio_transport_shutdown,
 440                 .cancel_pkt               = vhost_transport_cancel_pkt,
 441
 442                 .dgram_enqueue            = virtio_transport_dgram_enqueue,
 443                 .dgram_dequeue            = virtio_transport_dgram_dequeue,
 444                 .dgram_bind               = virtio_transport_dgram_bind,
 445                 .dgram_allow              = virtio_transport_dgram_allow,
 446
 447                 .stream_enqueue           = virtio_transport_stream_enqueue,
 448                 .stream_dequeue           = virtio_transport_stream_dequeue,
 449                 .stream_has_data          = virtio_transport_stream_has_data,
 450                 .stream_has_space         = virtio_transport_stream_has_space,
 451                 .stream_rcvhiwat          = virtio_transport_stream_rcvhiwat,
 452                 .stream_is_active         = virtio_transport_stream_is_active,
 453                 .stream_allow             = virtio_transport_stream_allow,
 454
 455                 .seqpacket_dequeue        = virtio_transport_seqpacket_dequeue,
 456                 .seqpacket_enqueue        = virtio_transport_seqpacket_enqueue,
 457                 .seqpacket_allow          = vhost_transport_seqpacket_allow,
 458                 .seqpacket_has_data       = virtio_transport_seqpacket_has_data,
 459
 460                 .notify_poll_in           = virtio_transport_notify_poll_in,
 461                 .notify_poll_out          = virtio_transport_notify_poll_out,
 462                 .notify_recv_init         = virtio_transport_notify_recv_init,
 463                 .notify_recv_pre_block    = virtio_transport_notify_recv_pre_block,
 464                 .notify_recv_pre_dequeue  = virtio_transport_notify_recv_pre_dequeue,
 465                 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
 466                 .notify_send_init         = virtio_transport_notify_send_init,
 467                 .notify_send_pre_block    = virtio_transport_notify_send_pre_block,
 468                 .notify_send_pre_enqueue  = virtio_transport_notify_send_pre_enqueue,
 469                 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
 470                 .notify_buffer_size       = virtio_transport_notify_buffer_size,
 471
 472         },
 473
 474         .send_pkt = vhost_transport_send_pkt,
 475 };
 476
 477 static bool vhost_transport_seqpacket_allow(u32 remote_cid)
 478 {
 479         struct vhost_vsock *vsock;
 480         bool seqpacket_allow = false;
 481
 482         rcu_read_lock();
 483         vsock = vhost_vsock_get(remote_cid);
 484
 485         if (vsock)
 486                 seqpacket_allow = vsock->seqpacket_allow;
 487
 488         rcu_read_unlock();
 489
 490         return seqpacket_allow;
 491 }
 492
 493 static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
 494 {
 495         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 496                                                   poll.work);
 497         struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
 498                                                  dev);
 499         struct virtio_vsock_pkt *pkt;
 500         int head, pkts = 0, total_len = 0;
 501         unsigned int out, in;
 502         bool added = false;
 503
 504         mutex_lock(&vq->mutex);
 505
 506         if (!vhost_vq_get_backend(vq))
 507                 goto out;
 508
 509         if (!vq_meta_prefetch(vq))
 510                 goto out;
 511
 512         vhost_disable_notify(&vsock->dev, vq);
 513         do {
 514                 u32 len;
 515
 516                 if (!vhost_vsock_more_replies(vsock)) {
 517                         /* Stop tx until the device processes already
 518                          * pending replies.  Leave tx virtqueue
 519                          * callbacks disabled.
 520                          */
 521                         goto no_more_replies;
 522                 }
 523
 524                 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
 525                                          &out, &in, NULL, NULL);
 526                 if (head < 0)
 527                         break;
 528
 529                 if (head == vq->num) {
 530                         if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
 531                                 vhost_disable_notify(&vsock->dev, vq);
 532                                 continue;
 533                         }
 534                         break;
 535                 }
 536
 537                 pkt = vhost_vsock_alloc_pkt(vq, out, in);
 538                 if (!pkt) {
 539                         vq_err(vq, "Faulted on pkt\n");
 540                         continue;
 541                 }
 542
 543                 len = pkt->len;
 544
 545                 /* Deliver to monitoring devices all received packets */
 546                 virtio_transport_deliver_tap_pkt(pkt);
 547
 548                 /* Only accept correctly addressed packets */
 549                 if (le64_to_cpu(pkt->hdr.src_cid) == vsock->guest_cid &&
 550                     le64_to_cpu(pkt->hdr.dst_cid) ==
 551                     vhost_transport_get_local_cid())
 552                         virtio_transport_recv_pkt(&vhost_transport, pkt);
 553                 else
 554                         virtio_transport_free_pkt(pkt);
 555
 556                 len += sizeof(pkt->hdr);
 557                 vhost_add_used(vq, head, 0);
 558                 total_len += len;
 559                 added = true;
 560         } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
 561
 562 no_more_replies:
 563         if (added)
 564                 vhost_signal(&vsock->dev, vq);
 565
 566 out:
 567         mutex_unlock(&vq->mutex);
 568 }
 569
 570 static void vhost_vsock_handle_rx_kick(struct vhost_work *work)
 571 {
 572         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 573                                                 poll.work);
 574         struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
 575                                                  dev);
 576
 577         vhost_transport_do_send_pkt(vsock, vq);
 578 }
 579
 580 static int vhost_vsock_start(struct vhost_vsock *vsock)
 581 {
 582         struct vhost_virtqueue *vq;
 583         size_t i;
 584         int ret;
 585
 586         mutex_lock(&vsock->dev.mutex);
 587
 588         ret = vhost_dev_check_owner(&vsock->dev);
 589         if (ret)
 590                 goto err;
 591
 592         for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
 593                 vq = &vsock->vqs[i];
 594
 595                 mutex_lock(&vq->mutex);
 596
 597                 if (!vhost_vq_access_ok(vq)) {
 598                         ret = -EFAULT;
 599                         goto err_vq;
 600                 }
 601
 602                 if (!vhost_vq_get_backend(vq)) {
 603                         vhost_vq_set_backend(vq, vsock);
 604                         ret = vhost_vq_init_access(vq);
 605                         if (ret)
 606                                 goto err_vq;
 607                 }
 608
 609                 mutex_unlock(&vq->mutex);
 610         }
 611
 612         /* Some packets may have been queued before the device was started,
 613          * let's kick the send worker to send them.
 614          */
 615         vhost_work_queue(&vsock->dev, &vsock->send_pkt_work);
 616
 617         mutex_unlock(&vsock->dev.mutex);
 618         return 0;
 619
 620 err_vq:
 621         vhost_vq_set_backend(vq, NULL);
 622         mutex_unlock(&vq->mutex);
 623
 624         for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
 625                 vq = &vsock->vqs[i];
 626
 627                 mutex_lock(&vq->mutex);
 628                 vhost_vq_set_backend(vq, NULL);
 629                 mutex_unlock(&vq->mutex);
 630         }
 631 err:
 632         mutex_unlock(&vsock->dev.mutex);
 633         return ret;
 634 }
 635
 636 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
 637 {
 638         size_t i;
 639         int ret = 0;
 640
 641         mutex_lock(&vsock->dev.mutex);
 642
 643         if (check_owner) {
 644                 ret = vhost_dev_check_owner(&vsock->dev);
 645                 if (ret)
 646                         goto err;
 647         }
 648
 649         for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
 650                 struct vhost_virtqueue *vq = &vsock->vqs[i];
 651
 652                 mutex_lock(&vq->mutex);
 653                 vhost_vq_set_backend(vq, NULL);
 654                 mutex_unlock(&vq->mutex);
 655         }
 656
 657 err:
 658         mutex_unlock(&vsock->dev.mutex);
 659         return ret;
 660 }
 661
 662 static void vhost_vsock_free(struct vhost_vsock *vsock)
 663 {
 664         kvfree(vsock);
 665 }
 666
 667 static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
 668 {
 669         struct vhost_virtqueue **vqs;
 670         struct vhost_vsock *vsock;
 671         int ret;
 672
 673         /* This struct is large and allocation could fail, fall back to vmalloc
 674          * if there is no other way.
 675          */
 676         vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
 677         if (!vsock)
 678                 return -ENOMEM;
 679
 680         vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL);
 681         if (!vqs) {
 682                 ret = -ENOMEM;
 683                 goto out;
 684         }
 685
 686         vsock->guest_cid = 0; /* no CID assigned yet */
 687
 688         atomic_set(&vsock->queued_replies, 0);
 689
 690         vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];
 691         vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];
 692         vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;
 693         vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;
 694
 695         vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
 696                        UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
 697                        VHOST_VSOCK_WEIGHT, true, NULL);
 698
 699         file->private_data = vsock;
 700         spin_lock_init(&vsock->send_pkt_list_lock);
 701         INIT_LIST_HEAD(&vsock->send_pkt_list);
 702         vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);
 703         return 0;
 704
 705 out:
 706         vhost_vsock_free(vsock);
 707         return ret;
 708 }
 709
 710 static void vhost_vsock_flush(struct vhost_vsock *vsock)
 711 {
 712         int i;
 713
 714         for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++)
 715                 if (vsock->vqs[i].handle_kick)
 716                         vhost_poll_flush(&vsock->vqs[i].poll);
 717         vhost_work_dev_flush(&vsock->dev);
 718 }
 719
 720 static void vhost_vsock_reset_orphans(struct sock *sk)
 721 {
 722         struct vsock_sock *vsk = vsock_sk(sk);
 723
 724         /* vmci_transport.c doesn't take sk_lock here either.  At least we're
 725          * under vsock_table_lock so the sock cannot disappear while we're
 726          * executing.
 727          */
 728
 729         /* If the peer is still valid, no need to reset connection */
 730         if (vhost_vsock_get(vsk->remote_addr.svm_cid))
 731                 return;
 732
 733         /* If the close timeout is pending, let it expire.  This avoids races
 734          * with the timeout callback.
 735          */
 736         if (vsk->close_work_scheduled)
 737                 return;
 738
 739         sock_set_flag(sk, SOCK_DONE);
 740         vsk->peer_shutdown = SHUTDOWN_MASK;
 741         sk->sk_state = SS_UNCONNECTED;
 742         sk->sk_err = ECONNRESET;
 743         sk_error_report(sk);
 744 }
 745
 746 static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
 747 {
 748         struct vhost_vsock *vsock = file->private_data;
 749
 750         mutex_lock(&vhost_vsock_mutex);
 751         if (vsock->guest_cid)
 752                 hash_del_rcu(&vsock->hash);
 753         mutex_unlock(&vhost_vsock_mutex);
 754
 755         /* Wait for other CPUs to finish using vsock */
 756         synchronize_rcu();
 757
 758         /* Iterating over all connections for all CIDs to find orphans is
 759          * inefficient.  Room for improvement here. */
 760         vsock_for_each_connected_socket(&vhost_transport.transport,
 761                                         vhost_vsock_reset_orphans);
 762
 763         /* Don't check the owner, because we are in the release path, so we
 764          * need to stop the vsock device in any case.
 765          * vhost_vsock_stop() can not fail in this case, so we don't need to
 766          * check the return code.
 767          */
 768         vhost_vsock_stop(vsock, false);
 769         vhost_vsock_flush(vsock);
 770         vhost_dev_stop(&vsock->dev);
 771
 772         spin_lock_bh(&vsock->send_pkt_list_lock);
 773         while (!list_empty(&vsock->send_pkt_list)) {
 774                 struct virtio_vsock_pkt *pkt;
 775
 776                 pkt = list_first_entry(&vsock->send_pkt_list,
 777                                 struct virtio_vsock_pkt, list);
 778                 list_del_init(&pkt->list);
 779                 virtio_transport_free_pkt(pkt);
 780         }
 781         spin_unlock_bh(&vsock->send_pkt_list_lock);
 782
 783         vhost_dev_cleanup(&vsock->dev);
 784         kfree(vsock->dev.vqs);
 785         vhost_vsock_free(vsock);
 786         return 0;
 787 }
 788
 789 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
 790 {
 791         struct vhost_vsock *other;
 792
 793         /* Refuse reserved CIDs */
 794         if (guest_cid <= VMADDR_CID_HOST ||
 795             guest_cid == U32_MAX)
 796                 return -EINVAL;
 797
 798         /* 64-bit CIDs are not yet supported */
 799         if (guest_cid > U32_MAX)
 800                 return -EINVAL;
 801
 802         /* Refuse if CID is assigned to the guest->host transport (i.e. nested
 803          * VM), to make the loopback work.
 804          */
 805         if (vsock_find_cid(guest_cid))
 806                 return -EADDRINUSE;
 807
 808         /* Refuse if CID is already in use */
 809         mutex_lock(&vhost_vsock_mutex);
 810         other = vhost_vsock_get(guest_cid);
 811         if (other && other != vsock) {
 812                 mutex_unlock(&vhost_vsock_mutex);
 813                 return -EADDRINUSE;
 814         }
 815
 816         if (vsock->guest_cid)
 817                 hash_del_rcu(&vsock->hash);
 818
 819         vsock->guest_cid = guest_cid;
 820         hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);
 821         mutex_unlock(&vhost_vsock_mutex);
 822
 823         return 0;
 824 }
 825
 826 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 827 {
 828         struct vhost_virtqueue *vq;
 829         int i;
 830
 831         if (features & ~VHOST_VSOCK_FEATURES)
 832                 return -EOPNOTSUPP;
 833
 834         mutex_lock(&vsock->dev.mutex);
 835         if ((features & (1 << VHOST_F_LOG_ALL)) &&
 836             !vhost_log_access_ok(&vsock->dev)) {
 837                 goto err;
 838         }
 839
 840         if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
 841                 if (vhost_init_device_iotlb(&vsock->dev, true))
 842                         goto err;
 843         }
 844
 845         if (features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET))
 846                 vsock->seqpacket_allow = true;
 847
 848         for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
 849                 vq = &vsock->vqs[i];
 850                 mutex_lock(&vq->mutex);
 851                 vq->acked_features = features;
 852                 mutex_unlock(&vq->mutex);
 853         }
 854         mutex_unlock(&vsock->dev.mutex);
 855         return 0;
 856
 857 err:
 858         mutex_unlock(&vsock->dev.mutex);
 859         return -EFAULT;
 860 }
 861
 862 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
 863                                   unsigned long arg)
 864 {
 865         struct vhost_vsock *vsock = f->private_data;
 866         void __user *argp = (void __user *)arg;
 867         u64 guest_cid;
 868         u64 features;
 869         int start;
 870         int r;
 871
 872         switch (ioctl) {
 873         case VHOST_VSOCK_SET_GUEST_CID:
 874                 if (copy_from_user(&guest_cid, argp, sizeof(guest_cid)))
 875                         return -EFAULT;
 876                 return vhost_vsock_set_cid(vsock, guest_cid);
 877         case VHOST_VSOCK_SET_RUNNING:
 878                 if (copy_from_user(&start, argp, sizeof(start)))
 879                         return -EFAULT;
 880                 if (start)
 881                         return vhost_vsock_start(vsock);
 882                 else
 883                         return vhost_vsock_stop(vsock, true);
 884         case VHOST_GET_FEATURES:
 885                 features = VHOST_VSOCK_FEATURES;
 886                 if (copy_to_user(argp, &features, sizeof(features)))
 887                         return -EFAULT;
 888                 return 0;
 889         case VHOST_SET_FEATURES:
 890                 if (copy_from_user(&features, argp, sizeof(features)))
 891                         return -EFAULT;
 892                 return vhost_vsock_set_features(vsock, features);
 893         case VHOST_GET_BACKEND_FEATURES:
 894                 features = VHOST_VSOCK_BACKEND_FEATURES;
 895                 if (copy_to_user(argp, &features, sizeof(features)))
 896                         return -EFAULT;
 897                 return 0;
 898         case VHOST_SET_BACKEND_FEATURES:
 899                 if (copy_from_user(&features, argp, sizeof(features)))
 900                         return -EFAULT;
 901                 if (features & ~VHOST_VSOCK_BACKEND_FEATURES)
 902                         return -EOPNOTSUPP;
 903                 vhost_set_backend_features(&vsock->dev, features);
 904                 return 0;
 905         default:
 906                 mutex_lock(&vsock->dev.mutex);
 907                 r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
 908                 if (r == -ENOIOCTLCMD)
 909                         r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);
 910                 else
 911                         vhost_vsock_flush(vsock);
 912                 mutex_unlock(&vsock->dev.mutex);
 913                 return r;
 914         }
 915 }
 916
 917 static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
 918 {
 919         struct file *file = iocb->ki_filp;
 920         struct vhost_vsock *vsock = file->private_data;
 921         struct vhost_dev *dev = &vsock->dev;
 922         int noblock = file->f_flags & O_NONBLOCK;
 923
 924         return vhost_chr_read_iter(dev, to, noblock);
 925 }
 926
 927 static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb,
 928                                         struct iov_iter *from)
 929 {
 930         struct file *file = iocb->ki_filp;
 931         struct vhost_vsock *vsock = file->private_data;
 932         struct vhost_dev *dev = &vsock->dev;
 933
 934         return vhost_chr_write_iter(dev, from);
 935 }
 936
 937 static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait)
 938 {
 939         struct vhost_vsock *vsock = file->private_data;
 940         struct vhost_dev *dev = &vsock->dev;
 941
 942         return vhost_chr_poll(file, dev, wait);
 943 }
 944
 945 static const struct file_operations vhost_vsock_fops = {
 946         .owner          = THIS_MODULE,
 947         .open           = vhost_vsock_dev_open,
 948         .release        = vhost_vsock_dev_release,
 949         .llseek         = noop_llseek,
 950         .unlocked_ioctl = vhost_vsock_dev_ioctl,
 951         .compat_ioctl   = compat_ptr_ioctl,
 952         .read_iter      = vhost_vsock_chr_read_iter,
 953         .write_iter     = vhost_vsock_chr_write_iter,
 954         .poll           = vhost_vsock_chr_poll,
 955 };
 956
 957 static struct miscdevice vhost_vsock_misc = {
 958         .minor = VHOST_VSOCK_MINOR,
 959         .name = "vhost-vsock",
 960         .fops = &vhost_vsock_fops,
 961 };
 962
 963 static int __init vhost_vsock_init(void)
 964 {
 965         int ret;
 966
 967         ret = vsock_core_register(&vhost_transport.transport,
 968                                   VSOCK_TRANSPORT_F_H2G);
 969         if (ret < 0)
 970                 return ret;
 971         return misc_register(&vhost_vsock_misc);
 972 };
 973
 974 static void __exit vhost_vsock_exit(void)
 975 {
 976         misc_deregister(&vhost_vsock_misc);
 977         vsock_core_unregister(&vhost_transport.transport);
 978 };
 979
 980 module_init(vhost_vsock_init);
 981 module_exit(vhost_vsock_exit);
 982 MODULE_LICENSE("GPL v2");
 983 MODULE_AUTHOR("Asias He");
 984 MODULE_DESCRIPTION("vhost transport for vsock ");
 985 MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR);
 986 MODULE_ALIAS("devname:vhost-vsock");