drivers/block/virtio_blk.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 //#define DEBUG
   3 #include <linux/spinlock.h>
   4 #include <linux/slab.h>
   5 #include <linux/blkdev.h>
   6 #include <linux/hdreg.h>
   7 #include <linux/module.h>
   8 #include <linux/mutex.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/virtio.h>
  11 #include <linux/virtio_blk.h>
  12 #include <linux/scatterlist.h>
  13 #include <linux/string_helpers.h>
  14 #include <linux/idr.h>
  15 #include <linux/blk-mq.h>
  16 #include <linux/blk-mq-virtio.h>
  17 #include <linux/numa.h>
  18 #include <uapi/linux/virtio_ring.h>
  19
  20 #define PART_BITS 4
  21 #define VQ_NAME_LEN 16
  22 #define MAX_DISCARD_SEGMENTS 256u
  23
  24 /* The maximum number of sg elements that fit into a virtqueue */
  25 #define VIRTIO_BLK_MAX_SG_ELEMS 32768
  26
  27 #ifdef CONFIG_ARCH_NO_SG_CHAIN
  28 #define VIRTIO_BLK_INLINE_SG_CNT        0
  29 #else
  30 #define VIRTIO_BLK_INLINE_SG_CNT        2
  31 #endif
  32
  33 static int major;
  34 static DEFINE_IDA(vd_index_ida);
  35
  36 static struct workqueue_struct *virtblk_wq;
  37
  38 struct virtio_blk_vq {
  39         struct virtqueue *vq;
  40         spinlock_t lock;
  41         char name[VQ_NAME_LEN];
  42 } ____cacheline_aligned_in_smp;
  43
  44 struct virtio_blk {
  45         /*
  46          * This mutex must be held by anything that may run after
  47          * virtblk_remove() sets vblk->vdev to NULL.
  48          *
  49          * blk-mq, virtqueue processing, and sysfs attribute code paths are
  50          * shut down before vblk->vdev is set to NULL and therefore do not need
  51          * to hold this mutex.
  52          */
  53         struct mutex vdev_mutex;
  54         struct virtio_device *vdev;
  55
  56         /* The disk structure for the kernel. */
  57         struct gendisk *disk;
  58
  59         /* Block layer tags. */
  60         struct blk_mq_tag_set tag_set;
  61
  62         /* Process context for config space updates */
  63         struct work_struct config_work;
  64
  65         /*
  66          * Tracks references from block_device_operations open/release and
  67          * virtio_driver probe/remove so this object can be freed once no
  68          * longer in use.
  69          */
  70         refcount_t refs;
  71
  72         /* What host tells us, plus 2 for header & tailer. */
  73         unsigned int sg_elems;
  74
  75         /* Ida index - used to track minor number allocations. */
  76         int index;
  77
  78         /* num of vqs */
  79         int num_vqs;
  80         struct virtio_blk_vq *vqs;
  81 };
  82
  83 struct virtblk_req {
  84         struct virtio_blk_outhdr out_hdr;
  85         u8 status;
  86         struct sg_table sg_table;
  87         struct scatterlist sg[];
  88 };
  89
  90 static inline blk_status_t virtblk_result(struct virtblk_req *vbr)
  91 {
  92         switch (vbr->status) {
  93         case VIRTIO_BLK_S_OK:
  94                 return BLK_STS_OK;
  95         case VIRTIO_BLK_S_UNSUPP:
  96                 return BLK_STS_NOTSUPP;
  97         default:
  98                 return BLK_STS_IOERR;
  99         }
 100 }
 101
 102 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr,
 103                 struct scatterlist *data_sg, bool have_data)
 104 {
 105         struct scatterlist hdr, status, *sgs[3];
 106         unsigned int num_out = 0, num_in = 0;
 107
 108         sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));
 109         sgs[num_out++] = &hdr;
 110
 111         if (have_data) {
 112                 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))
 113                         sgs[num_out++] = data_sg;
 114                 else
 115                         sgs[num_out + num_in++] = data_sg;
 116         }
 117
 118         sg_init_one(&status, &vbr->status, sizeof(vbr->status));
 119         sgs[num_out + num_in++] = &status;
 120
 121         return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);
 122 }
 123
 124 static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap)
 125 {
 126         unsigned short segments = blk_rq_nr_discard_segments(req);
 127         unsigned short n = 0;
 128         struct virtio_blk_discard_write_zeroes *range;
 129         struct bio *bio;
 130         u32 flags = 0;
 131
 132         if (unmap)
 133                 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP;
 134
 135         range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
 136         if (!range)
 137                 return -ENOMEM;
 138
 139         /*
 140          * Single max discard segment means multi-range discard isn't
 141          * supported, and block layer only runs contiguity merge like
 142          * normal RW request. So we can't reply on bio for retrieving
 143          * each range info.
 144          */
 145         if (queue_max_discard_segments(req->q) == 1) {
 146                 range[0].flags = cpu_to_le32(flags);
 147                 range[0].num_sectors = cpu_to_le32(blk_rq_sectors(req));
 148                 range[0].sector = cpu_to_le64(blk_rq_pos(req));
 149                 n = 1;
 150         } else {
 151                 __rq_for_each_bio(bio, req) {
 152                         u64 sector = bio->bi_iter.bi_sector;
 153                         u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT;
 154
 155                         range[n].flags = cpu_to_le32(flags);
 156                         range[n].num_sectors = cpu_to_le32(num_sectors);
 157                         range[n].sector = cpu_to_le64(sector);
 158                         n++;
 159                 }
 160         }
 161
 162         WARN_ON_ONCE(n != segments);
 163
 164         req->special_vec.bv_page = virt_to_page(range);
 165         req->special_vec.bv_offset = offset_in_page(range);
 166         req->special_vec.bv_len = sizeof(*range) * segments;
 167         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 168
 169         return 0;
 170 }
 171
 172 static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr)
 173 {
 174         if (blk_rq_nr_phys_segments(req))
 175                 sg_free_table_chained(&vbr->sg_table,
 176                                       VIRTIO_BLK_INLINE_SG_CNT);
 177 }
 178
 179 static int virtblk_map_data(struct blk_mq_hw_ctx *hctx, struct request *req,
 180                 struct virtblk_req *vbr)
 181 {
 182         int err;
 183
 184         if (!blk_rq_nr_phys_segments(req))
 185                 return 0;
 186
 187         vbr->sg_table.sgl = vbr->sg;
 188         err = sg_alloc_table_chained(&vbr->sg_table,
 189                                      blk_rq_nr_phys_segments(req),
 190                                      vbr->sg_table.sgl,
 191                                      VIRTIO_BLK_INLINE_SG_CNT);
 192         if (unlikely(err))
 193                 return -ENOMEM;
 194
 195         return blk_rq_map_sg(hctx->queue, req, vbr->sg_table.sgl);
 196 }
 197
 198 static void virtblk_cleanup_cmd(struct request *req)
 199 {
 200         if (req->rq_flags & RQF_SPECIAL_PAYLOAD)
 201                 kfree(bvec_virt(&req->special_vec));
 202 }
 203
 204 static int virtblk_setup_cmd(struct virtio_device *vdev, struct request *req,
 205                 struct virtblk_req *vbr)
 206 {
 207         bool unmap = false;
 208         u32 type;
 209
 210         vbr->out_hdr.sector = 0;
 211
 212         switch (req_op(req)) {
 213         case REQ_OP_READ:
 214                 type = VIRTIO_BLK_T_IN;
 215                 vbr->out_hdr.sector = cpu_to_virtio64(vdev,
 216                                                       blk_rq_pos(req));
 217                 break;
 218         case REQ_OP_WRITE:
 219                 type = VIRTIO_BLK_T_OUT;
 220                 vbr->out_hdr.sector = cpu_to_virtio64(vdev,
 221                                                       blk_rq_pos(req));
 222                 break;
 223         case REQ_OP_FLUSH:
 224                 type = VIRTIO_BLK_T_FLUSH;
 225                 break;
 226         case REQ_OP_DISCARD:
 227                 type = VIRTIO_BLK_T_DISCARD;
 228                 break;
 229         case REQ_OP_WRITE_ZEROES:
 230                 type = VIRTIO_BLK_T_WRITE_ZEROES;
 231                 unmap = !(req->cmd_flags & REQ_NOUNMAP);
 232                 break;
 233         case REQ_OP_DRV_IN:
 234                 type = VIRTIO_BLK_T_GET_ID;
 235                 break;
 236         default:
 237                 WARN_ON_ONCE(1);
 238                 return BLK_STS_IOERR;
 239         }
 240
 241         vbr->out_hdr.type = cpu_to_virtio32(vdev, type);
 242         vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req));
 243
 244         if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) {
 245                 if (virtblk_setup_discard_write_zeroes(req, unmap))
 246                         return BLK_STS_RESOURCE;
 247         }
 248
 249         return 0;
 250 }
 251
 252 static inline void virtblk_request_done(struct request *req)
 253 {
 254         struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
 255
 256         virtblk_unmap_data(req, vbr);
 257         virtblk_cleanup_cmd(req);
 258         blk_mq_end_request(req, virtblk_result(vbr));
 259 }
 260
 261 static void virtblk_done(struct virtqueue *vq)
 262 {
 263         struct virtio_blk *vblk = vq->vdev->priv;
 264         bool req_done = false;
 265         int qid = vq->index;
 266         struct virtblk_req *vbr;
 267         unsigned long flags;
 268         unsigned int len;
 269
 270         spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
 271         do {
 272                 virtqueue_disable_cb(vq);
 273                 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
 274                         struct request *req = blk_mq_rq_from_pdu(vbr);
 275
 276                         if (likely(!blk_should_fake_timeout(req->q)))
 277                                 blk_mq_complete_request(req);
 278                         req_done = true;
 279                 }
 280                 if (unlikely(virtqueue_is_broken(vq)))
 281                         break;
 282         } while (!virtqueue_enable_cb(vq));
 283
 284         /* In case queue is stopped waiting for more buffers. */
 285         if (req_done)
 286                 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
 287         spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
 288 }
 289
 290 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx)
 291 {
 292         struct virtio_blk *vblk = hctx->queue->queuedata;
 293         struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
 294         bool kick;
 295
 296         spin_lock_irq(&vq->lock);
 297         kick = virtqueue_kick_prepare(vq->vq);
 298         spin_unlock_irq(&vq->lock);
 299
 300         if (kick)
 301                 virtqueue_notify(vq->vq);
 302 }
 303
 304 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
 305                            const struct blk_mq_queue_data *bd)
 306 {
 307         struct virtio_blk *vblk = hctx->queue->queuedata;
 308         struct request *req = bd->rq;
 309         struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
 310         unsigned long flags;
 311         int num;
 312         int qid = hctx->queue_num;
 313         int err;
 314         bool notify = false;
 315
 316         BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);
 317
 318         err = virtblk_setup_cmd(vblk->vdev, req, vbr);
 319         if (unlikely(err))
 320                 return err;
 321
 322         blk_mq_start_request(req);
 323
 324         num = virtblk_map_data(hctx, req, vbr);
 325         if (unlikely(num < 0)) {
 326                 virtblk_cleanup_cmd(req);
 327                 return BLK_STS_RESOURCE;
 328         }
 329
 330         spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
 331         err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg_table.sgl, num);
 332         if (err) {
 333                 virtqueue_kick(vblk->vqs[qid].vq);
 334                 /* Don't stop the queue if -ENOMEM: we may have failed to
 335                  * bounce the buffer due to global resource outage.
 336                  */
 337                 if (err == -ENOSPC)
 338                         blk_mq_stop_hw_queue(hctx);
 339                 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
 340                 virtblk_unmap_data(req, vbr);
 341                 virtblk_cleanup_cmd(req);
 342                 switch (err) {
 343                 case -ENOSPC:
 344                         return BLK_STS_DEV_RESOURCE;
 345                 case -ENOMEM:
 346                         return BLK_STS_RESOURCE;
 347                 default:
 348                         return BLK_STS_IOERR;
 349                 }
 350         }
 351
 352         if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))
 353                 notify = true;
 354         spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
 355
 356         if (notify)
 357                 virtqueue_notify(vblk->vqs[qid].vq);
 358         return BLK_STS_OK;
 359 }
 360
 361 /* return id (s/n) string for *disk to *id_str
 362  */
 363 static int virtblk_get_id(struct gendisk *disk, char *id_str)
 364 {
 365         struct virtio_blk *vblk = disk->private_data;
 366         struct request_queue *q = vblk->disk->queue;
 367         struct request *req;
 368         int err;
 369
 370         req = blk_get_request(q, REQ_OP_DRV_IN, 0);
 371         if (IS_ERR(req))
 372                 return PTR_ERR(req);
 373
 374         err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL);
 375         if (err)
 376                 goto out;
 377
 378         blk_execute_rq(vblk->disk, req, false);
 379         err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));
 380 out:
 381         blk_put_request(req);
 382         return err;
 383 }
 384
 385 static void virtblk_get(struct virtio_blk *vblk)
 386 {
 387         refcount_inc(&vblk->refs);
 388 }
 389
 390 static void virtblk_put(struct virtio_blk *vblk)
 391 {
 392         if (refcount_dec_and_test(&vblk->refs)) {
 393                 ida_simple_remove(&vd_index_ida, vblk->index);
 394                 mutex_destroy(&vblk->vdev_mutex);
 395                 kfree(vblk);
 396         }
 397 }
 398
 399 static int virtblk_open(struct block_device *bd, fmode_t mode)
 400 {
 401         struct virtio_blk *vblk = bd->bd_disk->private_data;
 402         int ret = 0;
 403
 404         mutex_lock(&vblk->vdev_mutex);
 405
 406         if (vblk->vdev)
 407                 virtblk_get(vblk);
 408         else
 409                 ret = -ENXIO;
 410
 411         mutex_unlock(&vblk->vdev_mutex);
 412         return ret;
 413 }
 414
 415 static void virtblk_release(struct gendisk *disk, fmode_t mode)
 416 {
 417         struct virtio_blk *vblk = disk->private_data;
 418
 419         virtblk_put(vblk);
 420 }
 421
 422 /* We provide getgeo only to please some old bootloader/partitioning tools */
 423 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 424 {
 425         struct virtio_blk *vblk = bd->bd_disk->private_data;
 426         int ret = 0;
 427
 428         mutex_lock(&vblk->vdev_mutex);
 429
 430         if (!vblk->vdev) {
 431                 ret = -ENXIO;
 432                 goto out;
 433         }
 434
 435         /* see if the host passed in geometry config */
 436         if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
 437                 virtio_cread(vblk->vdev, struct virtio_blk_config,
 438                              geometry.cylinders, &geo->cylinders);
 439                 virtio_cread(vblk->vdev, struct virtio_blk_config,
 440                              geometry.heads, &geo->heads);
 441                 virtio_cread(vblk->vdev, struct virtio_blk_config,
 442                              geometry.sectors, &geo->sectors);
 443         } else {
 444                 /* some standard values, similar to sd */
 445                 geo->heads = 1 << 6;
 446                 geo->sectors = 1 << 5;
 447                 geo->cylinders = get_capacity(bd->bd_disk) >> 11;
 448         }
 449 out:
 450         mutex_unlock(&vblk->vdev_mutex);
 451         return ret;
 452 }
 453
 454 static const struct block_device_operations virtblk_fops = {
 455         .owner  = THIS_MODULE,
 456         .open = virtblk_open,
 457         .release = virtblk_release,
 458         .getgeo = virtblk_getgeo,
 459 };
 460
 461 static int index_to_minor(int index)
 462 {
 463         return index << PART_BITS;
 464 }
 465
 466 static int minor_to_index(int minor)
 467 {
 468         return minor >> PART_BITS;
 469 }
 470
 471 static ssize_t serial_show(struct device *dev,
 472                            struct device_attribute *attr, char *buf)
 473 {
 474         struct gendisk *disk = dev_to_disk(dev);
 475         int err;
 476
 477         /* sysfs gives us a PAGE_SIZE buffer */
 478         BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES);
 479
 480         buf[VIRTIO_BLK_ID_BYTES] = '\0';
 481         err = virtblk_get_id(disk, buf);
 482         if (!err)
 483                 return strlen(buf);
 484
 485         if (err == -EIO) /* Unsupported? Make it empty. */
 486                 return 0;
 487
 488         return err;
 489 }
 490
 491 static DEVICE_ATTR_RO(serial);
 492
 493 /* The queue's logical block size must be set before calling this */
 494 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)
 495 {
 496         struct virtio_device *vdev = vblk->vdev;
 497         struct request_queue *q = vblk->disk->queue;
 498         char cap_str_2[10], cap_str_10[10];
 499         unsigned long long nblocks;
 500         u64 capacity;
 501
 502         /* Host must always specify the capacity. */
 503         virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity);
 504
 505         nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9);
 506
 507         string_get_size(nblocks, queue_logical_block_size(q),
 508                         STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
 509         string_get_size(nblocks, queue_logical_block_size(q),
 510                         STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
 511
 512         dev_notice(&vdev->dev,
 513                    "[%s] %s%llu %d-byte logical blocks (%s/%s)\n",
 514                    vblk->disk->disk_name,
 515                    resize ? "new size: " : "",
 516                    nblocks,
 517                    queue_logical_block_size(q),
 518                    cap_str_10,
 519                    cap_str_2);
 520
 521         set_capacity_and_notify(vblk->disk, capacity);
 522 }
 523
 524 static void virtblk_config_changed_work(struct work_struct *work)
 525 {
 526         struct virtio_blk *vblk =
 527                 container_of(work, struct virtio_blk, config_work);
 528
 529         virtblk_update_capacity(vblk, true);
 530 }
 531
 532 static void virtblk_config_changed(struct virtio_device *vdev)
 533 {
 534         struct virtio_blk *vblk = vdev->priv;
 535
 536         queue_work(virtblk_wq, &vblk->config_work);
 537 }
 538
 539 static int init_vq(struct virtio_blk *vblk)
 540 {
 541         int err;
 542         int i;
 543         vq_callback_t **callbacks;
 544         const char **names;
 545         struct virtqueue **vqs;
 546         unsigned short num_vqs;
 547         struct virtio_device *vdev = vblk->vdev;
 548         struct irq_affinity desc = { 0, };
 549
 550         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
 551                                    struct virtio_blk_config, num_queues,
 552                                    &num_vqs);
 553         if (err)
 554                 num_vqs = 1;
 555
 556         num_vqs = min_t(unsigned int, nr_cpu_ids, num_vqs);
 557
 558         vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);
 559         if (!vblk->vqs)
 560                 return -ENOMEM;
 561
 562         names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL);
 563         callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL);
 564         vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL);
 565         if (!names || !callbacks || !vqs) {
 566                 err = -ENOMEM;
 567                 goto out;
 568         }
 569
 570         for (i = 0; i < num_vqs; i++) {
 571                 callbacks[i] = virtblk_done;
 572                 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);
 573                 names[i] = vblk->vqs[i].name;
 574         }
 575
 576         /* Discover virtqueues and write information to configuration.  */
 577         err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
 578         if (err)
 579                 goto out;
 580
 581         for (i = 0; i < num_vqs; i++) {
 582                 spin_lock_init(&vblk->vqs[i].lock);
 583                 vblk->vqs[i].vq = vqs[i];
 584         }
 585         vblk->num_vqs = num_vqs;
 586
 587 out:
 588         kfree(vqs);
 589         kfree(callbacks);
 590         kfree(names);
 591         if (err)
 592                 kfree(vblk->vqs);
 593         return err;
 594 }
 595
 596 /*
 597  * Legacy naming scheme used for virtio devices.  We are stuck with it for
 598  * virtio blk but don't ever use it for any new driver.
 599  */
 600 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen)
 601 {
 602         const int base = 'z' - 'a' + 1;
 603         char *begin = buf + strlen(prefix);
 604         char *end = buf + buflen;
 605         char *p;
 606         int unit;
 607
 608         p = end - 1;
 609         *p = '\0';
 610         unit = base;
 611         do {
 612                 if (p == begin)
 613                         return -EINVAL;
 614                 *--p = 'a' + (index % unit);
 615                 index = (index / unit) - 1;
 616         } while (index >= 0);
 617
 618         memmove(begin, p, end - p);
 619         memcpy(buf, prefix, strlen(prefix));
 620
 621         return 0;
 622 }
 623
 624 static int virtblk_get_cache_mode(struct virtio_device *vdev)
 625 {
 626         u8 writeback;
 627         int err;
 628
 629         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE,
 630                                    struct virtio_blk_config, wce,
 631                                    &writeback);
 632
 633         /*
 634          * If WCE is not configurable and flush is not available,
 635          * assume no writeback cache is in use.
 636          */
 637         if (err)
 638                 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH);
 639
 640         return writeback;
 641 }
 642
 643 static void virtblk_update_cache_mode(struct virtio_device *vdev)
 644 {
 645         u8 writeback = virtblk_get_cache_mode(vdev);
 646         struct virtio_blk *vblk = vdev->priv;
 647
 648         blk_queue_write_cache(vblk->disk->queue, writeback, false);
 649 }
 650
 651 static const char *const virtblk_cache_types[] = {
 652         "write through", "write back"
 653 };
 654
 655 static ssize_t
 656 cache_type_store(struct device *dev, struct device_attribute *attr,
 657                  const char *buf, size_t count)
 658 {
 659         struct gendisk *disk = dev_to_disk(dev);
 660         struct virtio_blk *vblk = disk->private_data;
 661         struct virtio_device *vdev = vblk->vdev;
 662         int i;
 663
 664         BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));
 665         i = sysfs_match_string(virtblk_cache_types, buf);
 666         if (i < 0)
 667                 return i;
 668
 669         virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i);
 670         virtblk_update_cache_mode(vdev);
 671         return count;
 672 }
 673
 674 static ssize_t
 675 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf)
 676 {
 677         struct gendisk *disk = dev_to_disk(dev);
 678         struct virtio_blk *vblk = disk->private_data;
 679         u8 writeback = virtblk_get_cache_mode(vblk->vdev);
 680
 681         BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types));
 682         return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]);
 683 }
 684
 685 static DEVICE_ATTR_RW(cache_type);
 686
 687 static struct attribute *virtblk_attrs[] = {
 688         &dev_attr_serial.attr,
 689         &dev_attr_cache_type.attr,
 690         NULL,
 691 };
 692
 693 static umode_t virtblk_attrs_are_visible(struct kobject *kobj,
 694                 struct attribute *a, int n)
 695 {
 696         struct device *dev = kobj_to_dev(kobj);
 697         struct gendisk *disk = dev_to_disk(dev);
 698         struct virtio_blk *vblk = disk->private_data;
 699         struct virtio_device *vdev = vblk->vdev;
 700
 701         if (a == &dev_attr_cache_type.attr &&
 702             !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE))
 703                 return S_IRUGO;
 704
 705         return a->mode;
 706 }
 707
 708 static const struct attribute_group virtblk_attr_group = {
 709         .attrs = virtblk_attrs,
 710         .is_visible = virtblk_attrs_are_visible,
 711 };
 712
 713 static const struct attribute_group *virtblk_attr_groups[] = {
 714         &virtblk_attr_group,
 715         NULL,
 716 };
 717
 718 static int virtblk_map_queues(struct blk_mq_tag_set *set)
 719 {
 720         struct virtio_blk *vblk = set->driver_data;
 721
 722         return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT],
 723                                         vblk->vdev, 0);
 724 }
 725
 726 static const struct blk_mq_ops virtio_mq_ops = {
 727         .queue_rq       = virtio_queue_rq,
 728         .commit_rqs     = virtio_commit_rqs,
 729         .complete       = virtblk_request_done,
 730         .map_queues     = virtblk_map_queues,
 731 };
 732
 733 static unsigned int virtblk_queue_depth;
 734 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444);
 735
 736 static int virtblk_probe(struct virtio_device *vdev)
 737 {
 738         struct virtio_blk *vblk;
 739         struct request_queue *q;
 740         int err, index;
 741
 742         u32 v, blk_size, max_size, sg_elems, opt_io_size;
 743         u16 min_io_size;
 744         u8 physical_block_exp, alignment_offset;
 745         unsigned int queue_depth;
 746
 747         if (!vdev->config->get) {
 748                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
 749                         __func__);
 750                 return -EINVAL;
 751         }
 752
 753         err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS),
 754                              GFP_KERNEL);
 755         if (err < 0)
 756                 goto out;
 757         index = err;
 758
 759         /* We need to know how many segments before we allocate. */
 760         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX,
 761                                    struct virtio_blk_config, seg_max,
 762                                    &sg_elems);
 763
 764         /* We need at least one SG element, whatever they say. */
 765         if (err || !sg_elems)
 766                 sg_elems = 1;
 767
 768         /* Prevent integer overflows and honor max vq size */
 769         sg_elems = min_t(u32, sg_elems, VIRTIO_BLK_MAX_SG_ELEMS - 2);
 770
 771         /* We need extra sg elements at head and tail. */
 772         sg_elems += 2;
 773         vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);
 774         if (!vblk) {
 775                 err = -ENOMEM;
 776                 goto out_free_index;
 777         }
 778
 779         /* This reference is dropped in virtblk_remove(). */
 780         refcount_set(&vblk->refs, 1);
 781         mutex_init(&vblk->vdev_mutex);
 782
 783         vblk->vdev = vdev;
 784         vblk->sg_elems = sg_elems;
 785
 786         INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
 787
 788         err = init_vq(vblk);
 789         if (err)
 790                 goto out_free_vblk;
 791
 792         /* Default queue sizing is to fill the ring. */
 793         if (!virtblk_queue_depth) {
 794                 queue_depth = vblk->vqs[0].vq->num_free;
 795                 /* ... but without indirect descs, we use 2 descs per req */
 796                 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
 797                         queue_depth /= 2;
 798         } else {
 799                 queue_depth = virtblk_queue_depth;
 800         }
 801
 802         memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));
 803         vblk->tag_set.ops = &virtio_mq_ops;
 804         vblk->tag_set.queue_depth = queue_depth;
 805         vblk->tag_set.numa_node = NUMA_NO_NODE;
 806         vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
 807         vblk->tag_set.cmd_size =
 808                 sizeof(struct virtblk_req) +
 809                 sizeof(struct scatterlist) * VIRTIO_BLK_INLINE_SG_CNT;
 810         vblk->tag_set.driver_data = vblk;
 811         vblk->tag_set.nr_hw_queues = vblk->num_vqs;
 812
 813         err = blk_mq_alloc_tag_set(&vblk->tag_set);
 814         if (err)
 815                 goto out_free_vq;
 816
 817         vblk->disk = blk_mq_alloc_disk(&vblk->tag_set, vblk);
 818         if (IS_ERR(vblk->disk)) {
 819                 err = PTR_ERR(vblk->disk);
 820                 goto out_free_tags;
 821         }
 822         q = vblk->disk->queue;
 823
 824         virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);
 825
 826         vblk->disk->major = major;
 827         vblk->disk->first_minor = index_to_minor(index);
 828         vblk->disk->minors = 1 << PART_BITS;
 829         vblk->disk->private_data = vblk;
 830         vblk->disk->fops = &virtblk_fops;
 831         vblk->disk->flags |= GENHD_FL_EXT_DEVT;
 832         vblk->index = index;
 833
 834         /* configure queue flush support */
 835         virtblk_update_cache_mode(vdev);
 836
 837         /* If disk is read-only in the host, the guest should obey */
 838         if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO))
 839                 set_disk_ro(vblk->disk, 1);
 840
 841         /* We can handle whatever the host told us to handle. */
 842         blk_queue_max_segments(q, vblk->sg_elems-2);
 843
 844         /* No real sector limit. */
 845         blk_queue_max_hw_sectors(q, -1U);
 846
 847         max_size = virtio_max_dma_size(vdev);
 848
 849         /* Host can optionally specify maximum segment size and number of
 850          * segments. */
 851         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX,
 852                                    struct virtio_blk_config, size_max, &v);
 853         if (!err)
 854                 max_size = min(max_size, v);
 855
 856         blk_queue_max_segment_size(q, max_size);
 857
 858         /* Host can optionally specify the block size of the device */
 859         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE,
 860                                    struct virtio_blk_config, blk_size,
 861                                    &blk_size);
 862         if (!err) {
 863                 err = blk_validate_block_size(blk_size);
 864                 if (err) {
 865                         dev_err(&vdev->dev,
 866                                 "virtio_blk: invalid block size: 0x%x\n",
 867                                 blk_size);
 868                         goto out_cleanup_disk;
 869                 }
 870
 871                 blk_queue_logical_block_size(q, blk_size);
 872         } else
 873                 blk_size = queue_logical_block_size(q);
 874
 875         /* Use topology information if available */
 876         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
 877                                    struct virtio_blk_config, physical_block_exp,
 878                                    &physical_block_exp);
 879         if (!err && physical_block_exp)
 880                 blk_queue_physical_block_size(q,
 881                                 blk_size * (1 << physical_block_exp));
 882
 883         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
 884                                    struct virtio_blk_config, alignment_offset,
 885                                    &alignment_offset);
 886         if (!err && alignment_offset)
 887                 blk_queue_alignment_offset(q, blk_size * alignment_offset);
 888
 889         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
 890                                    struct virtio_blk_config, min_io_size,
 891                                    &min_io_size);
 892         if (!err && min_io_size)
 893                 blk_queue_io_min(q, blk_size * min_io_size);
 894
 895         err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
 896                                    struct virtio_blk_config, opt_io_size,
 897                                    &opt_io_size);
 898         if (!err && opt_io_size)
 899                 blk_queue_io_opt(q, blk_size * opt_io_size);
 900
 901         if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) {
 902                 virtio_cread(vdev, struct virtio_blk_config,
 903                              discard_sector_alignment, &v);
 904                 if (v)
 905                         q->limits.discard_granularity = v << SECTOR_SHIFT;
 906                 else
 907                         q->limits.discard_granularity = blk_size;
 908
 909                 virtio_cread(vdev, struct virtio_blk_config,
 910                              max_discard_sectors, &v);
 911                 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX);
 912
 913                 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg,
 914                              &v);
 915
 916                 /*
 917                  * max_discard_seg == 0 is out of spec but we always
 918                  * handled it.
 919                  */
 920                 if (!v)
 921                         v = sg_elems - 2;
 922                 blk_queue_max_discard_segments(q,
 923                                                min(v, MAX_DISCARD_SEGMENTS));
 924
 925                 blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
 926         }
 927
 928         if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) {
 929                 virtio_cread(vdev, struct virtio_blk_config,
 930                              max_write_zeroes_sectors, &v);
 931                 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX);
 932         }
 933
 934         virtblk_update_capacity(vblk, false);
 935         virtio_device_ready(vdev);
 936
 937         err = device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);
 938         if (err)
 939                 goto out_cleanup_disk;
 940
 941         return 0;
 942
 943 out_cleanup_disk:
 944         blk_cleanup_disk(vblk->disk);
 945 out_free_tags:
 946         blk_mq_free_tag_set(&vblk->tag_set);
 947 out_free_vq:
 948         vdev->config->del_vqs(vdev);
 949         kfree(vblk->vqs);
 950 out_free_vblk:
 951         kfree(vblk);
 952 out_free_index:
 953         ida_simple_remove(&vd_index_ida, index);
 954 out:
 955         return err;
 956 }
 957
 958 static void virtblk_remove(struct virtio_device *vdev)
 959 {
 960         struct virtio_blk *vblk = vdev->priv;
 961
 962         /* Make sure no work handler is accessing the device. */
 963         flush_work(&vblk->config_work);
 964
 965         del_gendisk(vblk->disk);
 966         blk_cleanup_disk(vblk->disk);
 967         blk_mq_free_tag_set(&vblk->tag_set);
 968
 969         mutex_lock(&vblk->vdev_mutex);
 970
 971         /* Stop all the virtqueues. */
 972         vdev->config->reset(vdev);
 973
 974         /* Virtqueues are stopped, nothing can use vblk->vdev anymore. */
 975         vblk->vdev = NULL;
 976
 977         vdev->config->del_vqs(vdev);
 978         kfree(vblk->vqs);
 979
 980         mutex_unlock(&vblk->vdev_mutex);
 981
 982         virtblk_put(vblk);
 983 }
 984
 985 #ifdef CONFIG_PM_SLEEP
 986 static int virtblk_freeze(struct virtio_device *vdev)
 987 {
 988         struct virtio_blk *vblk = vdev->priv;
 989
 990         /* Ensure we don't receive any more interrupts */
 991         vdev->config->reset(vdev);
 992
 993         /* Make sure no work handler is accessing the device. */
 994         flush_work(&vblk->config_work);
 995
 996         blk_mq_quiesce_queue(vblk->disk->queue);
 997
 998         vdev->config->del_vqs(vdev);
 999         kfree(vblk->vqs);
1000
1001         return 0;
1002 }
1003
1004 static int virtblk_restore(struct virtio_device *vdev)
1005 {
1006         struct virtio_blk *vblk = vdev->priv;
1007         int ret;
1008
1009         ret = init_vq(vdev->priv);
1010         if (ret)
1011                 return ret;
1012
1013         virtio_device_ready(vdev);
1014
1015         blk_mq_unquiesce_queue(vblk->disk->queue);
1016         return 0;
1017 }
1018 #endif
1019
1020 static const struct virtio_device_id id_table[] = {
1021         { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1022         { 0 },
1023 };
1024
1025 static unsigned int features_legacy[] = {
1026         VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
1027         VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
1028         VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
1029         VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
1030 }
1031 ;
1032 static unsigned int features[] = {
1033         VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
1034         VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
1035         VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
1036         VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
1037 };
1038
1039 static struct virtio_driver virtio_blk = {
1040         .feature_table                  = features,
1041         .feature_table_size             = ARRAY_SIZE(features),
1042         .feature_table_legacy           = features_legacy,
1043         .feature_table_size_legacy      = ARRAY_SIZE(features_legacy),
1044         .driver.name                    = KBUILD_MODNAME,
1045         .driver.owner                   = THIS_MODULE,
1046         .id_table                       = id_table,
1047         .probe                          = virtblk_probe,
1048         .remove                         = virtblk_remove,
1049         .config_changed                 = virtblk_config_changed,
1050 #ifdef CONFIG_PM_SLEEP
1051         .freeze                         = virtblk_freeze,
1052         .restore                        = virtblk_restore,
1053 #endif
1054 };
1055
1056 static int __init init(void)
1057 {
1058         int error;
1059
1060         virtblk_wq = alloc_workqueue("virtio-blk", 0, 0);
1061         if (!virtblk_wq)
1062                 return -ENOMEM;
1063
1064         major = register_blkdev(0, "virtblk");
1065         if (major < 0) {
1066                 error = major;
1067                 goto out_destroy_workqueue;
1068         }
1069
1070         error = register_virtio_driver(&virtio_blk);
1071         if (error)
1072                 goto out_unregister_blkdev;
1073         return 0;
1074
1075 out_unregister_blkdev:
1076         unregister_blkdev(major, "virtblk");
1077 out_destroy_workqueue:
1078         destroy_workqueue(virtblk_wq);
1079         return error;
1080 }
1081
1082 static void __exit fini(void)
1083 {
1084         unregister_virtio_driver(&virtio_blk);
1085         unregister_blkdev(major, "virtblk");
1086         destroy_workqueue(virtblk_wq);
1087 }
1088 module_init(init);
1089 module_exit(fini);
1090
1091 MODULE_DEVICE_TABLE(virtio, id_table);
1092 MODULE_DESCRIPTION("Virtio block driver");
1093 MODULE_LICENSE("GPL");