drivers/block/nvme-core.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #include <linux/nvme.h>
  20 #include <linux/bio.h>
  21 #include <linux/bitops.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/delay.h>
  24 #include <linux/errno.h>
  25 #include <linux/fs.h>
  26 #include <linux/genhd.h>
  27 #include <linux/idr.h>
  28 #include <linux/init.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/io.h>
  31 #include <linux/kdev_t.h>
  32 #include <linux/kthread.h>
  33 #include <linux/kernel.h>
  34 #include <linux/mm.h>
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/pci.h>
  38 #include <linux/poison.h>
  39 #include <linux/ptrace.h>
  40 #include <linux/sched.h>
  41 #include <linux/slab.h>
  42 #include <linux/types.h>
  43 #include <scsi/sg.h>
  44 #include <asm-generic/io-64-nonatomic-lo-hi.h>
  45
  46 #define NVME_Q_DEPTH 1024
  47 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
  48 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
  49 #define NVME_MINORS 64
  50 #define ADMIN_TIMEOUT   (60 * HZ)
  51
  52 static int nvme_major;
  53 module_param(nvme_major, int, 0);
  54
  55 static int use_threaded_interrupts;
  56 module_param(use_threaded_interrupts, int, 0);
  57
  58 static DEFINE_SPINLOCK(dev_list_lock);
  59 static LIST_HEAD(dev_list);
  60 static struct task_struct *nvme_thread;
  61
  62 /*
  63  * An NVM Express queue.  Each device has at least two (one for admin
  64  * commands and one for I/O commands).
  65  */
  66 struct nvme_queue {
  67         struct device *q_dmadev;
  68         struct nvme_dev *dev;
  69         spinlock_t q_lock;
  70         struct nvme_command *sq_cmds;
  71         volatile struct nvme_completion *cqes;
  72         dma_addr_t sq_dma_addr;
  73         dma_addr_t cq_dma_addr;
  74         wait_queue_head_t sq_full;
  75         wait_queue_t sq_cong_wait;
  76         struct bio_list sq_cong;
  77         u32 __iomem *q_db;
  78         u16 q_depth;
  79         u16 cq_vector;
  80         u16 sq_head;
  81         u16 sq_tail;
  82         u16 cq_head;
  83         u8 cq_phase;
  84         u8 cqe_seen;
  85         u8 q_suspended;
  86         unsigned long cmdid_data[];
  87 };
  88
  89 /*
  90  * Check we didin't inadvertently grow the command struct
  91  */
  92 static inline void _nvme_check_size(void)
  93 {
  94         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
  95         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
  96         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
  97         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
  98         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
  99         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
 100         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
 101         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
 102         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
 103         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
 104         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 105 }
 106
 107 typedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
 108                                                 struct nvme_completion *);
 109
 110 struct nvme_cmd_info {
 111         nvme_completion_fn fn;
 112         void *ctx;
 113         unsigned long timeout;
 114 };
 115
 116 static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq)
 117 {
 118         return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)];
 119 }
 120
 121 static unsigned nvme_queue_extra(int depth)
 122 {
 123         return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info));
 124 }
 125
 126 /**
 127  * alloc_cmdid() - Allocate a Command ID
 128  * @nvmeq: The queue that will be used for this command
 129  * @ctx: A pointer that will be passed to the handler
 130  * @handler: The function to call on completion
 131  *
 132  * Allocate a Command ID for a queue.  The data passed in will
 133  * be passed to the completion handler.  This is implemented by using
 134  * the bottom two bits of the ctx pointer to store the handler ID.
 135  * Passing in a pointer that's not 4-byte aligned will cause a BUG.
 136  * We can change this if it becomes a problem.
 137  *
 138  * May be called with local interrupts disabled and the q_lock held,
 139  * or with interrupts enabled and no locks held.
 140  */
 141 static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx,
 142                                 nvme_completion_fn handler, unsigned timeout)
 143 {
 144         int depth = nvmeq->q_depth - 1;
 145         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 146         int cmdid;
 147
 148         do {
 149                 cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
 150                 if (cmdid >= depth)
 151                         return -EBUSY;
 152         } while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
 153
 154         info[cmdid].fn = handler;
 155         info[cmdid].ctx = ctx;
 156         info[cmdid].timeout = jiffies + timeout;
 157         return cmdid;
 158 }
 159
 160 static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
 161                                 nvme_completion_fn handler, unsigned timeout)
 162 {
 163         int cmdid;
 164         wait_event_killable(nvmeq->sq_full,
 165                 (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0);
 166         return (cmdid < 0) ? -EINTR : cmdid;
 167 }
 168
 169 /* Special values must be less than 0x1000 */
 170 #define CMD_CTX_BASE            ((void *)POISON_POINTER_DELTA)
 171 #define CMD_CTX_CANCELLED       (0x30C + CMD_CTX_BASE)
 172 #define CMD_CTX_COMPLETED       (0x310 + CMD_CTX_BASE)
 173 #define CMD_CTX_INVALID         (0x314 + CMD_CTX_BASE)
 174 #define CMD_CTX_FLUSH           (0x318 + CMD_CTX_BASE)
 175
 176 static void special_completion(struct nvme_dev *dev, void *ctx,
 177                                                 struct nvme_completion *cqe)
 178 {
 179         if (ctx == CMD_CTX_CANCELLED)
 180                 return;
 181         if (ctx == CMD_CTX_FLUSH)
 182                 return;
 183         if (ctx == CMD_CTX_COMPLETED) {
 184                 dev_warn(&dev->pci_dev->dev,
 185                                 "completed id %d twice on queue %d\n",
 186                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
 187                 return;
 188         }
 189         if (ctx == CMD_CTX_INVALID) {
 190                 dev_warn(&dev->pci_dev->dev,
 191                                 "invalid id %d completed on queue %d\n",
 192                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
 193                 return;
 194         }
 195
 196         dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
 197 }
 198
 199 /*
 200  * Called with local interrupts disabled and the q_lock held.  May not sleep.
 201  */
 202 static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid,
 203                                                 nvme_completion_fn *fn)
 204 {
 205         void *ctx;
 206         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 207
 208         if (cmdid >= nvmeq->q_depth) {
 209                 *fn = special_completion;
 210                 return CMD_CTX_INVALID;
 211         }
 212         if (fn)
 213                 *fn = info[cmdid].fn;
 214         ctx = info[cmdid].ctx;
 215         info[cmdid].fn = special_completion;
 216         info[cmdid].ctx = CMD_CTX_COMPLETED;
 217         clear_bit(cmdid, nvmeq->cmdid_data);
 218         wake_up(&nvmeq->sq_full);
 219         return ctx;
 220 }
 221
 222 static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
 223                                                 nvme_completion_fn *fn)
 224 {
 225         void *ctx;
 226         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 227         if (fn)
 228                 *fn = info[cmdid].fn;
 229         ctx = info[cmdid].ctx;
 230         info[cmdid].fn = special_completion;
 231         info[cmdid].ctx = CMD_CTX_CANCELLED;
 232         return ctx;
 233 }
 234
 235 struct nvme_queue *get_nvmeq(struct nvme_dev *dev)
 236 {
 237         return dev->queues[get_cpu() + 1];
 238 }
 239
 240 void put_nvmeq(struct nvme_queue *nvmeq)
 241 {
 242         put_cpu();
 243 }
 244
 245 /**
 246  * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
 247  * @nvmeq: The queue to use
 248  * @cmd: The command to send
 249  *
 250  * Safe to use from interrupt context
 251  */
 252 static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
 253 {
 254         unsigned long flags;
 255         u16 tail;
 256         spin_lock_irqsave(&nvmeq->q_lock, flags);
 257         tail = nvmeq->sq_tail;
 258         memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
 259         if (++tail == nvmeq->q_depth)
 260                 tail = 0;
 261         writel(tail, nvmeq->q_db);
 262         nvmeq->sq_tail = tail;
 263         spin_unlock_irqrestore(&nvmeq->q_lock, flags);
 264
 265         return 0;
 266 }
 267
 268 static __le64 **iod_list(struct nvme_iod *iod)
 269 {
 270         return ((void *)iod) + iod->offset;
 271 }
 272
 273 /*
 274  * Will slightly overestimate the number of pages needed.  This is OK
 275  * as it only leads to a small amount of wasted memory for the lifetime of
 276  * the I/O.
 277  */
 278 static int nvme_npages(unsigned size)
 279 {
 280         unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE);
 281         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 282 }
 283
 284 static struct nvme_iod *
 285 nvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp)
 286 {
 287         struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
 288                                 sizeof(__le64 *) * nvme_npages(nbytes) +
 289                                 sizeof(struct scatterlist) * nseg, gfp);
 290
 291         if (iod) {
 292                 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
 293                 iod->npages = -1;
 294                 iod->length = nbytes;
 295                 iod->nents = 0;
 296                 iod->start_time = jiffies;
 297         }
 298
 299         return iod;
 300 }
 301
 302 void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
 303 {
 304         const int last_prp = PAGE_SIZE / 8 - 1;
 305         int i;
 306         __le64 **list = iod_list(iod);
 307         dma_addr_t prp_dma = iod->first_dma;
 308
 309         if (iod->npages == 0)
 310                 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
 311         for (i = 0; i < iod->npages; i++) {
 312                 __le64 *prp_list = list[i];
 313                 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
 314                 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
 315                 prp_dma = next_prp_dma;
 316         }
 317         kfree(iod);
 318 }
 319
 320 static void nvme_start_io_acct(struct bio *bio)
 321 {
 322         struct gendisk *disk = bio->bi_bdev->bd_disk;
 323         const int rw = bio_data_dir(bio);
 324         int cpu = part_stat_lock();
 325         part_round_stats(cpu, &disk->part0);
 326         part_stat_inc(cpu, &disk->part0, ios[rw]);
 327         part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio));
 328         part_inc_in_flight(&disk->part0, rw);
 329         part_stat_unlock();
 330 }
 331
 332 static void nvme_end_io_acct(struct bio *bio, unsigned long start_time)
 333 {
 334         struct gendisk *disk = bio->bi_bdev->bd_disk;
 335         const int rw = bio_data_dir(bio);
 336         unsigned long duration = jiffies - start_time;
 337         int cpu = part_stat_lock();
 338         part_stat_add(cpu, &disk->part0, ticks[rw], duration);
 339         part_round_stats(cpu, &disk->part0);
 340         part_dec_in_flight(&disk->part0, rw);
 341         part_stat_unlock();
 342 }
 343
 344 static void bio_completion(struct nvme_dev *dev, void *ctx,
 345                                                 struct nvme_completion *cqe)
 346 {
 347         struct nvme_iod *iod = ctx;
 348         struct bio *bio = iod->private;
 349         u16 status = le16_to_cpup(&cqe->status) >> 1;
 350
 351         if (iod->nents) {
 352                 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
 353                         bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 354                 nvme_end_io_acct(bio, iod->start_time);
 355         }
 356         nvme_free_iod(dev, iod);
 357         if (status)
 358                 bio_endio(bio, -EIO);
 359         else
 360                 bio_endio(bio, 0);
 361 }
 362
 363 /* length is in bytes.  gfp flags indicates whether we may sleep. */
 364 int nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
 365                         struct nvme_iod *iod, int total_len, gfp_t gfp)
 366 {
 367         struct dma_pool *pool;
 368         int length = total_len;
 369         struct scatterlist *sg = iod->sg;
 370         int dma_len = sg_dma_len(sg);
 371         u64 dma_addr = sg_dma_address(sg);
 372         int offset = offset_in_page(dma_addr);
 373         __le64 *prp_list;
 374         __le64 **list = iod_list(iod);
 375         dma_addr_t prp_dma;
 376         int nprps, i;
 377
 378         cmd->prp1 = cpu_to_le64(dma_addr);
 379         length -= (PAGE_SIZE - offset);
 380         if (length <= 0)
 381                 return total_len;
 382
 383         dma_len -= (PAGE_SIZE - offset);
 384         if (dma_len) {
 385                 dma_addr += (PAGE_SIZE - offset);
 386         } else {
 387                 sg = sg_next(sg);
 388                 dma_addr = sg_dma_address(sg);
 389                 dma_len = sg_dma_len(sg);
 390         }
 391
 392         if (length <= PAGE_SIZE) {
 393                 cmd->prp2 = cpu_to_le64(dma_addr);
 394                 return total_len;
 395         }
 396
 397         nprps = DIV_ROUND_UP(length, PAGE_SIZE);
 398         if (nprps <= (256 / 8)) {
 399                 pool = dev->prp_small_pool;
 400                 iod->npages = 0;
 401         } else {
 402                 pool = dev->prp_page_pool;
 403                 iod->npages = 1;
 404         }
 405
 406         prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
 407         if (!prp_list) {
 408                 cmd->prp2 = cpu_to_le64(dma_addr);
 409                 iod->npages = -1;
 410                 return (total_len - length) + PAGE_SIZE;
 411         }
 412         list[0] = prp_list;
 413         iod->first_dma = prp_dma;
 414         cmd->prp2 = cpu_to_le64(prp_dma);
 415         i = 0;
 416         for (;;) {
 417                 if (i == PAGE_SIZE / 8) {
 418                         __le64 *old_prp_list = prp_list;
 419                         prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
 420                         if (!prp_list)
 421                                 return total_len - length;
 422                         list[iod->npages++] = prp_list;
 423                         prp_list[0] = old_prp_list[i - 1];
 424                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 425                         i = 1;
 426                 }
 427                 prp_list[i++] = cpu_to_le64(dma_addr);
 428                 dma_len -= PAGE_SIZE;
 429                 dma_addr += PAGE_SIZE;
 430                 length -= PAGE_SIZE;
 431                 if (length <= 0)
 432                         break;
 433                 if (dma_len > 0)
 434                         continue;
 435                 BUG_ON(dma_len < 0);
 436                 sg = sg_next(sg);
 437                 dma_addr = sg_dma_address(sg);
 438                 dma_len = sg_dma_len(sg);
 439         }
 440
 441         return total_len;
 442 }
 443
 444 struct nvme_bio_pair {
 445         struct bio b1, b2, *parent;
 446         struct bio_vec *bv1, *bv2;
 447         int err;
 448         atomic_t cnt;
 449 };
 450
 451 static void nvme_bio_pair_endio(struct bio *bio, int err)
 452 {
 453         struct nvme_bio_pair *bp = bio->bi_private;
 454
 455         if (err)
 456                 bp->err = err;
 457
 458         if (atomic_dec_and_test(&bp->cnt)) {
 459                 bio_endio(bp->parent, bp->err);
 460                 kfree(bp->bv1);
 461                 kfree(bp->bv2);
 462                 kfree(bp);
 463         }
 464 }
 465
 466 static struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx,
 467                                                         int len, int offset)
 468 {
 469         struct nvme_bio_pair *bp;
 470
 471         BUG_ON(len > bio->bi_size);
 472         BUG_ON(idx > bio->bi_vcnt);
 473
 474         bp = kmalloc(sizeof(*bp), GFP_ATOMIC);
 475         if (!bp)
 476                 return NULL;
 477         bp->err = 0;
 478
 479         bp->b1 = *bio;
 480         bp->b2 = *bio;
 481
 482         bp->b1.bi_size = len;
 483         bp->b2.bi_size -= len;
 484         bp->b1.bi_vcnt = idx;
 485         bp->b2.bi_idx = idx;
 486         bp->b2.bi_sector += len >> 9;
 487
 488         if (offset) {
 489                 bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
 490                                                                 GFP_ATOMIC);
 491                 if (!bp->bv1)
 492                         goto split_fail_1;
 493
 494                 bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
 495                                                                 GFP_ATOMIC);
 496                 if (!bp->bv2)
 497                         goto split_fail_2;
 498
 499                 memcpy(bp->bv1, bio->bi_io_vec,
 500                         bio->bi_max_vecs * sizeof(struct bio_vec));
 501                 memcpy(bp->bv2, bio->bi_io_vec,
 502                         bio->bi_max_vecs * sizeof(struct bio_vec));
 503
 504                 bp->b1.bi_io_vec = bp->bv1;
 505                 bp->b2.bi_io_vec = bp->bv2;
 506                 bp->b2.bi_io_vec[idx].bv_offset += offset;
 507                 bp->b2.bi_io_vec[idx].bv_len -= offset;
 508                 bp->b1.bi_io_vec[idx].bv_len = offset;
 509                 bp->b1.bi_vcnt++;
 510         } else
 511                 bp->bv1 = bp->bv2 = NULL;
 512
 513         bp->b1.bi_private = bp;
 514         bp->b2.bi_private = bp;
 515
 516         bp->b1.bi_end_io = nvme_bio_pair_endio;
 517         bp->b2.bi_end_io = nvme_bio_pair_endio;
 518
 519         bp->parent = bio;
 520         atomic_set(&bp->cnt, 2);
 521
 522         return bp;
 523
 524  split_fail_2:
 525         kfree(bp->bv1);
 526  split_fail_1:
 527         kfree(bp);
 528         return NULL;
 529 }
 530
 531 static int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
 532                                                 int idx, int len, int offset)
 533 {
 534         struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset);
 535         if (!bp)
 536                 return -ENOMEM;
 537
 538         if (bio_list_empty(&nvmeq->sq_cong))
 539                 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
 540         bio_list_add(&nvmeq->sq_cong, &bp->b1);
 541         bio_list_add(&nvmeq->sq_cong, &bp->b2);
 542
 543         return 0;
 544 }
 545
 546 /* NVMe scatterlists require no holes in the virtual address */
 547 #define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2)   ((vec2)->bv_offset || \
 548                         (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE))
 549
 550 static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
 551                 struct bio *bio, enum dma_data_direction dma_dir, int psegs)
 552 {
 553         struct bio_vec *bvec, *bvprv = NULL;
 554         struct scatterlist *sg = NULL;
 555         int i, length = 0, nsegs = 0, split_len = bio->bi_size;
 556
 557         if (nvmeq->dev->stripe_size)
 558                 split_len = nvmeq->dev->stripe_size -
 559                         ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1));
 560
 561         sg_init_table(iod->sg, psegs);
 562         bio_for_each_segment(bvec, bio, i) {
 563                 if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) {
 564                         sg->length += bvec->bv_len;
 565                 } else {
 566                         if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec))
 567                                 return nvme_split_and_submit(bio, nvmeq, i,
 568                                                                 length, 0);
 569
 570                         sg = sg ? sg + 1 : iod->sg;
 571                         sg_set_page(sg, bvec->bv_page, bvec->bv_len,
 572                                                         bvec->bv_offset);
 573                         nsegs++;
 574                 }
 575
 576                 if (split_len - length < bvec->bv_len)
 577                         return nvme_split_and_submit(bio, nvmeq, i, split_len,
 578                                                         split_len - length);
 579                 length += bvec->bv_len;
 580                 bvprv = bvec;
 581         }
 582         iod->nents = nsegs;
 583         sg_mark_end(sg);
 584         if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0)
 585                 return -ENOMEM;
 586
 587         BUG_ON(length != bio->bi_size);
 588         return length;
 589 }
 590
 591 /*
 592  * We reuse the small pool to allocate the 16-byte range here as it is not
 593  * worth having a special pool for these or additional cases to handle freeing
 594  * the iod.
 595  */
 596 static int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 597                 struct bio *bio, struct nvme_iod *iod, int cmdid)
 598 {
 599         struct nvme_dsm_range *range;
 600         struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 601
 602         range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
 603                                                         &iod->first_dma);
 604         if (!range)
 605                 return -ENOMEM;
 606
 607         iod_list(iod)[0] = (__le64 *)range;
 608         iod->npages = 0;
 609
 610         range->cattr = cpu_to_le32(0);
 611         range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift);
 612         range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
 613
 614         memset(cmnd, 0, sizeof(*cmnd));
 615         cmnd->dsm.opcode = nvme_cmd_dsm;
 616         cmnd->dsm.command_id = cmdid;
 617         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
 618         cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
 619         cmnd->dsm.nr = 0;
 620         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 621
 622         if (++nvmeq->sq_tail == nvmeq->q_depth)
 623                 nvmeq->sq_tail = 0;
 624         writel(nvmeq->sq_tail, nvmeq->q_db);
 625
 626         return 0;
 627 }
 628
 629 static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 630                                                                 int cmdid)
 631 {
 632         struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 633
 634         memset(cmnd, 0, sizeof(*cmnd));
 635         cmnd->common.opcode = nvme_cmd_flush;
 636         cmnd->common.command_id = cmdid;
 637         cmnd->common.nsid = cpu_to_le32(ns->ns_id);
 638
 639         if (++nvmeq->sq_tail == nvmeq->q_depth)
 640                 nvmeq->sq_tail = 0;
 641         writel(nvmeq->sq_tail, nvmeq->q_db);
 642
 643         return 0;
 644 }
 645
 646 int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns)
 647 {
 648         int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH,
 649                                         special_completion, NVME_IO_TIMEOUT);
 650         if (unlikely(cmdid < 0))
 651                 return cmdid;
 652
 653         return nvme_submit_flush(nvmeq, ns, cmdid);
 654 }
 655
 656 /*
 657  * Called with local interrupts disabled and the q_lock held.  May not sleep.
 658  */
 659 static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 660                                                                 struct bio *bio)
 661 {
 662         struct nvme_command *cmnd;
 663         struct nvme_iod *iod;
 664         enum dma_data_direction dma_dir;
 665         int cmdid, length, result;
 666         u16 control;
 667         u32 dsmgmt;
 668         int psegs = bio_phys_segments(ns->queue, bio);
 669
 670         if ((bio->bi_rw & REQ_FLUSH) && psegs) {
 671                 result = nvme_submit_flush_data(nvmeq, ns);
 672                 if (result)
 673                         return result;
 674         }
 675
 676         result = -ENOMEM;
 677         iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC);
 678         if (!iod)
 679                 goto nomem;
 680         iod->private = bio;
 681
 682         result = -EBUSY;
 683         cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT);
 684         if (unlikely(cmdid < 0))
 685                 goto free_iod;
 686
 687         if (bio->bi_rw & REQ_DISCARD) {
 688                 result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid);
 689                 if (result)
 690                         goto free_cmdid;
 691                 return result;
 692         }
 693         if ((bio->bi_rw & REQ_FLUSH) && !psegs)
 694                 return nvme_submit_flush(nvmeq, ns, cmdid);
 695
 696         control = 0;
 697         if (bio->bi_rw & REQ_FUA)
 698                 control |= NVME_RW_FUA;
 699         if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 700                 control |= NVME_RW_LR;
 701
 702         dsmgmt = 0;
 703         if (bio->bi_rw & REQ_RAHEAD)
 704                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 705
 706         cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 707
 708         memset(cmnd, 0, sizeof(*cmnd));
 709         if (bio_data_dir(bio)) {
 710                 cmnd->rw.opcode = nvme_cmd_write;
 711                 dma_dir = DMA_TO_DEVICE;
 712         } else {
 713                 cmnd->rw.opcode = nvme_cmd_read;
 714                 dma_dir = DMA_FROM_DEVICE;
 715         }
 716
 717         result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs);
 718         if (result <= 0)
 719                 goto free_cmdid;
 720         length = result;
 721
 722         cmnd->rw.command_id = cmdid;
 723         cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
 724         length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length,
 725                                                                 GFP_ATOMIC);
 726         cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
 727         cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1);
 728         cmnd->rw.control = cpu_to_le16(control);
 729         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 730
 731         nvme_start_io_acct(bio);
 732         if (++nvmeq->sq_tail == nvmeq->q_depth)
 733                 nvmeq->sq_tail = 0;
 734         writel(nvmeq->sq_tail, nvmeq->q_db);
 735
 736         return 0;
 737
 738  free_cmdid:
 739         free_cmdid(nvmeq, cmdid, NULL);
 740  free_iod:
 741         nvme_free_iod(nvmeq->dev, iod);
 742  nomem:
 743         return result;
 744 }
 745
 746 static int nvme_process_cq(struct nvme_queue *nvmeq)
 747 {
 748         u16 head, phase;
 749
 750         head = nvmeq->cq_head;
 751         phase = nvmeq->cq_phase;
 752
 753         for (;;) {
 754                 void *ctx;
 755                 nvme_completion_fn fn;
 756                 struct nvme_completion cqe = nvmeq->cqes[head];
 757                 if ((le16_to_cpu(cqe.status) & 1) != phase)
 758                         break;
 759                 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
 760                 if (++head == nvmeq->q_depth) {
 761                         head = 0;
 762                         phase = !phase;
 763                 }
 764
 765                 ctx = free_cmdid(nvmeq, cqe.command_id, &fn);
 766                 fn(nvmeq->dev, ctx, &cqe);
 767         }
 768
 769         /* If the controller ignores the cq head doorbell and continuously
 770          * writes to the queue, it is theoretically possible to wrap around
 771          * the queue twice and mistakenly return IRQ_NONE.  Linux only
 772          * requires that 0.1% of your interrupts are handled, so this isn't
 773          * a big problem.
 774          */
 775         if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
 776                 return 0;
 777
 778         writel(head, nvmeq->q_db + (1 << nvmeq->dev->db_stride));
 779         nvmeq->cq_head = head;
 780         nvmeq->cq_phase = phase;
 781
 782         nvmeq->cqe_seen = 1;
 783         return 1;
 784 }
 785
 786 static void nvme_make_request(struct request_queue *q, struct bio *bio)
 787 {
 788         struct nvme_ns *ns = q->queuedata;
 789         struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
 790         int result = -EBUSY;
 791
 792         spin_lock_irq(&nvmeq->q_lock);
 793         if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong))
 794                 result = nvme_submit_bio_queue(nvmeq, ns, bio);
 795         if (unlikely(result)) {
 796                 if (bio_list_empty(&nvmeq->sq_cong))
 797                         add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
 798                 bio_list_add(&nvmeq->sq_cong, bio);
 799         }
 800
 801         nvme_process_cq(nvmeq);
 802         spin_unlock_irq(&nvmeq->q_lock);
 803         put_nvmeq(nvmeq);
 804 }
 805
 806 static irqreturn_t nvme_irq(int irq, void *data)
 807 {
 808         irqreturn_t result;
 809         struct nvme_queue *nvmeq = data;
 810         spin_lock(&nvmeq->q_lock);
 811         nvme_process_cq(nvmeq);
 812         result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
 813         nvmeq->cqe_seen = 0;
 814         spin_unlock(&nvmeq->q_lock);
 815         return result;
 816 }
 817
 818 static irqreturn_t nvme_irq_check(int irq, void *data)
 819 {
 820         struct nvme_queue *nvmeq = data;
 821         struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
 822         if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
 823                 return IRQ_NONE;
 824         return IRQ_WAKE_THREAD;
 825 }
 826
 827 static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
 828 {
 829         spin_lock_irq(&nvmeq->q_lock);
 830         cancel_cmdid(nvmeq, cmdid, NULL);
 831         spin_unlock_irq(&nvmeq->q_lock);
 832 }
 833
 834 struct sync_cmd_info {
 835         struct task_struct *task;
 836         u32 result;
 837         int status;
 838 };
 839
 840 static void sync_completion(struct nvme_dev *dev, void *ctx,
 841                                                 struct nvme_completion *cqe)
 842 {
 843         struct sync_cmd_info *cmdinfo = ctx;
 844         cmdinfo->result = le32_to_cpup(&cqe->result);
 845         cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
 846         wake_up_process(cmdinfo->task);
 847 }
 848
 849 /*
 850  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 851  * if the result is positive, it's an NVM Express status code
 852  */
 853 int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
 854                                                 u32 *result, unsigned timeout)
 855 {
 856         int cmdid;
 857         struct sync_cmd_info cmdinfo;
 858
 859         cmdinfo.task = current;
 860         cmdinfo.status = -EINTR;
 861
 862         cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion,
 863                                                                 timeout);
 864         if (cmdid < 0)
 865                 return cmdid;
 866         cmd->common.command_id = cmdid;
 867
 868         set_current_state(TASK_KILLABLE);
 869         nvme_submit_cmd(nvmeq, cmd);
 870         schedule_timeout(timeout);
 871
 872         if (cmdinfo.status == -EINTR) {
 873                 nvme_abort_command(nvmeq, cmdid);
 874                 return -EINTR;
 875         }
 876
 877         if (result)
 878                 *result = cmdinfo.result;
 879
 880         return cmdinfo.status;
 881 }
 882
 883 int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
 884                                                                 u32 *result)
 885 {
 886         return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT);
 887 }
 888
 889 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 890 {
 891         int status;
 892         struct nvme_command c;
 893
 894         memset(&c, 0, sizeof(c));
 895         c.delete_queue.opcode = opcode;
 896         c.delete_queue.qid = cpu_to_le16(id);
 897
 898         status = nvme_submit_admin_cmd(dev, &c, NULL);
 899         if (status)
 900                 return -EIO;
 901         return 0;
 902 }
 903
 904 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 905                                                 struct nvme_queue *nvmeq)
 906 {
 907         int status;
 908         struct nvme_command c;
 909         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
 910
 911         memset(&c, 0, sizeof(c));
 912         c.create_cq.opcode = nvme_admin_create_cq;
 913         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
 914         c.create_cq.cqid = cpu_to_le16(qid);
 915         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 916         c.create_cq.cq_flags = cpu_to_le16(flags);
 917         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 918
 919         status = nvme_submit_admin_cmd(dev, &c, NULL);
 920         if (status)
 921                 return -EIO;
 922         return 0;
 923 }
 924
 925 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
 926                                                 struct nvme_queue *nvmeq)
 927 {
 928         int status;
 929         struct nvme_command c;
 930         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
 931
 932         memset(&c, 0, sizeof(c));
 933         c.create_sq.opcode = nvme_admin_create_sq;
 934         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
 935         c.create_sq.sqid = cpu_to_le16(qid);
 936         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 937         c.create_sq.sq_flags = cpu_to_le16(flags);
 938         c.create_sq.cqid = cpu_to_le16(qid);
 939
 940         status = nvme_submit_admin_cmd(dev, &c, NULL);
 941         if (status)
 942                 return -EIO;
 943         return 0;
 944 }
 945
 946 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
 947 {
 948         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
 949 }
 950
 951 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 952 {
 953         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 954 }
 955
 956 int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
 957                                                         dma_addr_t dma_addr)
 958 {
 959         struct nvme_command c;
 960
 961         memset(&c, 0, sizeof(c));
 962         c.identify.opcode = nvme_admin_identify;
 963         c.identify.nsid = cpu_to_le32(nsid);
 964         c.identify.prp1 = cpu_to_le64(dma_addr);
 965         c.identify.cns = cpu_to_le32(cns);
 966
 967         return nvme_submit_admin_cmd(dev, &c, NULL);
 968 }
 969
 970 int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
 971                                         dma_addr_t dma_addr, u32 *result)
 972 {
 973         struct nvme_command c;
 974
 975         memset(&c, 0, sizeof(c));
 976         c.features.opcode = nvme_admin_get_features;
 977         c.features.nsid = cpu_to_le32(nsid);
 978         c.features.prp1 = cpu_to_le64(dma_addr);
 979         c.features.fid = cpu_to_le32(fid);
 980
 981         return nvme_submit_admin_cmd(dev, &c, result);
 982 }
 983
 984 int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
 985                                         dma_addr_t dma_addr, u32 *result)
 986 {
 987         struct nvme_command c;
 988
 989         memset(&c, 0, sizeof(c));
 990         c.features.opcode = nvme_admin_set_features;
 991         c.features.prp1 = cpu_to_le64(dma_addr);
 992         c.features.fid = cpu_to_le32(fid);
 993         c.features.dword11 = cpu_to_le32(dword11);
 994
 995         return nvme_submit_admin_cmd(dev, &c, result);
 996 }
 997
 998 /**
 999  * nvme_cancel_ios - Cancel outstanding I/Os
1000  * @queue: The queue to cancel I/Os on
1001  * @timeout: True to only cancel I/Os which have timed out
1002  */
1003 static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
1004 {
1005         int depth = nvmeq->q_depth - 1;
1006         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
1007         unsigned long now = jiffies;
1008         int cmdid;
1009
1010         for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
1011                 void *ctx;
1012                 nvme_completion_fn fn;
1013                 static struct nvme_completion cqe = {
1014                         .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1),
1015                 };
1016
1017                 if (timeout && !time_after(now, info[cmdid].timeout))
1018                         continue;
1019                 if (info[cmdid].ctx == CMD_CTX_CANCELLED)
1020                         continue;
1021                 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
1022                 ctx = cancel_cmdid(nvmeq, cmdid, &fn);
1023                 fn(nvmeq->dev, ctx, &cqe);
1024         }
1025 }
1026
1027 static void nvme_free_queue(struct nvme_queue *nvmeq)
1028 {
1029         spin_lock_irq(&nvmeq->q_lock);
1030         while (bio_list_peek(&nvmeq->sq_cong)) {
1031                 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1032                 bio_endio(bio, -EIO);
1033         }
1034         spin_unlock_irq(&nvmeq->q_lock);
1035
1036         dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1037                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1038         dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1039                                         nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1040         kfree(nvmeq);
1041 }
1042
1043 static void nvme_free_queues(struct nvme_dev *dev)
1044 {
1045         int i;
1046
1047         for (i = dev->queue_count - 1; i >= 0; i--) {
1048                 nvme_free_queue(dev->queues[i]);
1049                 dev->queue_count--;
1050                 dev->queues[i] = NULL;
1051         }
1052 }
1053
1054 static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1055 {
1056         struct nvme_queue *nvmeq = dev->queues[qid];
1057         int vector = dev->entry[nvmeq->cq_vector].vector;
1058
1059         spin_lock_irq(&nvmeq->q_lock);
1060         if (nvmeq->q_suspended) {
1061                 spin_unlock_irq(&nvmeq->q_lock);
1062                 return;
1063         }
1064         nvmeq->q_suspended = 1;
1065         spin_unlock_irq(&nvmeq->q_lock);
1066
1067         irq_set_affinity_hint(vector, NULL);
1068         free_irq(vector, nvmeq);
1069
1070         /* Don't tell the adapter to delete the admin queue */
1071         if (qid) {
1072                 adapter_delete_sq(dev, qid);
1073                 adapter_delete_cq(dev, qid);
1074         }
1075
1076         spin_lock_irq(&nvmeq->q_lock);
1077         nvme_process_cq(nvmeq);
1078         nvme_cancel_ios(nvmeq, false);
1079         spin_unlock_irq(&nvmeq->q_lock);
1080 }
1081
1082 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1083                                                         int depth, int vector)
1084 {
1085         struct device *dmadev = &dev->pci_dev->dev;
1086         unsigned extra = nvme_queue_extra(depth);
1087         struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
1088         if (!nvmeq)
1089                 return NULL;
1090
1091         nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
1092                                         &nvmeq->cq_dma_addr, GFP_KERNEL);
1093         if (!nvmeq->cqes)
1094                 goto free_nvmeq;
1095         memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
1096
1097         nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
1098                                         &nvmeq->sq_dma_addr, GFP_KERNEL);
1099         if (!nvmeq->sq_cmds)
1100                 goto free_cqdma;
1101
1102         nvmeq->q_dmadev = dmadev;
1103         nvmeq->dev = dev;
1104         spin_lock_init(&nvmeq->q_lock);
1105         nvmeq->cq_head = 0;
1106         nvmeq->cq_phase = 1;
1107         init_waitqueue_head(&nvmeq->sq_full);
1108         init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread);
1109         bio_list_init(&nvmeq->sq_cong);
1110         nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)];
1111         nvmeq->q_depth = depth;
1112         nvmeq->cq_vector = vector;
1113         nvmeq->q_suspended = 1;
1114         dev->queue_count++;
1115
1116         return nvmeq;
1117
1118  free_cqdma:
1119         dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1120                                                         nvmeq->cq_dma_addr);
1121  free_nvmeq:
1122         kfree(nvmeq);
1123         return NULL;
1124 }
1125
1126 static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1127                                                         const char *name)
1128 {
1129         if (use_threaded_interrupts)
1130                 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1131                                         nvme_irq_check, nvme_irq,
1132                                         IRQF_DISABLED | IRQF_SHARED,
1133                                         name, nvmeq);
1134         return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1135                                 IRQF_DISABLED | IRQF_SHARED, name, nvmeq);
1136 }
1137
1138 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1139 {
1140         struct nvme_dev *dev = nvmeq->dev;
1141         unsigned extra = nvme_queue_extra(nvmeq->q_depth);
1142
1143         nvmeq->sq_tail = 0;
1144         nvmeq->cq_head = 0;
1145         nvmeq->cq_phase = 1;
1146         nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)];
1147         memset(nvmeq->cmdid_data, 0, extra);
1148         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1149         nvme_cancel_ios(nvmeq, false);
1150         nvmeq->q_suspended = 0;
1151 }
1152
1153 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1154 {
1155         struct nvme_dev *dev = nvmeq->dev;
1156         int result;
1157
1158         result = adapter_alloc_cq(dev, qid, nvmeq);
1159         if (result < 0)
1160                 return result;
1161
1162         result = adapter_alloc_sq(dev, qid, nvmeq);
1163         if (result < 0)
1164                 goto release_cq;
1165
1166         result = queue_request_irq(dev, nvmeq, "nvme");
1167         if (result < 0)
1168                 goto release_sq;
1169
1170         spin_lock(&nvmeq->q_lock);
1171         nvme_init_queue(nvmeq, qid);
1172         spin_unlock(&nvmeq->q_lock);
1173
1174         return result;
1175
1176  release_sq:
1177         adapter_delete_sq(dev, qid);
1178  release_cq:
1179         adapter_delete_cq(dev, qid);
1180         return result;
1181 }
1182
1183 static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1184 {
1185         unsigned long timeout;
1186         u32 bit = enabled ? NVME_CSTS_RDY : 0;
1187
1188         timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1189
1190         while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1191                 msleep(100);
1192                 if (fatal_signal_pending(current))
1193                         return -EINTR;
1194                 if (time_after(jiffies, timeout)) {
1195                         dev_err(&dev->pci_dev->dev,
1196                                 "Device not ready; aborting initialisation\n");
1197                         return -ENODEV;
1198                 }
1199         }
1200
1201         return 0;
1202 }
1203
1204 /*
1205  * If the device has been passed off to us in an enabled state, just clear
1206  * the enabled bit.  The spec says we should set the 'shutdown notification
1207  * bits', but doing so may cause the device to complete commands to the
1208  * admin queue ... and we don't know what memory that might be pointing at!
1209  */
1210 static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1211 {
1212         u32 cc = readl(&dev->bar->cc);
1213
1214         if (cc & NVME_CC_ENABLE)
1215                 writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc);
1216         return nvme_wait_ready(dev, cap, false);
1217 }
1218
1219 static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1220 {
1221         return nvme_wait_ready(dev, cap, true);
1222 }
1223
1224 static int nvme_shutdown_ctrl(struct nvme_dev *dev)
1225 {
1226         unsigned long timeout;
1227         u32 cc;
1228
1229         cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL;
1230         writel(cc, &dev->bar->cc);
1231
1232         timeout = 2 * HZ + jiffies;
1233         while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
1234                                                         NVME_CSTS_SHST_CMPLT) {
1235                 msleep(100);
1236                 if (fatal_signal_pending(current))
1237                         return -EINTR;
1238                 if (time_after(jiffies, timeout)) {
1239                         dev_err(&dev->pci_dev->dev,
1240                                 "Device shutdown incomplete; abort shutdown\n");
1241                         return -ENODEV;
1242                 }
1243         }
1244
1245         return 0;
1246 }
1247
1248 static int nvme_configure_admin_queue(struct nvme_dev *dev)
1249 {
1250         int result;
1251         u32 aqa;
1252         u64 cap = readq(&dev->bar->cap);
1253         struct nvme_queue *nvmeq;
1254
1255         result = nvme_disable_ctrl(dev, cap);
1256         if (result < 0)
1257                 return result;
1258
1259         nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
1260         if (!nvmeq)
1261                 return -ENOMEM;
1262
1263         aqa = nvmeq->q_depth - 1;
1264         aqa |= aqa << 16;
1265
1266         dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
1267         dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1268         dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1269         dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1270
1271         writel(aqa, &dev->bar->aqa);
1272         writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1273         writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
1274         writel(dev->ctrl_config, &dev->bar->cc);
1275
1276         result = nvme_enable_ctrl(dev, cap);
1277         if (result)
1278                 goto free_q;
1279
1280         result = queue_request_irq(dev, nvmeq, "nvme admin");
1281         if (result)
1282                 goto free_q;
1283
1284         dev->queues[0] = nvmeq;
1285         spin_lock(&nvmeq->q_lock);
1286         nvme_init_queue(nvmeq, 0);
1287         spin_unlock(&nvmeq->q_lock);
1288         return result;
1289
1290  free_q:
1291         nvme_free_queue(nvmeq);
1292         return result;
1293 }
1294
1295 struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
1296                                 unsigned long addr, unsigned length)
1297 {
1298         int i, err, count, nents, offset;
1299         struct scatterlist *sg;
1300         struct page **pages;
1301         struct nvme_iod *iod;
1302
1303         if (addr & 3)
1304                 return ERR_PTR(-EINVAL);
1305         if (!length || length > INT_MAX - PAGE_SIZE)
1306                 return ERR_PTR(-EINVAL);
1307
1308         offset = offset_in_page(addr);
1309         count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1310         pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
1311         if (!pages)
1312                 return ERR_PTR(-ENOMEM);
1313
1314         err = get_user_pages_fast(addr, count, 1, pages);
1315         if (err < count) {
1316                 count = err;
1317                 err = -EFAULT;
1318                 goto put_pages;
1319         }
1320
1321         iod = nvme_alloc_iod(count, length, GFP_KERNEL);
1322         sg = iod->sg;
1323         sg_init_table(sg, count);
1324         for (i = 0; i < count; i++) {
1325                 sg_set_page(&sg[i], pages[i],
1326                             min_t(unsigned, length, PAGE_SIZE - offset),
1327                             offset);
1328                 length -= (PAGE_SIZE - offset);
1329                 offset = 0;
1330         }
1331         sg_mark_end(&sg[i - 1]);
1332         iod->nents = count;
1333
1334         err = -ENOMEM;
1335         nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
1336                                 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1337         if (!nents)
1338                 goto free_iod;
1339
1340         kfree(pages);
1341         return iod;
1342
1343  free_iod:
1344         kfree(iod);
1345  put_pages:
1346         for (i = 0; i < count; i++)
1347                 put_page(pages[i]);
1348         kfree(pages);
1349         return ERR_PTR(err);
1350 }
1351
1352 void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
1353                         struct nvme_iod *iod)
1354 {
1355         int i;
1356
1357         dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
1358                                 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1359
1360         for (i = 0; i < iod->nents; i++)
1361                 put_page(sg_page(&iod->sg[i]));
1362 }
1363
1364 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1365 {
1366         struct nvme_dev *dev = ns->dev;
1367         struct nvme_queue *nvmeq;
1368         struct nvme_user_io io;
1369         struct nvme_command c;
1370         unsigned length, meta_len;
1371         int status, i;
1372         struct nvme_iod *iod, *meta_iod = NULL;
1373         dma_addr_t meta_dma_addr;
1374         void *meta, *uninitialized_var(meta_mem);
1375
1376         if (copy_from_user(&io, uio, sizeof(io)))
1377                 return -EFAULT;
1378         length = (io.nblocks + 1) << ns->lba_shift;
1379         meta_len = (io.nblocks + 1) * ns->ms;
1380
1381         if (meta_len && ((io.metadata & 3) || !io.metadata))
1382                 return -EINVAL;
1383
1384         switch (io.opcode) {
1385         case nvme_cmd_write:
1386         case nvme_cmd_read:
1387         case nvme_cmd_compare:
1388                 iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
1389                 break;
1390         default:
1391                 return -EINVAL;
1392         }
1393
1394         if (IS_ERR(iod))
1395                 return PTR_ERR(iod);
1396
1397         memset(&c, 0, sizeof(c));
1398         c.rw.opcode = io.opcode;
1399         c.rw.flags = io.flags;
1400         c.rw.nsid = cpu_to_le32(ns->ns_id);
1401         c.rw.slba = cpu_to_le64(io.slba);
1402         c.rw.length = cpu_to_le16(io.nblocks);
1403         c.rw.control = cpu_to_le16(io.control);
1404         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1405         c.rw.reftag = cpu_to_le32(io.reftag);
1406         c.rw.apptag = cpu_to_le16(io.apptag);
1407         c.rw.appmask = cpu_to_le16(io.appmask);
1408
1409         if (meta_len) {
1410                 meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata,
1411                                                                 meta_len);
1412                 if (IS_ERR(meta_iod)) {
1413                         status = PTR_ERR(meta_iod);
1414                         meta_iod = NULL;
1415                         goto unmap;
1416                 }
1417
1418                 meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
1419                                                 &meta_dma_addr, GFP_KERNEL);
1420                 if (!meta_mem) {
1421                         status = -ENOMEM;
1422                         goto unmap;
1423                 }
1424
1425                 if (io.opcode & 1) {
1426                         int meta_offset = 0;
1427
1428                         for (i = 0; i < meta_iod->nents; i++) {
1429                                 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1430                                                 meta_iod->sg[i].offset;
1431                                 memcpy(meta_mem + meta_offset, meta,
1432                                                 meta_iod->sg[i].length);
1433                                 kunmap_atomic(meta);
1434                                 meta_offset += meta_iod->sg[i].length;
1435                         }
1436                 }
1437
1438                 c.rw.metadata = cpu_to_le64(meta_dma_addr);
1439         }
1440
1441         length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL);
1442
1443         nvmeq = get_nvmeq(dev);
1444         /*
1445          * Since nvme_submit_sync_cmd sleeps, we can't keep preemption
1446          * disabled.  We may be preempted at any point, and be rescheduled
1447          * to a different CPU.  That will cause cacheline bouncing, but no
1448          * additional races since q_lock already protects against other CPUs.
1449          */
1450         put_nvmeq(nvmeq);
1451         if (length != (io.nblocks + 1) << ns->lba_shift)
1452                 status = -ENOMEM;
1453         else if (!nvmeq || nvmeq->q_suspended)
1454                 status = -EBUSY;
1455         else
1456                 status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT);
1457
1458         if (meta_len) {
1459                 if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
1460                         int meta_offset = 0;
1461
1462                         for (i = 0; i < meta_iod->nents; i++) {
1463                                 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1464                                                 meta_iod->sg[i].offset;
1465                                 memcpy(meta, meta_mem + meta_offset,
1466                                                 meta_iod->sg[i].length);
1467                                 kunmap_atomic(meta);
1468                                 meta_offset += meta_iod->sg[i].length;
1469                         }
1470                 }
1471
1472                 dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
1473                                                                 meta_dma_addr);
1474         }
1475
1476  unmap:
1477         nvme_unmap_user_pages(dev, io.opcode & 1, iod);
1478         nvme_free_iod(dev, iod);
1479
1480         if (meta_iod) {
1481                 nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
1482                 nvme_free_iod(dev, meta_iod);
1483         }
1484
1485         return status;
1486 }
1487
1488 static int nvme_user_admin_cmd(struct nvme_dev *dev,
1489                                         struct nvme_admin_cmd __user *ucmd)
1490 {
1491         struct nvme_admin_cmd cmd;
1492         struct nvme_command c;
1493         int status, length;
1494         struct nvme_iod *uninitialized_var(iod);
1495         unsigned timeout;
1496
1497         if (!capable(CAP_SYS_ADMIN))
1498                 return -EACCES;
1499         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1500                 return -EFAULT;
1501
1502         memset(&c, 0, sizeof(c));
1503         c.common.opcode = cmd.opcode;
1504         c.common.flags = cmd.flags;
1505         c.common.nsid = cpu_to_le32(cmd.nsid);
1506         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1507         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1508         c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1509         c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1510         c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1511         c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1512         c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1513         c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1514
1515         length = cmd.data_len;
1516         if (cmd.data_len) {
1517                 iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1518                                                                 length);
1519                 if (IS_ERR(iod))
1520                         return PTR_ERR(iod);
1521                 length = nvme_setup_prps(dev, &c.common, iod, length,
1522                                                                 GFP_KERNEL);
1523         }
1524
1525         timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
1526                                                                 ADMIN_TIMEOUT;
1527         if (length != cmd.data_len)
1528                 status = -ENOMEM;
1529         else
1530                 status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result,
1531                                                                 timeout);
1532
1533         if (cmd.data_len) {
1534                 nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
1535                 nvme_free_iod(dev, iod);
1536         }
1537
1538         if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
1539                                                         sizeof(cmd.result)))
1540                 status = -EFAULT;
1541
1542         return status;
1543 }
1544
1545 static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1546                                                         unsigned long arg)
1547 {
1548         struct nvme_ns *ns = bdev->bd_disk->private_data;
1549
1550         switch (cmd) {
1551         case NVME_IOCTL_ID:
1552                 force_successful_syscall_return();
1553                 return ns->ns_id;
1554         case NVME_IOCTL_ADMIN_CMD:
1555                 return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
1556         case NVME_IOCTL_SUBMIT_IO:
1557                 return nvme_submit_io(ns, (void __user *)arg);
1558         case SG_GET_VERSION_NUM:
1559                 return nvme_sg_get_version_num((void __user *)arg);
1560         case SG_IO:
1561                 return nvme_sg_io(ns, (void __user *)arg);
1562         default:
1563                 return -ENOTTY;
1564         }
1565 }
1566
1567 static const struct block_device_operations nvme_fops = {
1568         .owner          = THIS_MODULE,
1569         .ioctl          = nvme_ioctl,
1570         .compat_ioctl   = nvme_ioctl,
1571 };
1572
1573 static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
1574 {
1575         while (bio_list_peek(&nvmeq->sq_cong)) {
1576                 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1577                 struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;
1578
1579                 if (bio_list_empty(&nvmeq->sq_cong))
1580                         remove_wait_queue(&nvmeq->sq_full,
1581                                                         &nvmeq->sq_cong_wait);
1582                 if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
1583                         if (bio_list_empty(&nvmeq->sq_cong))
1584                                 add_wait_queue(&nvmeq->sq_full,
1585                                                         &nvmeq->sq_cong_wait);
1586                         bio_list_add_head(&nvmeq->sq_cong, bio);
1587                         break;
1588                 }
1589         }
1590 }
1591
1592 static int nvme_kthread(void *data)
1593 {
1594         struct nvme_dev *dev;
1595
1596         while (!kthread_should_stop()) {
1597                 set_current_state(TASK_INTERRUPTIBLE);
1598                 spin_lock(&dev_list_lock);
1599                 list_for_each_entry(dev, &dev_list, node) {
1600                         int i;
1601                         for (i = 0; i < dev->queue_count; i++) {
1602                                 struct nvme_queue *nvmeq = dev->queues[i];
1603                                 if (!nvmeq)
1604                                         continue;
1605                                 spin_lock_irq(&nvmeq->q_lock);
1606                                 if (nvmeq->q_suspended)
1607                                         goto unlock;
1608                                 nvme_process_cq(nvmeq);
1609                                 nvme_cancel_ios(nvmeq, true);
1610                                 nvme_resubmit_bios(nvmeq);
1611  unlock:
1612                                 spin_unlock_irq(&nvmeq->q_lock);
1613                         }
1614                 }
1615                 spin_unlock(&dev_list_lock);
1616                 schedule_timeout(round_jiffies_relative(HZ));
1617         }
1618         return 0;
1619 }
1620
1621 static DEFINE_IDA(nvme_index_ida);
1622
1623 static int nvme_get_ns_idx(void)
1624 {
1625         int index, error;
1626
1627         do {
1628                 if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL))
1629                         return -1;
1630
1631                 spin_lock(&dev_list_lock);
1632                 error = ida_get_new(&nvme_index_ida, &index);
1633                 spin_unlock(&dev_list_lock);
1634         } while (error == -EAGAIN);
1635
1636         if (error)
1637                 index = -1;
1638         return index;
1639 }
1640
1641 static void nvme_put_ns_idx(int index)
1642 {
1643         spin_lock(&dev_list_lock);
1644         ida_remove(&nvme_index_ida, index);
1645         spin_unlock(&dev_list_lock);
1646 }
1647
1648 static void nvme_config_discard(struct nvme_ns *ns)
1649 {
1650         u32 logical_block_size = queue_logical_block_size(ns->queue);
1651         ns->queue->limits.discard_zeroes_data = 0;
1652         ns->queue->limits.discard_alignment = logical_block_size;
1653         ns->queue->limits.discard_granularity = logical_block_size;
1654         ns->queue->limits.max_discard_sectors = 0xffffffff;
1655         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1656 }
1657
1658 static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid,
1659                         struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
1660 {
1661         struct nvme_ns *ns;
1662         struct gendisk *disk;
1663         int lbaf;
1664
1665         if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
1666                 return NULL;
1667
1668         ns = kzalloc(sizeof(*ns), GFP_KERNEL);
1669         if (!ns)
1670                 return NULL;
1671         ns->queue = blk_alloc_queue(GFP_KERNEL);
1672         if (!ns->queue)
1673                 goto out_free_ns;
1674         ns->queue->queue_flags = QUEUE_FLAG_DEFAULT;
1675         queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
1676         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1677         blk_queue_make_request(ns->queue, nvme_make_request);
1678         ns->dev = dev;
1679         ns->queue->queuedata = ns;
1680
1681         disk = alloc_disk(NVME_MINORS);
1682         if (!disk)
1683                 goto out_free_queue;
1684         ns->ns_id = nsid;
1685         ns->disk = disk;
1686         lbaf = id->flbas & 0xf;
1687         ns->lba_shift = id->lbaf[lbaf].ds;
1688         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1689         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1690         if (dev->max_hw_sectors)
1691                 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
1692
1693         disk->major = nvme_major;
1694         disk->minors = NVME_MINORS;
1695         disk->first_minor = NVME_MINORS * nvme_get_ns_idx();
1696         disk->fops = &nvme_fops;
1697         disk->private_data = ns;
1698         disk->queue = ns->queue;
1699         disk->driverfs_dev = &dev->pci_dev->dev;
1700         sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
1701         set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1702
1703         if (dev->oncs & NVME_CTRL_ONCS_DSM)
1704                 nvme_config_discard(ns);
1705
1706         return ns;
1707
1708  out_free_queue:
1709         blk_cleanup_queue(ns->queue);
1710  out_free_ns:
1711         kfree(ns);
1712         return NULL;
1713 }
1714
1715 static void nvme_ns_free(struct nvme_ns *ns)
1716 {
1717         int index = ns->disk->first_minor / NVME_MINORS;
1718         put_disk(ns->disk);
1719         nvme_put_ns_idx(index);
1720         blk_cleanup_queue(ns->queue);
1721         kfree(ns);
1722 }
1723
1724 static int set_queue_count(struct nvme_dev *dev, int count)
1725 {
1726         int status;
1727         u32 result;
1728         u32 q_count = (count - 1) | ((count - 1) << 16);
1729
1730         status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
1731                                                                 &result);
1732         if (status)
1733                 return status < 0 ? -EIO : -EBUSY;
1734         return min(result & 0xffff, result >> 16) + 1;
1735 }
1736
1737 static int nvme_setup_io_queues(struct nvme_dev *dev)
1738 {
1739         struct pci_dev *pdev = dev->pci_dev;
1740         int result, cpu, i, vecs, nr_io_queues, db_bar_size, q_depth;
1741
1742         nr_io_queues = num_online_cpus();
1743         result = set_queue_count(dev, nr_io_queues);
1744         if (result < 0)
1745                 return result;
1746         if (result < nr_io_queues)
1747                 nr_io_queues = result;
1748
1749         /* Deregister the admin queue's interrupt */
1750         free_irq(dev->entry[0].vector, dev->queues[0]);
1751
1752         db_bar_size = 4096 + ((nr_io_queues + 1) << (dev->db_stride + 3));
1753         if (db_bar_size > 8192) {
1754                 iounmap(dev->bar);
1755                 dev->bar = ioremap(pci_resource_start(pdev, 0), db_bar_size);
1756                 dev->dbs = ((void __iomem *)dev->bar) + 4096;
1757                 dev->queues[0]->q_db = dev->dbs;
1758         }
1759
1760         vecs = nr_io_queues;
1761         for (i = 0; i < vecs; i++)
1762                 dev->entry[i].entry = i;
1763         for (;;) {
1764                 result = pci_enable_msix(pdev, dev->entry, vecs);
1765                 if (result <= 0)
1766                         break;
1767                 vecs = result;
1768         }
1769
1770         if (result < 0) {
1771                 vecs = nr_io_queues;
1772                 if (vecs > 32)
1773                         vecs = 32;
1774                 for (;;) {
1775                         result = pci_enable_msi_block(pdev, vecs);
1776                         if (result == 0) {
1777                                 for (i = 0; i < vecs; i++)
1778                                         dev->entry[i].vector = i + pdev->irq;
1779                                 break;
1780                         } else if (result < 0) {
1781                                 vecs = 1;
1782                                 break;
1783                         }
1784                         vecs = result;
1785                 }
1786         }
1787
1788         /*
1789          * Should investigate if there's a performance win from allocating
1790          * more queues than interrupt vectors; it might allow the submission
1791          * path to scale better, even if the receive path is limited by the
1792          * number of interrupts.
1793          */
1794         nr_io_queues = vecs;
1795
1796         result = queue_request_irq(dev, dev->queues[0], "nvme admin");
1797         if (result)
1798                 goto free_queues;
1799
1800         cpu = cpumask_first(cpu_online_mask);
1801         for (i = 0; i < nr_io_queues; i++) {
1802                 irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
1803                 cpu = cpumask_next(cpu, cpu_online_mask);
1804         }
1805
1806         q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
1807                                                                 NVME_Q_DEPTH);
1808         for (i = 0; i < nr_io_queues; i++) {
1809                 dev->queues[i + 1] = nvme_alloc_queue(dev, i + 1, q_depth, i);
1810                 if (!dev->queues[i + 1]) {
1811                         result = -ENOMEM;
1812                         goto free_queues;
1813                 }
1814         }
1815
1816         for (; i < num_possible_cpus(); i++) {
1817                 int target = i % rounddown_pow_of_two(dev->queue_count - 1);
1818                 dev->queues[i + 1] = dev->queues[target + 1];
1819         }
1820
1821         for (i = 1; i < dev->queue_count; i++) {
1822                 result = nvme_create_queue(dev->queues[i], i);
1823                 if (result) {
1824                         for (--i; i > 0; i--)
1825                                 nvme_disable_queue(dev, i);
1826                         goto free_queues;
1827                 }
1828         }
1829
1830         return 0;
1831
1832  free_queues:
1833         nvme_free_queues(dev);
1834         return result;
1835 }
1836
1837 /*
1838  * Return: error value if an error occurred setting up the queues or calling
1839  * Identify Device.  0 if these succeeded, even if adding some of the
1840  * namespaces failed.  At the moment, these failures are silent.  TBD which
1841  * failures should be reported.
1842  */
1843 static int nvme_dev_add(struct nvme_dev *dev)
1844 {
1845         int res;
1846         unsigned nn, i;
1847         struct nvme_ns *ns;
1848         struct nvme_id_ctrl *ctrl;
1849         struct nvme_id_ns *id_ns;
1850         void *mem;
1851         dma_addr_t dma_addr;
1852         int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
1853
1854         mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
1855                                                                 GFP_KERNEL);
1856         if (!mem)
1857                 return -ENOMEM;
1858
1859         res = nvme_identify(dev, 0, 1, dma_addr);
1860         if (res) {
1861                 res = -EIO;
1862                 goto out;
1863         }
1864
1865         ctrl = mem;
1866         nn = le32_to_cpup(&ctrl->nn);
1867         dev->oncs = le16_to_cpup(&ctrl->oncs);
1868         memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
1869         memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
1870         memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
1871         if (ctrl->mdts)
1872                 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
1873         if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) &&
1874                         (dev->pci_dev->device == 0x0953) && ctrl->vs[3])
1875                 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
1876
1877         id_ns = mem;
1878         for (i = 1; i <= nn; i++) {
1879                 res = nvme_identify(dev, i, 0, dma_addr);
1880                 if (res)
1881                         continue;
1882
1883                 if (id_ns->ncap == 0)
1884                         continue;
1885
1886                 res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i,
1887                                                         dma_addr + 4096, NULL);
1888                 if (res)
1889                         memset(mem + 4096, 0, 4096);
1890
1891                 ns = nvme_alloc_ns(dev, i, mem, mem + 4096);
1892                 if (ns)
1893                         list_add_tail(&ns->list, &dev->namespaces);
1894         }
1895         list_for_each_entry(ns, &dev->namespaces, list)
1896                 add_disk(ns->disk);
1897         res = 0;
1898
1899  out:
1900         dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr);
1901         return res;
1902 }
1903
1904 static int nvme_dev_map(struct nvme_dev *dev)
1905 {
1906         int bars, result = -ENOMEM;
1907         struct pci_dev *pdev = dev->pci_dev;
1908
1909         if (pci_enable_device_mem(pdev))
1910                 return result;
1911
1912         dev->entry[0].vector = pdev->irq;
1913         pci_set_master(pdev);
1914         bars = pci_select_bars(pdev, IORESOURCE_MEM);
1915         if (pci_request_selected_regions(pdev, bars, "nvme"))
1916                 goto disable_pci;
1917
1918         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)))
1919                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1920         else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)))
1921                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1922         else
1923                 goto disable_pci;
1924
1925         pci_set_drvdata(pdev, dev);
1926         dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1927         if (!dev->bar)
1928                 goto disable;
1929
1930         dev->db_stride = NVME_CAP_STRIDE(readq(&dev->bar->cap));
1931         dev->dbs = ((void __iomem *)dev->bar) + 4096;
1932
1933         return 0;
1934
1935  disable:
1936         pci_release_regions(pdev);
1937  disable_pci:
1938         pci_disable_device(pdev);
1939         return result;
1940 }
1941
1942 static void nvme_dev_unmap(struct nvme_dev *dev)
1943 {
1944         if (dev->pci_dev->msi_enabled)
1945                 pci_disable_msi(dev->pci_dev);
1946         else if (dev->pci_dev->msix_enabled)
1947                 pci_disable_msix(dev->pci_dev);
1948
1949         if (dev->bar) {
1950                 iounmap(dev->bar);
1951                 dev->bar = NULL;
1952         }
1953
1954         pci_release_regions(dev->pci_dev);
1955         if (pci_is_enabled(dev->pci_dev))
1956                 pci_disable_device(dev->pci_dev);
1957 }
1958
1959 static void nvme_dev_shutdown(struct nvme_dev *dev)
1960 {
1961         int i;
1962
1963         for (i = dev->queue_count - 1; i >= 0; i--)
1964                 nvme_disable_queue(dev, i);
1965
1966         spin_lock(&dev_list_lock);
1967         list_del_init(&dev->node);
1968         spin_unlock(&dev_list_lock);
1969
1970         if (dev->bar)
1971                 nvme_shutdown_ctrl(dev);
1972         nvme_dev_unmap(dev);
1973 }
1974
1975 static void nvme_dev_remove(struct nvme_dev *dev)
1976 {
1977         struct nvme_ns *ns, *next;
1978
1979         list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1980                 list_del(&ns->list);
1981                 del_gendisk(ns->disk);
1982                 nvme_ns_free(ns);
1983         }
1984 }
1985
1986 static int nvme_setup_prp_pools(struct nvme_dev *dev)
1987 {
1988         struct device *dmadev = &dev->pci_dev->dev;
1989         dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
1990                                                 PAGE_SIZE, PAGE_SIZE, 0);
1991         if (!dev->prp_page_pool)
1992                 return -ENOMEM;
1993
1994         /* Optimisation for I/Os between 4k and 128k */
1995         dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
1996                                                 256, 256, 0);
1997         if (!dev->prp_small_pool) {
1998                 dma_pool_destroy(dev->prp_page_pool);
1999                 return -ENOMEM;
2000         }
2001         return 0;
2002 }
2003
2004 static void nvme_release_prp_pools(struct nvme_dev *dev)
2005 {
2006         dma_pool_destroy(dev->prp_page_pool);
2007         dma_pool_destroy(dev->prp_small_pool);
2008 }
2009
2010 static DEFINE_IDA(nvme_instance_ida);
2011
2012 static int nvme_set_instance(struct nvme_dev *dev)
2013 {
2014         int instance, error;
2015
2016         do {
2017                 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2018                         return -ENODEV;
2019
2020                 spin_lock(&dev_list_lock);
2021                 error = ida_get_new(&nvme_instance_ida, &instance);
2022                 spin_unlock(&dev_list_lock);
2023         } while (error == -EAGAIN);
2024
2025         if (error)
2026                 return -ENODEV;
2027
2028         dev->instance = instance;
2029         return 0;
2030 }
2031
2032 static void nvme_release_instance(struct nvme_dev *dev)
2033 {
2034         spin_lock(&dev_list_lock);
2035         ida_remove(&nvme_instance_ida, dev->instance);
2036         spin_unlock(&dev_list_lock);
2037 }
2038
2039 static void nvme_free_dev(struct kref *kref)
2040 {
2041         struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
2042         nvme_dev_remove(dev);
2043         nvme_dev_shutdown(dev);
2044         nvme_free_queues(dev);
2045         nvme_release_instance(dev);
2046         nvme_release_prp_pools(dev);
2047         kfree(dev->queues);
2048         kfree(dev->entry);
2049         kfree(dev);
2050 }
2051
2052 static int nvme_dev_open(struct inode *inode, struct file *f)
2053 {
2054         struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
2055                                                                 miscdev);
2056         kref_get(&dev->kref);
2057         f->private_data = dev;
2058         return 0;
2059 }
2060
2061 static int nvme_dev_release(struct inode *inode, struct file *f)
2062 {
2063         struct nvme_dev *dev = f->private_data;
2064         kref_put(&dev->kref, nvme_free_dev);
2065         return 0;
2066 }
2067
2068 static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
2069 {
2070         struct nvme_dev *dev = f->private_data;
2071         switch (cmd) {
2072         case NVME_IOCTL_ADMIN_CMD:
2073                 return nvme_user_admin_cmd(dev, (void __user *)arg);
2074         default:
2075                 return -ENOTTY;
2076         }
2077 }
2078
2079 static const struct file_operations nvme_dev_fops = {
2080         .owner          = THIS_MODULE,
2081         .open           = nvme_dev_open,
2082         .release        = nvme_dev_release,
2083         .unlocked_ioctl = nvme_dev_ioctl,
2084         .compat_ioctl   = nvme_dev_ioctl,
2085 };
2086
2087 static int nvme_dev_start(struct nvme_dev *dev)
2088 {
2089         int result;
2090
2091         result = nvme_dev_map(dev);
2092         if (result)
2093                 return result;
2094
2095         result = nvme_configure_admin_queue(dev);
2096         if (result)
2097                 goto unmap;
2098
2099         spin_lock(&dev_list_lock);
2100         list_add(&dev->node, &dev_list);
2101         spin_unlock(&dev_list_lock);
2102
2103         result = nvme_setup_io_queues(dev);
2104         if (result)
2105                 goto disable;
2106
2107         return 0;
2108
2109  disable:
2110         spin_lock(&dev_list_lock);
2111         list_del_init(&dev->node);
2112         spin_unlock(&dev_list_lock);
2113  unmap:
2114         nvme_dev_unmap(dev);
2115         return result;
2116 }
2117
2118 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2119 {
2120         int result = -ENOMEM;
2121         struct nvme_dev *dev;
2122
2123         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2124         if (!dev)
2125                 return -ENOMEM;
2126         dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
2127                                                                 GFP_KERNEL);
2128         if (!dev->entry)
2129                 goto free;
2130         dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
2131                                                                 GFP_KERNEL);
2132         if (!dev->queues)
2133                 goto free;
2134
2135         INIT_LIST_HEAD(&dev->namespaces);
2136         dev->pci_dev = pdev;
2137         result = nvme_set_instance(dev);
2138         if (result)
2139                 goto free;
2140
2141         result = nvme_setup_prp_pools(dev);
2142         if (result)
2143                 goto release;
2144
2145         result = nvme_dev_start(dev);
2146         if (result)
2147                 goto release_pools;
2148
2149         result = nvme_dev_add(dev);
2150         if (result && result != -EBUSY)
2151                 goto shutdown;
2152
2153         scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
2154         dev->miscdev.minor = MISC_DYNAMIC_MINOR;
2155         dev->miscdev.parent = &pdev->dev;
2156         dev->miscdev.name = dev->name;
2157         dev->miscdev.fops = &nvme_dev_fops;
2158         result = misc_register(&dev->miscdev);
2159         if (result)
2160                 goto remove;
2161
2162         kref_init(&dev->kref);
2163         return 0;
2164
2165  remove:
2166         nvme_dev_remove(dev);
2167  shutdown:
2168         nvme_dev_shutdown(dev);
2169  release_pools:
2170         nvme_free_queues(dev);
2171         nvme_release_prp_pools(dev);
2172  release:
2173         nvme_release_instance(dev);
2174  free:
2175         kfree(dev->queues);
2176         kfree(dev->entry);
2177         kfree(dev);
2178         return result;
2179 }
2180
2181 static void nvme_remove(struct pci_dev *pdev)
2182 {
2183         struct nvme_dev *dev = pci_get_drvdata(pdev);
2184         misc_deregister(&dev->miscdev);
2185         kref_put(&dev->kref, nvme_free_dev);
2186 }
2187
2188 /* These functions are yet to be implemented */
2189 #define nvme_error_detected NULL
2190 #define nvme_dump_registers NULL
2191 #define nvme_link_reset NULL
2192 #define nvme_slot_reset NULL
2193 #define nvme_error_resume NULL
2194 #define nvme_suspend NULL
2195 #define nvme_resume NULL
2196
2197 static const struct pci_error_handlers nvme_err_handler = {
2198         .error_detected = nvme_error_detected,
2199         .mmio_enabled   = nvme_dump_registers,
2200         .link_reset     = nvme_link_reset,
2201         .slot_reset     = nvme_slot_reset,
2202         .resume         = nvme_error_resume,
2203 };
2204
2205 /* Move to pci_ids.h later */
2206 #define PCI_CLASS_STORAGE_EXPRESS       0x010802
2207
2208 static DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
2209         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2210         { 0, }
2211 };
2212 MODULE_DEVICE_TABLE(pci, nvme_id_table);
2213
2214 static struct pci_driver nvme_driver = {
2215         .name           = "nvme",
2216         .id_table       = nvme_id_table,
2217         .probe          = nvme_probe,
2218         .remove         = nvme_remove,
2219         .suspend        = nvme_suspend,
2220         .resume         = nvme_resume,
2221         .err_handler    = &nvme_err_handler,
2222 };
2223
2224 static int __init nvme_init(void)
2225 {
2226         int result;
2227
2228         nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2229         if (IS_ERR(nvme_thread))
2230                 return PTR_ERR(nvme_thread);
2231
2232         result = register_blkdev(nvme_major, "nvme");
2233         if (result < 0)
2234                 goto kill_kthread;
2235         else if (result > 0)
2236                 nvme_major = result;
2237
2238         result = pci_register_driver(&nvme_driver);
2239         if (result)
2240                 goto unregister_blkdev;
2241         return 0;
2242
2243  unregister_blkdev:
2244         unregister_blkdev(nvme_major, "nvme");
2245  kill_kthread:
2246         kthread_stop(nvme_thread);
2247         return result;
2248 }
2249
2250 static void __exit nvme_exit(void)
2251 {
2252         pci_unregister_driver(&nvme_driver);
2253         unregister_blkdev(nvme_major, "nvme");
2254         kthread_stop(nvme_thread);
2255 }
2256
2257 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2258 MODULE_LICENSE("GPL");
2259 MODULE_VERSION("0.8");
2260 module_init(nvme_init);
2261 module_exit(nvme_exit);