drivers/nvme/host/pci.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/acpi.h>
   8 #include <linux/aer.h>
   9 #include <linux/async.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/blk-mq.h>
  12 #include <linux/blk-mq-pci.h>
  13 #include <linux/blk-integrity.h>
  14 #include <linux/dmi.h>
  15 #include <linux/init.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/io.h>
  18 #include <linux/memremap.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21 #include <linux/mutex.h>
  22 #include <linux/once.h>
  23 #include <linux/pci.h>
  24 #include <linux/suspend.h>
  25 #include <linux/t10-pi.h>
  26 #include <linux/types.h>
  27 #include <linux/io-64-nonatomic-lo-hi.h>
  28 #include <linux/io-64-nonatomic-hi-lo.h>
  29 #include <linux/sed-opal.h>
  30 #include <linux/pci-p2pdma.h>
  31
  32 #include "trace.h"
  33 #include "nvme.h"
  34
  35 #define SQ_SIZE(q)      ((q)->q_depth << (q)->sqes)
  36 #define CQ_SIZE(q)      ((q)->q_depth * sizeof(struct nvme_completion))
  37
  38 #define SGES_PER_PAGE   (PAGE_SIZE / sizeof(struct nvme_sgl_desc))
  39
  40 /*
  41  * These can be higher, but we need to ensure that any command doesn't
  42  * require an sg allocation that needs more than a page of data.
  43  */
  44 #define NVME_MAX_KB_SZ  4096
  45 #define NVME_MAX_SEGS   127
  46
  47 static int use_threaded_interrupts;
  48 module_param(use_threaded_interrupts, int, 0444);
  49
  50 static bool use_cmb_sqes = true;
  51 module_param(use_cmb_sqes, bool, 0444);
  52 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
  53
  54 static unsigned int max_host_mem_size_mb = 128;
  55 module_param(max_host_mem_size_mb, uint, 0444);
  56 MODULE_PARM_DESC(max_host_mem_size_mb,
  57         "Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
  58
  59 static unsigned int sgl_threshold = SZ_32K;
  60 module_param(sgl_threshold, uint, 0644);
  61 MODULE_PARM_DESC(sgl_threshold,
  62                 "Use SGLs when average request segment size is larger or equal to "
  63                 "this size. Use 0 to disable SGLs.");
  64
  65 #define NVME_PCI_MIN_QUEUE_SIZE 2
  66 #define NVME_PCI_MAX_QUEUE_SIZE 4095
  67 static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
  68 static const struct kernel_param_ops io_queue_depth_ops = {
  69         .set = io_queue_depth_set,
  70         .get = param_get_uint,
  71 };
  72
  73 static unsigned int io_queue_depth = 1024;
  74 module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
  75 MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2 and < 4096");
  76
  77 static int io_queue_count_set(const char *val, const struct kernel_param *kp)
  78 {
  79         unsigned int n;
  80         int ret;
  81
  82         ret = kstrtouint(val, 10, &n);
  83         if (ret != 0 || n > num_possible_cpus())
  84                 return -EINVAL;
  85         return param_set_uint(val, kp);
  86 }
  87
  88 static const struct kernel_param_ops io_queue_count_ops = {
  89         .set = io_queue_count_set,
  90         .get = param_get_uint,
  91 };
  92
  93 static unsigned int write_queues;
  94 module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644);
  95 MODULE_PARM_DESC(write_queues,
  96         "Number of queues to use for writes. If not set, reads and writes "
  97         "will share a queue set.");
  98
  99 static unsigned int poll_queues;
 100 module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644);
 101 MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");
 102
 103 static bool noacpi;
 104 module_param(noacpi, bool, 0444);
 105 MODULE_PARM_DESC(noacpi, "disable acpi bios quirks");
 106
 107 struct nvme_dev;
 108 struct nvme_queue;
 109
 110 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
 111 static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode);
 112
 113 /*
 114  * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 115  */
 116 struct nvme_dev {
 117         struct nvme_queue *queues;
 118         struct blk_mq_tag_set tagset;
 119         struct blk_mq_tag_set admin_tagset;
 120         u32 __iomem *dbs;
 121         struct device *dev;
 122         struct dma_pool *prp_page_pool;
 123         struct dma_pool *prp_small_pool;
 124         unsigned online_queues;
 125         unsigned max_qid;
 126         unsigned io_queues[HCTX_MAX_TYPES];
 127         unsigned int num_vecs;
 128         u32 q_depth;
 129         int io_sqes;
 130         u32 db_stride;
 131         void __iomem *bar;
 132         unsigned long bar_mapped_size;
 133         struct work_struct remove_work;
 134         struct mutex shutdown_lock;
 135         bool subsystem;
 136         u64 cmb_size;
 137         bool cmb_use_sqes;
 138         u32 cmbsz;
 139         u32 cmbloc;
 140         struct nvme_ctrl ctrl;
 141         u32 last_ps;
 142         bool hmb;
 143
 144         mempool_t *iod_mempool;
 145
 146         /* shadow doorbell buffer support: */
 147         u32 *dbbuf_dbs;
 148         dma_addr_t dbbuf_dbs_dma_addr;
 149         u32 *dbbuf_eis;
 150         dma_addr_t dbbuf_eis_dma_addr;
 151
 152         /* host memory buffer support: */
 153         u64 host_mem_size;
 154         u32 nr_host_mem_descs;
 155         dma_addr_t host_mem_descs_dma;
 156         struct nvme_host_mem_buf_desc *host_mem_descs;
 157         void **host_mem_desc_bufs;
 158         unsigned int nr_allocated_queues;
 159         unsigned int nr_write_queues;
 160         unsigned int nr_poll_queues;
 161
 162         bool attrs_added;
 163 };
 164
 165 static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
 166 {
 167         return param_set_uint_minmax(val, kp, NVME_PCI_MIN_QUEUE_SIZE,
 168                         NVME_PCI_MAX_QUEUE_SIZE);
 169 }
 170
 171 static inline unsigned int sq_idx(unsigned int qid, u32 stride)
 172 {
 173         return qid * 2 * stride;
 174 }
 175
 176 static inline unsigned int cq_idx(unsigned int qid, u32 stride)
 177 {
 178         return (qid * 2 + 1) * stride;
 179 }
 180
 181 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 182 {
 183         return container_of(ctrl, struct nvme_dev, ctrl);
 184 }
 185
 186 /*
 187  * An NVM Express queue.  Each device has at least two (one for admin
 188  * commands and one for I/O commands).
 189  */
 190 struct nvme_queue {
 191         struct nvme_dev *dev;
 192         spinlock_t sq_lock;
 193         void *sq_cmds;
 194          /* only used for poll queues: */
 195         spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
 196         struct nvme_completion *cqes;
 197         dma_addr_t sq_dma_addr;
 198         dma_addr_t cq_dma_addr;
 199         u32 __iomem *q_db;
 200         u32 q_depth;
 201         u16 cq_vector;
 202         u16 sq_tail;
 203         u16 last_sq_tail;
 204         u16 cq_head;
 205         u16 qid;
 206         u8 cq_phase;
 207         u8 sqes;
 208         unsigned long flags;
 209 #define NVMEQ_ENABLED           0
 210 #define NVMEQ_SQ_CMB            1
 211 #define NVMEQ_DELETE_ERROR      2
 212 #define NVMEQ_POLLED            3
 213         u32 *dbbuf_sq_db;
 214         u32 *dbbuf_cq_db;
 215         u32 *dbbuf_sq_ei;
 216         u32 *dbbuf_cq_ei;
 217         struct completion delete_done;
 218 };
 219
 220 /*
 221  * The nvme_iod describes the data in an I/O.
 222  *
 223  * The sg pointer contains the list of PRP/SGL chunk allocations in addition
 224  * to the actual struct scatterlist.
 225  */
 226 struct nvme_iod {
 227         struct nvme_request req;
 228         struct nvme_command cmd;
 229         bool use_sgl;
 230         int aborted;
 231         int npages;             /* In the PRP list. 0 means small pool in use */
 232         dma_addr_t first_dma;
 233         unsigned int dma_len;   /* length of single DMA segment mapping */
 234         dma_addr_t meta_dma;
 235         struct sg_table sgt;
 236 };
 237
 238 static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev)
 239 {
 240         return dev->nr_allocated_queues * 8 * dev->db_stride;
 241 }
 242
 243 static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
 244 {
 245         unsigned int mem_size = nvme_dbbuf_size(dev);
 246
 247         if (dev->dbbuf_dbs) {
 248                 /*
 249                  * Clear the dbbuf memory so the driver doesn't observe stale
 250                  * values from the previous instantiation.
 251                  */
 252                 memset(dev->dbbuf_dbs, 0, mem_size);
 253                 memset(dev->dbbuf_eis, 0, mem_size);
 254                 return 0;
 255         }
 256
 257         dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
 258                                             &dev->dbbuf_dbs_dma_addr,
 259                                             GFP_KERNEL);
 260         if (!dev->dbbuf_dbs)
 261                 return -ENOMEM;
 262         dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
 263                                             &dev->dbbuf_eis_dma_addr,
 264                                             GFP_KERNEL);
 265         if (!dev->dbbuf_eis) {
 266                 dma_free_coherent(dev->dev, mem_size,
 267                                   dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
 268                 dev->dbbuf_dbs = NULL;
 269                 return -ENOMEM;
 270         }
 271
 272         return 0;
 273 }
 274
 275 static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
 276 {
 277         unsigned int mem_size = nvme_dbbuf_size(dev);
 278
 279         if (dev->dbbuf_dbs) {
 280                 dma_free_coherent(dev->dev, mem_size,
 281                                   dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
 282                 dev->dbbuf_dbs = NULL;
 283         }
 284         if (dev->dbbuf_eis) {
 285                 dma_free_coherent(dev->dev, mem_size,
 286                                   dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
 287                 dev->dbbuf_eis = NULL;
 288         }
 289 }
 290
 291 static void nvme_dbbuf_init(struct nvme_dev *dev,
 292                             struct nvme_queue *nvmeq, int qid)
 293 {
 294         if (!dev->dbbuf_dbs || !qid)
 295                 return;
 296
 297         nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
 298         nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
 299         nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
 300         nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
 301 }
 302
 303 static void nvme_dbbuf_free(struct nvme_queue *nvmeq)
 304 {
 305         if (!nvmeq->qid)
 306                 return;
 307
 308         nvmeq->dbbuf_sq_db = NULL;
 309         nvmeq->dbbuf_cq_db = NULL;
 310         nvmeq->dbbuf_sq_ei = NULL;
 311         nvmeq->dbbuf_cq_ei = NULL;
 312 }
 313
 314 static void nvme_dbbuf_set(struct nvme_dev *dev)
 315 {
 316         struct nvme_command c = { };
 317         unsigned int i;
 318
 319         if (!dev->dbbuf_dbs)
 320                 return;
 321
 322         c.dbbuf.opcode = nvme_admin_dbbuf;
 323         c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
 324         c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
 325
 326         if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
 327                 dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
 328                 /* Free memory and continue on */
 329                 nvme_dbbuf_dma_free(dev);
 330
 331                 for (i = 1; i <= dev->online_queues; i++)
 332                         nvme_dbbuf_free(&dev->queues[i]);
 333         }
 334 }
 335
 336 static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
 337 {
 338         return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
 339 }
 340
 341 /* Update dbbuf and return true if an MMIO is required */
 342 static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
 343                                               volatile u32 *dbbuf_ei)
 344 {
 345         if (dbbuf_db) {
 346                 u16 old_value;
 347
 348                 /*
 349                  * Ensure that the queue is written before updating
 350                  * the doorbell in memory
 351                  */
 352                 wmb();
 353
 354                 old_value = *dbbuf_db;
 355                 *dbbuf_db = value;
 356
 357                 /*
 358                  * Ensure that the doorbell is updated before reading the event
 359                  * index from memory.  The controller needs to provide similar
 360                  * ordering to ensure the envent index is updated before reading
 361                  * the doorbell.
 362                  */
 363                 mb();
 364
 365                 if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
 366                         return false;
 367         }
 368
 369         return true;
 370 }
 371
 372 /*
 373  * Will slightly overestimate the number of pages needed.  This is OK
 374  * as it only leads to a small amount of wasted memory for the lifetime of
 375  * the I/O.
 376  */
 377 static int nvme_pci_npages_prp(void)
 378 {
 379         unsigned nprps = DIV_ROUND_UP(NVME_MAX_KB_SZ + NVME_CTRL_PAGE_SIZE,
 380                                       NVME_CTRL_PAGE_SIZE);
 381         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 382 }
 383
 384 /*
 385  * Calculates the number of pages needed for the SGL segments. For example a 4k
 386  * page can accommodate 256 SGL descriptors.
 387  */
 388 static int nvme_pci_npages_sgl(void)
 389 {
 390         return DIV_ROUND_UP(NVME_MAX_SEGS * sizeof(struct nvme_sgl_desc),
 391                         PAGE_SIZE);
 392 }
 393
 394 static size_t nvme_pci_iod_alloc_size(void)
 395 {
 396         size_t npages = max(nvme_pci_npages_prp(), nvme_pci_npages_sgl());
 397
 398         return sizeof(__le64 *) * npages +
 399                 sizeof(struct scatterlist) * NVME_MAX_SEGS;
 400 }
 401
 402 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 403                                 unsigned int hctx_idx)
 404 {
 405         struct nvme_dev *dev = data;
 406         struct nvme_queue *nvmeq = &dev->queues[0];
 407
 408         WARN_ON(hctx_idx != 0);
 409         WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
 410
 411         hctx->driver_data = nvmeq;
 412         return 0;
 413 }
 414
 415 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 416                           unsigned int hctx_idx)
 417 {
 418         struct nvme_dev *dev = data;
 419         struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
 420
 421         WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
 422         hctx->driver_data = nvmeq;
 423         return 0;
 424 }
 425
 426 static int nvme_pci_init_request(struct blk_mq_tag_set *set,
 427                 struct request *req, unsigned int hctx_idx,
 428                 unsigned int numa_node)
 429 {
 430         struct nvme_dev *dev = set->driver_data;
 431         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 432
 433         nvme_req(req)->ctrl = &dev->ctrl;
 434         nvme_req(req)->cmd = &iod->cmd;
 435         return 0;
 436 }
 437
 438 static int queue_irq_offset(struct nvme_dev *dev)
 439 {
 440         /* if we have more than 1 vec, admin queue offsets us by 1 */
 441         if (dev->num_vecs > 1)
 442                 return 1;
 443
 444         return 0;
 445 }
 446
 447 static void nvme_pci_map_queues(struct blk_mq_tag_set *set)
 448 {
 449         struct nvme_dev *dev = set->driver_data;
 450         int i, qoff, offset;
 451
 452         offset = queue_irq_offset(dev);
 453         for (i = 0, qoff = 0; i < set->nr_maps; i++) {
 454                 struct blk_mq_queue_map *map = &set->map[i];
 455
 456                 map->nr_queues = dev->io_queues[i];
 457                 if (!map->nr_queues) {
 458                         BUG_ON(i == HCTX_TYPE_DEFAULT);
 459                         continue;
 460                 }
 461
 462                 /*
 463                  * The poll queue(s) doesn't have an IRQ (and hence IRQ
 464                  * affinity), so use the regular blk-mq cpu mapping
 465                  */
 466                 map->queue_offset = qoff;
 467                 if (i != HCTX_TYPE_POLL && offset)
 468                         blk_mq_pci_map_queues(map, to_pci_dev(dev->dev), offset);
 469                 else
 470                         blk_mq_map_queues(map);
 471                 qoff += map->nr_queues;
 472                 offset += map->nr_queues;
 473         }
 474 }
 475
 476 /*
 477  * Write sq tail if we are asked to, or if the next command would wrap.
 478  */
 479 static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq)
 480 {
 481         if (!write_sq) {
 482                 u16 next_tail = nvmeq->sq_tail + 1;
 483
 484                 if (next_tail == nvmeq->q_depth)
 485                         next_tail = 0;
 486                 if (next_tail != nvmeq->last_sq_tail)
 487                         return;
 488         }
 489
 490         if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,
 491                         nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))
 492                 writel(nvmeq->sq_tail, nvmeq->q_db);
 493         nvmeq->last_sq_tail = nvmeq->sq_tail;
 494 }
 495
 496 static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq,
 497                                     struct nvme_command *cmd)
 498 {
 499         memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes),
 500                 absolute_pointer(cmd), sizeof(*cmd));
 501         if (++nvmeq->sq_tail == nvmeq->q_depth)
 502                 nvmeq->sq_tail = 0;
 503 }
 504
 505 static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)
 506 {
 507         struct nvme_queue *nvmeq = hctx->driver_data;
 508
 509         spin_lock(&nvmeq->sq_lock);
 510         if (nvmeq->sq_tail != nvmeq->last_sq_tail)
 511                 nvme_write_sq_db(nvmeq, true);
 512         spin_unlock(&nvmeq->sq_lock);
 513 }
 514
 515 static void **nvme_pci_iod_list(struct request *req)
 516 {
 517         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 518         return (void **)(iod->sgt.sgl + blk_rq_nr_phys_segments(req));
 519 }
 520
 521 static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
 522 {
 523         struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 524         int nseg = blk_rq_nr_phys_segments(req);
 525         unsigned int avg_seg_size;
 526
 527         avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
 528
 529         if (!nvme_ctrl_sgl_supported(&dev->ctrl))
 530                 return false;
 531         if (!nvmeq->qid)
 532                 return false;
 533         if (!sgl_threshold || avg_seg_size < sgl_threshold)
 534                 return false;
 535         return true;
 536 }
 537
 538 static void nvme_free_prps(struct nvme_dev *dev, struct request *req)
 539 {
 540         const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
 541         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 542         dma_addr_t dma_addr = iod->first_dma;
 543         int i;
 544
 545         for (i = 0; i < iod->npages; i++) {
 546                 __le64 *prp_list = nvme_pci_iod_list(req)[i];
 547                 dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
 548
 549                 dma_pool_free(dev->prp_page_pool, prp_list, dma_addr);
 550                 dma_addr = next_dma_addr;
 551         }
 552 }
 553
 554 static void nvme_free_sgls(struct nvme_dev *dev, struct request *req)
 555 {
 556         const int last_sg = SGES_PER_PAGE - 1;
 557         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 558         dma_addr_t dma_addr = iod->first_dma;
 559         int i;
 560
 561         for (i = 0; i < iod->npages; i++) {
 562                 struct nvme_sgl_desc *sg_list = nvme_pci_iod_list(req)[i];
 563                 dma_addr_t next_dma_addr = le64_to_cpu((sg_list[last_sg]).addr);
 564
 565                 dma_pool_free(dev->prp_page_pool, sg_list, dma_addr);
 566                 dma_addr = next_dma_addr;
 567         }
 568 }
 569
 570 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 571 {
 572         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 573
 574         if (iod->dma_len) {
 575                 dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
 576                                rq_dma_dir(req));
 577                 return;
 578         }
 579
 580         WARN_ON_ONCE(!iod->sgt.nents);
 581
 582         dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
 583
 584         if (iod->npages == 0)
 585                 dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
 586                               iod->first_dma);
 587         else if (iod->use_sgl)
 588                 nvme_free_sgls(dev, req);
 589         else
 590                 nvme_free_prps(dev, req);
 591         mempool_free(iod->sgt.sgl, dev->iod_mempool);
 592 }
 593
 594 static void nvme_print_sgl(struct scatterlist *sgl, int nents)
 595 {
 596         int i;
 597         struct scatterlist *sg;
 598
 599         for_each_sg(sgl, sg, nents, i) {
 600                 dma_addr_t phys = sg_phys(sg);
 601                 pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
 602                         "dma_address:%pad dma_length:%d\n",
 603                         i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
 604                         sg_dma_len(sg));
 605         }
 606 }
 607
 608 static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
 609                 struct request *req, struct nvme_rw_command *cmnd)
 610 {
 611         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 612         struct dma_pool *pool;
 613         int length = blk_rq_payload_bytes(req);
 614         struct scatterlist *sg = iod->sgt.sgl;
 615         int dma_len = sg_dma_len(sg);
 616         u64 dma_addr = sg_dma_address(sg);
 617         int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
 618         __le64 *prp_list;
 619         void **list = nvme_pci_iod_list(req);
 620         dma_addr_t prp_dma;
 621         int nprps, i;
 622
 623         length -= (NVME_CTRL_PAGE_SIZE - offset);
 624         if (length <= 0) {
 625                 iod->first_dma = 0;
 626                 goto done;
 627         }
 628
 629         dma_len -= (NVME_CTRL_PAGE_SIZE - offset);
 630         if (dma_len) {
 631                 dma_addr += (NVME_CTRL_PAGE_SIZE - offset);
 632         } else {
 633                 sg = sg_next(sg);
 634                 dma_addr = sg_dma_address(sg);
 635                 dma_len = sg_dma_len(sg);
 636         }
 637
 638         if (length <= NVME_CTRL_PAGE_SIZE) {
 639                 iod->first_dma = dma_addr;
 640                 goto done;
 641         }
 642
 643         nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
 644         if (nprps <= (256 / 8)) {
 645                 pool = dev->prp_small_pool;
 646                 iod->npages = 0;
 647         } else {
 648                 pool = dev->prp_page_pool;
 649                 iod->npages = 1;
 650         }
 651
 652         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 653         if (!prp_list) {
 654                 iod->npages = -1;
 655                 return BLK_STS_RESOURCE;
 656         }
 657         list[0] = prp_list;
 658         iod->first_dma = prp_dma;
 659         i = 0;
 660         for (;;) {
 661                 if (i == NVME_CTRL_PAGE_SIZE >> 3) {
 662                         __le64 *old_prp_list = prp_list;
 663                         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 664                         if (!prp_list)
 665                                 goto free_prps;
 666                         list[iod->npages++] = prp_list;
 667                         prp_list[0] = old_prp_list[i - 1];
 668                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 669                         i = 1;
 670                 }
 671                 prp_list[i++] = cpu_to_le64(dma_addr);
 672                 dma_len -= NVME_CTRL_PAGE_SIZE;
 673                 dma_addr += NVME_CTRL_PAGE_SIZE;
 674                 length -= NVME_CTRL_PAGE_SIZE;
 675                 if (length <= 0)
 676                         break;
 677                 if (dma_len > 0)
 678                         continue;
 679                 if (unlikely(dma_len < 0))
 680                         goto bad_sgl;
 681                 sg = sg_next(sg);
 682                 dma_addr = sg_dma_address(sg);
 683                 dma_len = sg_dma_len(sg);
 684         }
 685 done:
 686         cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sgt.sgl));
 687         cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
 688         return BLK_STS_OK;
 689 free_prps:
 690         nvme_free_prps(dev, req);
 691         return BLK_STS_RESOURCE;
 692 bad_sgl:
 693         WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents),
 694                         "Invalid SGL for payload:%d nents:%d\n",
 695                         blk_rq_payload_bytes(req), iod->sgt.nents);
 696         return BLK_STS_IOERR;
 697 }
 698
 699 static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
 700                 struct scatterlist *sg)
 701 {
 702         sge->addr = cpu_to_le64(sg_dma_address(sg));
 703         sge->length = cpu_to_le32(sg_dma_len(sg));
 704         sge->type = NVME_SGL_FMT_DATA_DESC << 4;
 705 }
 706
 707 static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
 708                 dma_addr_t dma_addr, int entries)
 709 {
 710         sge->addr = cpu_to_le64(dma_addr);
 711         if (entries < SGES_PER_PAGE) {
 712                 sge->length = cpu_to_le32(entries * sizeof(*sge));
 713                 sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
 714         } else {
 715                 sge->length = cpu_to_le32(PAGE_SIZE);
 716                 sge->type = NVME_SGL_FMT_SEG_DESC << 4;
 717         }
 718 }
 719
 720 static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
 721                 struct request *req, struct nvme_rw_command *cmd)
 722 {
 723         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 724         struct dma_pool *pool;
 725         struct nvme_sgl_desc *sg_list;
 726         struct scatterlist *sg = iod->sgt.sgl;
 727         unsigned int entries = iod->sgt.nents;
 728         dma_addr_t sgl_dma;
 729         int i = 0;
 730
 731         /* setting the transfer type as SGL */
 732         cmd->flags = NVME_CMD_SGL_METABUF;
 733
 734         if (entries == 1) {
 735                 nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
 736                 return BLK_STS_OK;
 737         }
 738
 739         if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
 740                 pool = dev->prp_small_pool;
 741                 iod->npages = 0;
 742         } else {
 743                 pool = dev->prp_page_pool;
 744                 iod->npages = 1;
 745         }
 746
 747         sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
 748         if (!sg_list) {
 749                 iod->npages = -1;
 750                 return BLK_STS_RESOURCE;
 751         }
 752
 753         nvme_pci_iod_list(req)[0] = sg_list;
 754         iod->first_dma = sgl_dma;
 755
 756         nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);
 757
 758         do {
 759                 if (i == SGES_PER_PAGE) {
 760                         struct nvme_sgl_desc *old_sg_desc = sg_list;
 761                         struct nvme_sgl_desc *link = &old_sg_desc[i - 1];
 762
 763                         sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
 764                         if (!sg_list)
 765                                 goto free_sgls;
 766
 767                         i = 0;
 768                         nvme_pci_iod_list(req)[iod->npages++] = sg_list;
 769                         sg_list[i++] = *link;
 770                         nvme_pci_sgl_set_seg(link, sgl_dma, entries);
 771                 }
 772
 773                 nvme_pci_sgl_set_data(&sg_list[i++], sg);
 774                 sg = sg_next(sg);
 775         } while (--entries > 0);
 776
 777         return BLK_STS_OK;
 778 free_sgls:
 779         nvme_free_sgls(dev, req);
 780         return BLK_STS_RESOURCE;
 781 }
 782
 783 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
 784                 struct request *req, struct nvme_rw_command *cmnd,
 785                 struct bio_vec *bv)
 786 {
 787         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 788         unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
 789         unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset;
 790
 791         iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
 792         if (dma_mapping_error(dev->dev, iod->first_dma))
 793                 return BLK_STS_RESOURCE;
 794         iod->dma_len = bv->bv_len;
 795
 796         cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);
 797         if (bv->bv_len > first_prp_len)
 798                 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);
 799         return BLK_STS_OK;
 800 }
 801
 802 static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev,
 803                 struct request *req, struct nvme_rw_command *cmnd,
 804                 struct bio_vec *bv)
 805 {
 806         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 807
 808         iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
 809         if (dma_mapping_error(dev->dev, iod->first_dma))
 810                 return BLK_STS_RESOURCE;
 811         iod->dma_len = bv->bv_len;
 812
 813         cmnd->flags = NVME_CMD_SGL_METABUF;
 814         cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma);
 815         cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len);
 816         cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4;
 817         return BLK_STS_OK;
 818 }
 819
 820 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 821                 struct nvme_command *cmnd)
 822 {
 823         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 824         blk_status_t ret = BLK_STS_RESOURCE;
 825         int rc;
 826
 827         if (blk_rq_nr_phys_segments(req) == 1) {
 828                 struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 829                 struct bio_vec bv = req_bvec(req);
 830
 831                 if (!is_pci_p2pdma_page(bv.bv_page)) {
 832                         if (bv.bv_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)
 833                                 return nvme_setup_prp_simple(dev, req,
 834                                                              &cmnd->rw, &bv);
 835
 836                         if (nvmeq->qid && sgl_threshold &&
 837                             nvme_ctrl_sgl_supported(&dev->ctrl))
 838                                 return nvme_setup_sgl_simple(dev, req,
 839                                                              &cmnd->rw, &bv);
 840                 }
 841         }
 842
 843         iod->dma_len = 0;
 844         iod->sgt.sgl = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
 845         if (!iod->sgt.sgl)
 846                 return BLK_STS_RESOURCE;
 847         sg_init_table(iod->sgt.sgl, blk_rq_nr_phys_segments(req));
 848         iod->sgt.orig_nents = blk_rq_map_sg(req->q, req, iod->sgt.sgl);
 849         if (!iod->sgt.orig_nents)
 850                 goto out_free_sg;
 851
 852         rc = dma_map_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req),
 853                              DMA_ATTR_NO_WARN);
 854         if (rc) {
 855                 if (rc == -EREMOTEIO)
 856                         ret = BLK_STS_TARGET;
 857                 goto out_free_sg;
 858         }
 859
 860         iod->use_sgl = nvme_pci_use_sgls(dev, req);
 861         if (iod->use_sgl)
 862                 ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw);
 863         else
 864                 ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
 865         if (ret != BLK_STS_OK)
 866                 goto out_unmap_sg;
 867         return BLK_STS_OK;
 868
 869 out_unmap_sg:
 870         dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
 871 out_free_sg:
 872         mempool_free(iod->sgt.sgl, dev->iod_mempool);
 873         return ret;
 874 }
 875
 876 static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,
 877                 struct nvme_command *cmnd)
 878 {
 879         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 880
 881         iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
 882                         rq_dma_dir(req), 0);
 883         if (dma_mapping_error(dev->dev, iod->meta_dma))
 884                 return BLK_STS_IOERR;
 885         cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
 886         return BLK_STS_OK;
 887 }
 888
 889 static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
 890 {
 891         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 892         blk_status_t ret;
 893
 894         iod->aborted = 0;
 895         iod->npages = -1;
 896         iod->sgt.nents = 0;
 897
 898         ret = nvme_setup_cmd(req->q->queuedata, req);
 899         if (ret)
 900                 return ret;
 901
 902         if (blk_rq_nr_phys_segments(req)) {
 903                 ret = nvme_map_data(dev, req, &iod->cmd);
 904                 if (ret)
 905                         goto out_free_cmd;
 906         }
 907
 908         if (blk_integrity_rq(req)) {
 909                 ret = nvme_map_metadata(dev, req, &iod->cmd);
 910                 if (ret)
 911                         goto out_unmap_data;
 912         }
 913
 914         blk_mq_start_request(req);
 915         return BLK_STS_OK;
 916 out_unmap_data:
 917         nvme_unmap_data(dev, req);
 918 out_free_cmd:
 919         nvme_cleanup_cmd(req);
 920         return ret;
 921 }
 922
 923 /*
 924  * NOTE: ns is NULL when called on the admin queue.
 925  */
 926 static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 927                          const struct blk_mq_queue_data *bd)
 928 {
 929         struct nvme_queue *nvmeq = hctx->driver_data;
 930         struct nvme_dev *dev = nvmeq->dev;
 931         struct request *req = bd->rq;
 932         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 933         blk_status_t ret;
 934
 935         /*
 936          * We should not need to do this, but we're still using this to
 937          * ensure we can drain requests on a dying queue.
 938          */
 939         if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
 940                 return BLK_STS_IOERR;
 941
 942         if (unlikely(!nvme_check_ready(&dev->ctrl, req, true)))
 943                 return nvme_fail_nonready_command(&dev->ctrl, req);
 944
 945         ret = nvme_prep_rq(dev, req);
 946         if (unlikely(ret))
 947                 return ret;
 948         spin_lock(&nvmeq->sq_lock);
 949         nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 950         nvme_write_sq_db(nvmeq, bd->last);
 951         spin_unlock(&nvmeq->sq_lock);
 952         return BLK_STS_OK;
 953 }
 954
 955 static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)
 956 {
 957         spin_lock(&nvmeq->sq_lock);
 958         while (!rq_list_empty(*rqlist)) {
 959                 struct request *req = rq_list_pop(rqlist);
 960                 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 961
 962                 nvme_sq_copy_cmd(nvmeq, &iod->cmd);
 963         }
 964         nvme_write_sq_db(nvmeq, true);
 965         spin_unlock(&nvmeq->sq_lock);
 966 }
 967
 968 static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
 969 {
 970         /*
 971          * We should not need to do this, but we're still using this to
 972          * ensure we can drain requests on a dying queue.
 973          */
 974         if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
 975                 return false;
 976         if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
 977                 return false;
 978
 979         req->mq_hctx->tags->rqs[req->tag] = req;
 980         return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
 981 }
 982
 983 static void nvme_queue_rqs(struct request **rqlist)
 984 {
 985         struct request *req, *next, *prev = NULL;
 986         struct request *requeue_list = NULL;
 987
 988         rq_list_for_each_safe(rqlist, req, next) {
 989                 struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 990
 991                 if (!nvme_prep_rq_batch(nvmeq, req)) {
 992                         /* detach 'req' and add to remainder list */
 993                         rq_list_move(rqlist, &requeue_list, req, prev);
 994
 995                         req = prev;
 996                         if (!req)
 997                                 continue;
 998                 }
 999
1000                 if (!next || req->mq_hctx != next->mq_hctx) {
1001                         /* detach rest of list, and submit */
1002                         req->rq_next = NULL;
1003                         nvme_submit_cmds(nvmeq, rqlist);
1004                         *rqlist = next;
1005                         prev = NULL;
1006                 } else
1007                         prev = req;
1008         }
1009
1010         *rqlist = requeue_list;
1011 }
1012
1013 static __always_inline void nvme_pci_unmap_rq(struct request *req)
1014 {
1015         struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1016         struct nvme_dev *dev = nvmeq->dev;
1017
1018         if (blk_integrity_rq(req)) {
1019                 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1020
1021                 dma_unmap_page(dev->dev, iod->meta_dma,
1022                                rq_integrity_vec(req)->bv_len, rq_data_dir(req));
1023         }
1024
1025         if (blk_rq_nr_phys_segments(req))
1026                 nvme_unmap_data(dev, req);
1027 }
1028
1029 static void nvme_pci_complete_rq(struct request *req)
1030 {
1031         nvme_pci_unmap_rq(req);
1032         nvme_complete_rq(req);
1033 }
1034
1035 static void nvme_pci_complete_batch(struct io_comp_batch *iob)
1036 {
1037         nvme_complete_batch(iob, nvme_pci_unmap_rq);
1038 }
1039
1040 /* We read the CQE phase first to check if the rest of the entry is valid */
1041 static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
1042 {
1043         struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
1044
1045         return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
1046 }
1047
1048 static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
1049 {
1050         u16 head = nvmeq->cq_head;
1051
1052         if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
1053                                               nvmeq->dbbuf_cq_ei))
1054                 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
1055 }
1056
1057 static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
1058 {
1059         if (!nvmeq->qid)
1060                 return nvmeq->dev->admin_tagset.tags[0];
1061         return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
1062 }
1063
1064 static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
1065                                    struct io_comp_batch *iob, u16 idx)
1066 {
1067         struct nvme_completion *cqe = &nvmeq->cqes[idx];
1068         __u16 command_id = READ_ONCE(cqe->command_id);
1069         struct request *req;
1070
1071         /*
1072          * AEN requests are special as they don't time out and can
1073          * survive any kind of queue freeze and often don't respond to
1074          * aborts.  We don't even bother to allocate a struct request
1075          * for them but rather special case them here.
1076          */
1077         if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) {
1078                 nvme_complete_async_event(&nvmeq->dev->ctrl,
1079                                 cqe->status, &cqe->result);
1080                 return;
1081         }
1082
1083         req = nvme_find_rq(nvme_queue_tagset(nvmeq), command_id);
1084         if (unlikely(!req)) {
1085                 dev_warn(nvmeq->dev->ctrl.device,
1086                         "invalid id %d completed on queue %d\n",
1087                         command_id, le16_to_cpu(cqe->sq_id));
1088                 return;
1089         }
1090
1091         trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
1092         if (!nvme_try_complete_req(req, cqe->status, cqe->result) &&
1093             !blk_mq_add_to_batch(req, iob, nvme_req(req)->status,
1094                                         nvme_pci_complete_batch))
1095                 nvme_pci_complete_rq(req);
1096 }
1097
1098 static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
1099 {
1100         u32 tmp = nvmeq->cq_head + 1;
1101
1102         if (tmp == nvmeq->q_depth) {
1103                 nvmeq->cq_head = 0;
1104                 nvmeq->cq_phase ^= 1;
1105         } else {
1106                 nvmeq->cq_head = tmp;
1107         }
1108 }
1109
1110 static inline int nvme_poll_cq(struct nvme_queue *nvmeq,
1111                                struct io_comp_batch *iob)
1112 {
1113         int found = 0;
1114
1115         while (nvme_cqe_pending(nvmeq)) {
1116                 found++;
1117                 /*
1118                  * load-load control dependency between phase and the rest of
1119                  * the cqe requires a full read memory barrier
1120                  */
1121                 dma_rmb();
1122                 nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
1123                 nvme_update_cq_head(nvmeq);
1124         }
1125
1126         if (found)
1127                 nvme_ring_cq_doorbell(nvmeq);
1128         return found;
1129 }
1130
1131 static irqreturn_t nvme_irq(int irq, void *data)
1132 {
1133         struct nvme_queue *nvmeq = data;
1134         DEFINE_IO_COMP_BATCH(iob);
1135
1136         if (nvme_poll_cq(nvmeq, &iob)) {
1137                 if (!rq_list_empty(iob.req_list))
1138                         nvme_pci_complete_batch(&iob);
1139                 return IRQ_HANDLED;
1140         }
1141         return IRQ_NONE;
1142 }
1143
1144 static irqreturn_t nvme_irq_check(int irq, void *data)
1145 {
1146         struct nvme_queue *nvmeq = data;
1147
1148         if (nvme_cqe_pending(nvmeq))
1149                 return IRQ_WAKE_THREAD;
1150         return IRQ_NONE;
1151 }
1152
1153 /*
1154  * Poll for completions for any interrupt driven queue
1155  * Can be called from any context.
1156  */
1157 static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
1158 {
1159         struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1160
1161         WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
1162
1163         disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
1164         nvme_poll_cq(nvmeq, NULL);
1165         enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
1166 }
1167
1168 static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1169 {
1170         struct nvme_queue *nvmeq = hctx->driver_data;
1171         bool found;
1172
1173         if (!nvme_cqe_pending(nvmeq))
1174                 return 0;
1175
1176         spin_lock(&nvmeq->cq_poll_lock);
1177         found = nvme_poll_cq(nvmeq, iob);
1178         spin_unlock(&nvmeq->cq_poll_lock);
1179
1180         return found;
1181 }
1182
1183 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
1184 {
1185         struct nvme_dev *dev = to_nvme_dev(ctrl);
1186         struct nvme_queue *nvmeq = &dev->queues[0];
1187         struct nvme_command c = { };
1188
1189         c.common.opcode = nvme_admin_async_event;
1190         c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1191
1192         spin_lock(&nvmeq->sq_lock);
1193         nvme_sq_copy_cmd(nvmeq, &c);
1194         nvme_write_sq_db(nvmeq, true);
1195         spin_unlock(&nvmeq->sq_lock);
1196 }
1197
1198 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1199 {
1200         struct nvme_command c = { };
1201
1202         c.delete_queue.opcode = opcode;
1203         c.delete_queue.qid = cpu_to_le16(id);
1204
1205         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1206 }
1207
1208 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1209                 struct nvme_queue *nvmeq, s16 vector)
1210 {
1211         struct nvme_command c = { };
1212         int flags = NVME_QUEUE_PHYS_CONTIG;
1213
1214         if (!test_bit(NVMEQ_POLLED, &nvmeq->flags))
1215                 flags |= NVME_CQ_IRQ_ENABLED;
1216
1217         /*
1218          * Note: we (ab)use the fact that the prp fields survive if no data
1219          * is attached to the request.
1220          */
1221         c.create_cq.opcode = nvme_admin_create_cq;
1222         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1223         c.create_cq.cqid = cpu_to_le16(qid);
1224         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1225         c.create_cq.cq_flags = cpu_to_le16(flags);
1226         c.create_cq.irq_vector = cpu_to_le16(vector);
1227
1228         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1229 }
1230
1231 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1232                                                 struct nvme_queue *nvmeq)
1233 {
1234         struct nvme_ctrl *ctrl = &dev->ctrl;
1235         struct nvme_command c = { };
1236         int flags = NVME_QUEUE_PHYS_CONTIG;
1237
1238         /*
1239          * Some drives have a bug that auto-enables WRRU if MEDIUM isn't
1240          * set. Since URGENT priority is zeroes, it makes all queues
1241          * URGENT.
1242          */
1243         if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
1244                 flags |= NVME_SQ_PRIO_MEDIUM;
1245
1246         /*
1247          * Note: we (ab)use the fact that the prp fields survive if no data
1248          * is attached to the request.
1249          */
1250         c.create_sq.opcode = nvme_admin_create_sq;
1251         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1252         c.create_sq.sqid = cpu_to_le16(qid);
1253         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1254         c.create_sq.sq_flags = cpu_to_le16(flags);
1255         c.create_sq.cqid = cpu_to_le16(qid);
1256
1257         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1258 }
1259
1260 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1261 {
1262         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1263 }
1264
1265 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1266 {
1267         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1268 }
1269
1270 static void abort_endio(struct request *req, blk_status_t error)
1271 {
1272         struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1273
1274         dev_warn(nvmeq->dev->ctrl.device,
1275                  "Abort status: 0x%x", nvme_req(req)->status);
1276         atomic_inc(&nvmeq->dev->ctrl.abort_limit);
1277         blk_mq_free_request(req);
1278 }
1279
1280 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
1281 {
1282         /* If true, indicates loss of adapter communication, possibly by a
1283          * NVMe Subsystem reset.
1284          */
1285         bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
1286
1287         /* If there is a reset/reinit ongoing, we shouldn't reset again. */
1288         switch (dev->ctrl.state) {
1289         case NVME_CTRL_RESETTING:
1290         case NVME_CTRL_CONNECTING:
1291                 return false;
1292         default:
1293                 break;
1294         }
1295
1296         /* We shouldn't reset unless the controller is on fatal error state
1297          * _or_ if we lost the communication with it.
1298          */
1299         if (!(csts & NVME_CSTS_CFS) && !nssro)
1300                 return false;
1301
1302         return true;
1303 }
1304
1305 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1306 {
1307         /* Read a config register to help see what died. */
1308         u16 pci_status;
1309         int result;
1310
1311         result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1312                                       &pci_status);
1313         if (result == PCIBIOS_SUCCESSFUL)
1314                 dev_warn(dev->ctrl.device,
1315                          "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
1316                          csts, pci_status);
1317         else
1318                 dev_warn(dev->ctrl.device,
1319                          "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
1320                          csts, result);
1321
1322         if (csts != ~0)
1323                 return;
1324
1325         dev_warn(dev->ctrl.device,
1326                  "Does your device have a faulty power saving mode enabled?\n");
1327         dev_warn(dev->ctrl.device,
1328                  "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off\" and report a bug\n");
1329 }
1330
1331 static enum blk_eh_timer_return nvme_timeout(struct request *req)
1332 {
1333         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1334         struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1335         struct nvme_dev *dev = nvmeq->dev;
1336         struct request *abort_req;
1337         struct nvme_command cmd = { };
1338         u32 csts = readl(dev->bar + NVME_REG_CSTS);
1339
1340         /* If PCI error recovery process is happening, we cannot reset or
1341          * the recovery mechanism will surely fail.
1342          */
1343         mb();
1344         if (pci_channel_offline(to_pci_dev(dev->dev)))
1345                 return BLK_EH_RESET_TIMER;
1346
1347         /*
1348          * Reset immediately if the controller is failed
1349          */
1350         if (nvme_should_reset(dev, csts)) {
1351                 nvme_warn_reset(dev, csts);
1352                 nvme_dev_disable(dev, false);
1353                 nvme_reset_ctrl(&dev->ctrl);
1354                 return BLK_EH_DONE;
1355         }
1356
1357         /*
1358          * Did we miss an interrupt?
1359          */
1360         if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
1361                 nvme_poll(req->mq_hctx, NULL);
1362         else
1363                 nvme_poll_irqdisable(nvmeq);
1364
1365         if (blk_mq_request_completed(req)) {
1366                 dev_warn(dev->ctrl.device,
1367                          "I/O %d QID %d timeout, completion polled\n",
1368                          req->tag, nvmeq->qid);
1369                 return BLK_EH_DONE;
1370         }
1371
1372         /*
1373          * Shutdown immediately if controller times out while starting. The
1374          * reset work will see the pci device disabled when it gets the forced
1375          * cancellation error. All outstanding requests are completed on
1376          * shutdown, so we return BLK_EH_DONE.
1377          */
1378         switch (dev->ctrl.state) {
1379         case NVME_CTRL_CONNECTING:
1380                 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
1381                 fallthrough;
1382         case NVME_CTRL_DELETING:
1383                 dev_warn_ratelimited(dev->ctrl.device,
1384                          "I/O %d QID %d timeout, disable controller\n",
1385                          req->tag, nvmeq->qid);
1386                 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1387                 nvme_dev_disable(dev, true);
1388                 return BLK_EH_DONE;
1389         case NVME_CTRL_RESETTING:
1390                 return BLK_EH_RESET_TIMER;
1391         default:
1392                 break;
1393         }
1394
1395         /*
1396          * Shutdown the controller immediately and schedule a reset if the
1397          * command was already aborted once before and still hasn't been
1398          * returned to the driver, or if this is the admin queue.
1399          */
1400         if (!nvmeq->qid || iod->aborted) {
1401                 dev_warn(dev->ctrl.device,
1402                          "I/O %d QID %d timeout, reset controller\n",
1403                          req->tag, nvmeq->qid);
1404                 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1405                 nvme_dev_disable(dev, false);
1406                 nvme_reset_ctrl(&dev->ctrl);
1407
1408                 return BLK_EH_DONE;
1409         }
1410
1411         if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1412                 atomic_inc(&dev->ctrl.abort_limit);
1413                 return BLK_EH_RESET_TIMER;
1414         }
1415         iod->aborted = 1;
1416
1417         cmd.abort.opcode = nvme_admin_abort_cmd;
1418         cmd.abort.cid = nvme_cid(req);
1419         cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1420
1421         dev_warn(nvmeq->dev->ctrl.device,
1422                 "I/O %d (%s) QID %d timeout, aborting\n",
1423                  req->tag,
1424                  nvme_get_opcode_str(nvme_req(req)->cmd->common.opcode),
1425                  nvmeq->qid);
1426
1427         abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd),
1428                                          BLK_MQ_REQ_NOWAIT);
1429         if (IS_ERR(abort_req)) {
1430                 atomic_inc(&dev->ctrl.abort_limit);
1431                 return BLK_EH_RESET_TIMER;
1432         }
1433         nvme_init_request(abort_req, &cmd);
1434
1435         abort_req->end_io = abort_endio;
1436         abort_req->end_io_data = NULL;
1437         abort_req->rq_flags |= RQF_QUIET;
1438         blk_execute_rq_nowait(abort_req, false);
1439
1440         /*
1441          * The aborted req will be completed on receiving the abort req.
1442          * We enable the timer again. If hit twice, it'll cause a device reset,
1443          * as the device then is in a faulty state.
1444          */
1445         return BLK_EH_RESET_TIMER;
1446 }
1447
1448 static void nvme_free_queue(struct nvme_queue *nvmeq)
1449 {
1450         dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
1451                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1452         if (!nvmeq->sq_cmds)
1453                 return;
1454
1455         if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) {
1456                 pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
1457                                 nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1458         } else {
1459                 dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
1460                                 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1461         }
1462 }
1463
1464 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1465 {
1466         int i;
1467
1468         for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1469                 dev->ctrl.queue_count--;
1470                 nvme_free_queue(&dev->queues[i]);
1471         }
1472 }
1473
1474 /**
1475  * nvme_suspend_queue - put queue into suspended state
1476  * @nvmeq: queue to suspend
1477  */
1478 static int nvme_suspend_queue(struct nvme_queue *nvmeq)
1479 {
1480         if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
1481                 return 1;
1482
1483         /* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
1484         mb();
1485
1486         nvmeq->dev->online_queues--;
1487         if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1488                 nvme_stop_admin_queue(&nvmeq->dev->ctrl);
1489         if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
1490                 pci_free_irq(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector, nvmeq);
1491         return 0;
1492 }
1493
1494 static void nvme_suspend_io_queues(struct nvme_dev *dev)
1495 {
1496         int i;
1497
1498         for (i = dev->ctrl.queue_count - 1; i > 0; i--)
1499                 nvme_suspend_queue(&dev->queues[i]);
1500 }
1501
1502 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
1503 {
1504         struct nvme_queue *nvmeq = &dev->queues[0];
1505
1506         if (shutdown)
1507                 nvme_shutdown_ctrl(&dev->ctrl);
1508         else
1509                 nvme_disable_ctrl(&dev->ctrl);
1510
1511         nvme_poll_irqdisable(nvmeq);
1512 }
1513
1514 /*
1515  * Called only on a device that has been disabled and after all other threads
1516  * that can check this device's completion queues have synced, except
1517  * nvme_poll(). This is the last chance for the driver to see a natural
1518  * completion before nvme_cancel_request() terminates all incomplete requests.
1519  */
1520 static void nvme_reap_pending_cqes(struct nvme_dev *dev)
1521 {
1522         int i;
1523
1524         for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
1525                 spin_lock(&dev->queues[i].cq_poll_lock);
1526                 nvme_poll_cq(&dev->queues[i], NULL);
1527                 spin_unlock(&dev->queues[i].cq_poll_lock);
1528         }
1529 }
1530
1531 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1532                                 int entry_size)
1533 {
1534         int q_depth = dev->q_depth;
1535         unsigned q_size_aligned = roundup(q_depth * entry_size,
1536                                           NVME_CTRL_PAGE_SIZE);
1537
1538         if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1539                 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1540
1541                 mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE);
1542                 q_depth = div_u64(mem_per_q, entry_size);
1543
1544                 /*
1545                  * Ensure the reduced q_depth is above some threshold where it
1546                  * would be better to map queues in system memory with the
1547                  * original depth
1548                  */
1549                 if (q_depth < 64)
1550                         return -ENOMEM;
1551         }
1552
1553         return q_depth;
1554 }
1555
1556 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1557                                 int qid)
1558 {
1559         struct pci_dev *pdev = to_pci_dev(dev->dev);
1560
1561         if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1562                 nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq));
1563                 if (nvmeq->sq_cmds) {
1564                         nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev,
1565                                                         nvmeq->sq_cmds);
1566                         if (nvmeq->sq_dma_addr) {
1567                                 set_bit(NVMEQ_SQ_CMB, &nvmeq->flags);
1568                                 return 0;
1569                         }
1570
1571                         pci_free_p2pmem(pdev, nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1572                 }
1573         }
1574
1575         nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq),
1576                                 &nvmeq->sq_dma_addr, GFP_KERNEL);
1577         if (!nvmeq->sq_cmds)
1578                 return -ENOMEM;
1579         return 0;
1580 }
1581
1582 static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
1583 {
1584         struct nvme_queue *nvmeq = &dev->queues[qid];
1585
1586         if (dev->ctrl.queue_count > qid)
1587                 return 0;
1588
1589         nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES;
1590         nvmeq->q_depth = depth;
1591         nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq),
1592                                          &nvmeq->cq_dma_addr, GFP_KERNEL);
1593         if (!nvmeq->cqes)
1594                 goto free_nvmeq;
1595
1596         if (nvme_alloc_sq_cmds(dev, nvmeq, qid))
1597                 goto free_cqdma;
1598
1599         nvmeq->dev = dev;
1600         spin_lock_init(&nvmeq->sq_lock);
1601         spin_lock_init(&nvmeq->cq_poll_lock);
1602         nvmeq->cq_head = 0;
1603         nvmeq->cq_phase = 1;
1604         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1605         nvmeq->qid = qid;
1606         dev->ctrl.queue_count++;
1607
1608         return 0;
1609
1610  free_cqdma:
1611         dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes,
1612                           nvmeq->cq_dma_addr);
1613  free_nvmeq:
1614         return -ENOMEM;
1615 }
1616
1617 static int queue_request_irq(struct nvme_queue *nvmeq)
1618 {
1619         struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1620         int nr = nvmeq->dev->ctrl.instance;
1621
1622         if (use_threaded_interrupts) {
1623                 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
1624                                 nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
1625         } else {
1626                 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
1627                                 NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
1628         }
1629 }
1630
1631 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1632 {
1633         struct nvme_dev *dev = nvmeq->dev;
1634
1635         nvmeq->sq_tail = 0;
1636         nvmeq->last_sq_tail = 0;
1637         nvmeq->cq_head = 0;
1638         nvmeq->cq_phase = 1;
1639         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1640         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
1641         nvme_dbbuf_init(dev, nvmeq, qid);
1642         dev->online_queues++;
1643         wmb(); /* ensure the first interrupt sees the initialization */
1644 }
1645
1646 /*
1647  * Try getting shutdown_lock while setting up IO queues.
1648  */
1649 static int nvme_setup_io_queues_trylock(struct nvme_dev *dev)
1650 {
1651         /*
1652          * Give up if the lock is being held by nvme_dev_disable.
1653          */
1654         if (!mutex_trylock(&dev->shutdown_lock))
1655                 return -ENODEV;
1656
1657         /*
1658          * Controller is in wrong state, fail early.
1659          */
1660         if (dev->ctrl.state != NVME_CTRL_CONNECTING) {
1661                 mutex_unlock(&dev->shutdown_lock);
1662                 return -ENODEV;
1663         }
1664
1665         return 0;
1666 }
1667
1668 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
1669 {
1670         struct nvme_dev *dev = nvmeq->dev;
1671         int result;
1672         u16 vector = 0;
1673
1674         clear_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
1675
1676         /*
1677          * A queue's vector matches the queue identifier unless the controller
1678          * has only one vector available.
1679          */
1680         if (!polled)
1681                 vector = dev->num_vecs == 1 ? 0 : qid;
1682         else
1683                 set_bit(NVMEQ_POLLED, &nvmeq->flags);
1684
1685         result = adapter_alloc_cq(dev, qid, nvmeq, vector);
1686         if (result)
1687                 return result;
1688
1689         result = adapter_alloc_sq(dev, qid, nvmeq);
1690         if (result < 0)
1691                 return result;
1692         if (result)
1693                 goto release_cq;
1694
1695         nvmeq->cq_vector = vector;
1696
1697         result = nvme_setup_io_queues_trylock(dev);
1698         if (result)
1699                 return result;
1700         nvme_init_queue(nvmeq, qid);
1701         if (!polled) {
1702                 result = queue_request_irq(nvmeq);
1703                 if (result < 0)
1704                         goto release_sq;
1705         }
1706
1707         set_bit(NVMEQ_ENABLED, &nvmeq->flags);
1708         mutex_unlock(&dev->shutdown_lock);
1709         return result;
1710
1711 release_sq:
1712         dev->online_queues--;
1713         mutex_unlock(&dev->shutdown_lock);
1714         adapter_delete_sq(dev, qid);
1715 release_cq:
1716         adapter_delete_cq(dev, qid);
1717         return result;
1718 }
1719
1720 static const struct blk_mq_ops nvme_mq_admin_ops = {
1721         .queue_rq       = nvme_queue_rq,
1722         .complete       = nvme_pci_complete_rq,
1723         .init_hctx      = nvme_admin_init_hctx,
1724         .init_request   = nvme_pci_init_request,
1725         .timeout        = nvme_timeout,
1726 };
1727
1728 static const struct blk_mq_ops nvme_mq_ops = {
1729         .queue_rq       = nvme_queue_rq,
1730         .queue_rqs      = nvme_queue_rqs,
1731         .complete       = nvme_pci_complete_rq,
1732         .commit_rqs     = nvme_commit_rqs,
1733         .init_hctx      = nvme_init_hctx,
1734         .init_request   = nvme_pci_init_request,
1735         .map_queues     = nvme_pci_map_queues,
1736         .timeout        = nvme_timeout,
1737         .poll           = nvme_poll,
1738 };
1739
1740 static void nvme_dev_remove_admin(struct nvme_dev *dev)
1741 {
1742         if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1743                 /*
1744                  * If the controller was reset during removal, it's possible
1745                  * user requests may be waiting on a stopped queue. Start the
1746                  * queue to flush these to completion.
1747                  */
1748                 nvme_start_admin_queue(&dev->ctrl);
1749                 blk_mq_destroy_queue(dev->ctrl.admin_q);
1750                 blk_mq_free_tag_set(&dev->admin_tagset);
1751         }
1752 }
1753
1754 static int nvme_pci_alloc_admin_tag_set(struct nvme_dev *dev)
1755 {
1756         struct blk_mq_tag_set *set = &dev->admin_tagset;
1757
1758         set->ops = &nvme_mq_admin_ops;
1759         set->nr_hw_queues = 1;
1760
1761         set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1762         set->timeout = NVME_ADMIN_TIMEOUT;
1763         set->numa_node = dev->ctrl.numa_node;
1764         set->cmd_size = sizeof(struct nvme_iod);
1765         set->flags = BLK_MQ_F_NO_SCHED;
1766         set->driver_data = dev;
1767
1768         if (blk_mq_alloc_tag_set(set))
1769                 return -ENOMEM;
1770         dev->ctrl.admin_tagset = set;
1771
1772         dev->ctrl.admin_q = blk_mq_init_queue(set);
1773         if (IS_ERR(dev->ctrl.admin_q)) {
1774                 blk_mq_free_tag_set(set);
1775                 dev->ctrl.admin_q = NULL;
1776                 return -ENOMEM;
1777         }
1778         if (!blk_get_queue(dev->ctrl.admin_q)) {
1779                 nvme_dev_remove_admin(dev);
1780                 dev->ctrl.admin_q = NULL;
1781                 return -ENODEV;
1782         }
1783         return 0;
1784 }
1785
1786 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1787 {
1788         return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
1789 }
1790
1791 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
1792 {
1793         struct pci_dev *pdev = to_pci_dev(dev->dev);
1794
1795         if (size <= dev->bar_mapped_size)
1796                 return 0;
1797         if (size > pci_resource_len(pdev, 0))
1798                 return -ENOMEM;
1799         if (dev->bar)
1800                 iounmap(dev->bar);
1801         dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1802         if (!dev->bar) {
1803                 dev->bar_mapped_size = 0;
1804                 return -ENOMEM;
1805         }
1806         dev->bar_mapped_size = size;
1807         dev->dbs = dev->bar + NVME_REG_DBS;
1808
1809         return 0;
1810 }
1811
1812 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
1813 {
1814         int result;
1815         u32 aqa;
1816         struct nvme_queue *nvmeq;
1817
1818         result = nvme_remap_bar(dev, db_bar_size(dev, 0));
1819         if (result < 0)
1820                 return result;
1821
1822         dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
1823                                 NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
1824
1825         if (dev->subsystem &&
1826             (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
1827                 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1828
1829         result = nvme_disable_ctrl(&dev->ctrl);
1830         if (result < 0)
1831                 return result;
1832
1833         result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1834         if (result)
1835                 return result;
1836
1837         dev->ctrl.numa_node = dev_to_node(dev->dev);
1838
1839         nvmeq = &dev->queues[0];
1840         aqa = nvmeq->q_depth - 1;
1841         aqa |= aqa << 16;
1842
1843         writel(aqa, dev->bar + NVME_REG_AQA);
1844         lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
1845         lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
1846
1847         result = nvme_enable_ctrl(&dev->ctrl);
1848         if (result)
1849                 return result;
1850
1851         nvmeq->cq_vector = 0;
1852         nvme_init_queue(nvmeq, 0);
1853         result = queue_request_irq(nvmeq);
1854         if (result) {
1855                 dev->online_queues--;
1856                 return result;
1857         }
1858
1859         set_bit(NVMEQ_ENABLED, &nvmeq->flags);
1860         return result;
1861 }
1862
1863 static int nvme_create_io_queues(struct nvme_dev *dev)
1864 {
1865         unsigned i, max, rw_queues;
1866         int ret = 0;
1867
1868         for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1869                 if (nvme_alloc_queue(dev, i, dev->q_depth)) {
1870                         ret = -ENOMEM;
1871                         break;
1872                 }
1873         }
1874
1875         max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1876         if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
1877                 rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
1878                                 dev->io_queues[HCTX_TYPE_READ];
1879         } else {
1880                 rw_queues = max;
1881         }
1882
1883         for (i = dev->online_queues; i <= max; i++) {
1884                 bool polled = i > rw_queues;
1885
1886                 ret = nvme_create_queue(&dev->queues[i], i, polled);
1887                 if (ret)
1888                         break;
1889         }
1890
1891         /*
1892          * Ignore failing Create SQ/CQ commands, we can continue with less
1893          * than the desired amount of queues, and even a controller without
1894          * I/O queues can still be used to issue admin commands.  This might
1895          * be useful to upgrade a buggy firmware for example.
1896          */
1897         return ret >= 0 ? 0 : ret;
1898 }
1899
1900 static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
1901 {
1902         u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
1903
1904         return 1ULL << (12 + 4 * szu);
1905 }
1906
1907 static u32 nvme_cmb_size(struct nvme_dev *dev)
1908 {
1909         return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
1910 }
1911
1912 static void nvme_map_cmb(struct nvme_dev *dev)
1913 {
1914         u64 size, offset;
1915         resource_size_t bar_size;
1916         struct pci_dev *pdev = to_pci_dev(dev->dev);
1917         int bar;
1918
1919         if (dev->cmb_size)
1920                 return;
1921
1922         if (NVME_CAP_CMBS(dev->ctrl.cap))
1923                 writel(NVME_CMBMSC_CRE, dev->bar + NVME_REG_CMBMSC);
1924
1925         dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1926         if (!dev->cmbsz)
1927                 return;
1928         dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1929
1930         size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
1931         offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
1932         bar = NVME_CMB_BIR(dev->cmbloc);
1933         bar_size = pci_resource_len(pdev, bar);
1934
1935         if (offset > bar_size)
1936                 return;
1937
1938         /*
1939          * Tell the controller about the host side address mapping the CMB,
1940          * and enable CMB decoding for the NVMe 1.4+ scheme:
1941          */
1942         if (NVME_CAP_CMBS(dev->ctrl.cap)) {
1943                 hi_lo_writeq(NVME_CMBMSC_CRE | NVME_CMBMSC_CMSE |
1944                              (pci_bus_address(pdev, bar) + offset),
1945                              dev->bar + NVME_REG_CMBMSC);
1946         }
1947
1948         /*
1949          * Controllers may support a CMB size larger than their BAR,
1950          * for example, due to being behind a bridge. Reduce the CMB to
1951          * the reported size of the BAR
1952          */
1953         if (size > bar_size - offset)
1954                 size = bar_size - offset;
1955
1956         if (pci_p2pdma_add_resource(pdev, bar, size, offset)) {
1957                 dev_warn(dev->ctrl.device,
1958                          "failed to register the CMB\n");
1959                 return;
1960         }
1961
1962         dev->cmb_size = size;
1963         dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS);
1964
1965         if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) ==
1966                         (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS))
1967                 pci_p2pmem_publish(pdev, true);
1968 }
1969
1970 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
1971 {
1972         u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;
1973         u64 dma_addr = dev->host_mem_descs_dma;
1974         struct nvme_command c = { };
1975         int ret;
1976
1977         c.features.opcode       = nvme_admin_set_features;
1978         c.features.fid          = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
1979         c.features.dword11      = cpu_to_le32(bits);
1980         c.features.dword12      = cpu_to_le32(host_mem_size);
1981         c.features.dword13      = cpu_to_le32(lower_32_bits(dma_addr));
1982         c.features.dword14      = cpu_to_le32(upper_32_bits(dma_addr));
1983         c.features.dword15      = cpu_to_le32(dev->nr_host_mem_descs);
1984
1985         ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1986         if (ret) {
1987                 dev_warn(dev->ctrl.device,
1988                          "failed to set host mem (err %d, flags %#x).\n",
1989                          ret, bits);
1990         } else
1991                 dev->hmb = bits & NVME_HOST_MEM_ENABLE;
1992
1993         return ret;
1994 }
1995
1996 static void nvme_free_host_mem(struct nvme_dev *dev)
1997 {
1998         int i;
1999
2000         for (i = 0; i < dev->nr_host_mem_descs; i++) {
2001                 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
2002                 size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE;
2003
2004                 dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],
2005                                le64_to_cpu(desc->addr),
2006                                DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2007         }
2008
2009         kfree(dev->host_mem_desc_bufs);
2010         dev->host_mem_desc_bufs = NULL;
2011         dma_free_coherent(dev->dev,
2012                         dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
2013                         dev->host_mem_descs, dev->host_mem_descs_dma);
2014         dev->host_mem_descs = NULL;
2015         dev->nr_host_mem_descs = 0;
2016 }
2017
2018 static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
2019                 u32 chunk_size)
2020 {
2021         struct nvme_host_mem_buf_desc *descs;
2022         u32 max_entries, len;
2023         dma_addr_t descs_dma;
2024         int i = 0;
2025         void **bufs;
2026         u64 size, tmp;
2027
2028         tmp = (preferred + chunk_size - 1);
2029         do_div(tmp, chunk_size);
2030         max_entries = tmp;
2031
2032         if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
2033                 max_entries = dev->ctrl.hmmaxd;
2034
2035         descs = dma_alloc_coherent(dev->dev, max_entries * sizeof(*descs),
2036                                    &descs_dma, GFP_KERNEL);
2037         if (!descs)
2038                 goto out;
2039
2040         bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
2041         if (!bufs)
2042                 goto out_free_descs;
2043
2044         for (size = 0; size < preferred && i < max_entries; size += len) {
2045                 dma_addr_t dma_addr;
2046
2047                 len = min_t(u64, chunk_size, preferred - size);
2048                 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
2049                                 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2050                 if (!bufs[i])
2051                         break;
2052
2053                 descs[i].addr = cpu_to_le64(dma_addr);
2054                 descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE);
2055                 i++;
2056         }
2057
2058         if (!size)
2059                 goto out_free_bufs;
2060
2061         dev->nr_host_mem_descs = i;
2062         dev->host_mem_size = size;
2063         dev->host_mem_descs = descs;
2064         dev->host_mem_descs_dma = descs_dma;
2065         dev->host_mem_desc_bufs = bufs;
2066         return 0;
2067
2068 out_free_bufs:
2069         while (--i >= 0) {
2070                 size_t size = le32_to_cpu(descs[i].size) * NVME_CTRL_PAGE_SIZE;
2071
2072                 dma_free_attrs(dev->dev, size, bufs[i],
2073                                le64_to_cpu(descs[i].addr),
2074                                DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2075         }
2076
2077         kfree(bufs);
2078 out_free_descs:
2079         dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
2080                         descs_dma);
2081 out:
2082         dev->host_mem_descs = NULL;
2083         return -ENOMEM;
2084 }
2085
2086 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
2087 {
2088         u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
2089         u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
2090         u64 chunk_size;
2091
2092         /* start big and work our way down */
2093         for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) {
2094                 if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
2095                         if (!min || dev->host_mem_size >= min)
2096                                 return 0;
2097                         nvme_free_host_mem(dev);
2098                 }
2099         }
2100
2101         return -ENOMEM;
2102 }
2103
2104 static int nvme_setup_host_mem(struct nvme_dev *dev)
2105 {
2106         u64 max = (u64)max_host_mem_size_mb * SZ_1M;
2107         u64 preferred = (u64)dev->ctrl.hmpre * 4096;
2108         u64 min = (u64)dev->ctrl.hmmin * 4096;
2109         u32 enable_bits = NVME_HOST_MEM_ENABLE;
2110         int ret;
2111
2112         preferred = min(preferred, max);
2113         if (min > max) {
2114                 dev_warn(dev->ctrl.device,
2115                         "min host memory (%lld MiB) above limit (%d MiB).\n",
2116                         min >> ilog2(SZ_1M), max_host_mem_size_mb);
2117                 nvme_free_host_mem(dev);
2118                 return 0;
2119         }
2120
2121         /*
2122          * If we already have a buffer allocated check if we can reuse it.
2123          */
2124         if (dev->host_mem_descs) {
2125                 if (dev->host_mem_size >= min)
2126                         enable_bits |= NVME_HOST_MEM_RETURN;
2127                 else
2128                         nvme_free_host_mem(dev);
2129         }
2130
2131         if (!dev->host_mem_descs) {
2132                 if (nvme_alloc_host_mem(dev, min, preferred)) {
2133                         dev_warn(dev->ctrl.device,
2134                                 "failed to allocate host memory buffer.\n");
2135                         return 0; /* controller must work without HMB */
2136                 }
2137
2138                 dev_info(dev->ctrl.device,
2139                         "allocated %lld MiB host memory buffer.\n",
2140                         dev->host_mem_size >> ilog2(SZ_1M));
2141         }
2142
2143         ret = nvme_set_host_mem(dev, enable_bits);
2144         if (ret)
2145                 nvme_free_host_mem(dev);
2146         return ret;
2147 }
2148
2149 static ssize_t cmb_show(struct device *dev, struct device_attribute *attr,
2150                 char *buf)
2151 {
2152         struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2153
2154         return sysfs_emit(buf, "cmbloc : x%08x\ncmbsz  : x%08x\n",
2155                        ndev->cmbloc, ndev->cmbsz);
2156 }
2157 static DEVICE_ATTR_RO(cmb);
2158
2159 static ssize_t cmbloc_show(struct device *dev, struct device_attribute *attr,
2160                 char *buf)
2161 {
2162         struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2163
2164         return sysfs_emit(buf, "%u\n", ndev->cmbloc);
2165 }
2166 static DEVICE_ATTR_RO(cmbloc);
2167
2168 static ssize_t cmbsz_show(struct device *dev, struct device_attribute *attr,
2169                 char *buf)
2170 {
2171         struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2172
2173         return sysfs_emit(buf, "%u\n", ndev->cmbsz);
2174 }
2175 static DEVICE_ATTR_RO(cmbsz);
2176
2177 static ssize_t hmb_show(struct device *dev, struct device_attribute *attr,
2178                         char *buf)
2179 {
2180         struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2181
2182         return sysfs_emit(buf, "%d\n", ndev->hmb);
2183 }
2184
2185 static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
2186                          const char *buf, size_t count)
2187 {
2188         struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2189         bool new;
2190         int ret;
2191
2192         if (strtobool(buf, &new) < 0)
2193                 return -EINVAL;
2194
2195         if (new == ndev->hmb)
2196                 return count;
2197
2198         if (new) {
2199                 ret = nvme_setup_host_mem(ndev);
2200         } else {
2201                 ret = nvme_set_host_mem(ndev, 0);
2202                 if (!ret)
2203                         nvme_free_host_mem(ndev);
2204         }
2205
2206         if (ret < 0)
2207                 return ret;
2208
2209         return count;
2210 }
2211 static DEVICE_ATTR_RW(hmb);
2212
2213 static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
2214                 struct attribute *a, int n)
2215 {
2216         struct nvme_ctrl *ctrl =
2217                 dev_get_drvdata(container_of(kobj, struct device, kobj));
2218         struct nvme_dev *dev = to_nvme_dev(ctrl);
2219
2220         if (a == &dev_attr_cmb.attr ||
2221             a == &dev_attr_cmbloc.attr ||
2222             a == &dev_attr_cmbsz.attr) {
2223                 if (!dev->cmbsz)
2224                         return 0;
2225         }
2226         if (a == &dev_attr_hmb.attr && !ctrl->hmpre)
2227                 return 0;
2228
2229         return a->mode;
2230 }
2231
2232 static struct attribute *nvme_pci_attrs[] = {
2233         &dev_attr_cmb.attr,
2234         &dev_attr_cmbloc.attr,
2235         &dev_attr_cmbsz.attr,
2236         &dev_attr_hmb.attr,
2237         NULL,
2238 };
2239
2240 static const struct attribute_group nvme_pci_attr_group = {
2241         .attrs          = nvme_pci_attrs,
2242         .is_visible     = nvme_pci_attrs_are_visible,
2243 };
2244
2245 /*
2246  * nirqs is the number of interrupts available for write and read
2247  * queues. The core already reserved an interrupt for the admin queue.
2248  */
2249 static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs)
2250 {
2251         struct nvme_dev *dev = affd->priv;
2252         unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues;
2253
2254         /*
2255          * If there is no interrupt available for queues, ensure that
2256          * the default queue is set to 1. The affinity set size is
2257          * also set to one, but the irq core ignores it for this case.
2258          *
2259          * If only one interrupt is available or 'write_queue' == 0, combine
2260          * write and read queues.
2261          *
2262          * If 'write_queues' > 0, ensure it leaves room for at least one read
2263          * queue.
2264          */
2265         if (!nrirqs) {
2266                 nrirqs = 1;
2267                 nr_read_queues = 0;
2268         } else if (nrirqs == 1 || !nr_write_queues) {
2269                 nr_read_queues = 0;
2270         } else if (nr_write_queues >= nrirqs) {
2271                 nr_read_queues = 1;
2272         } else {
2273                 nr_read_queues = nrirqs - nr_write_queues;
2274         }
2275
2276         dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2277         affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2278         dev->io_queues[HCTX_TYPE_READ] = nr_read_queues;
2279         affd->set_size[HCTX_TYPE_READ] = nr_read_queues;
2280         affd->nr_sets = nr_read_queues ? 2 : 1;
2281 }
2282
2283 static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
2284 {
2285         struct pci_dev *pdev = to_pci_dev(dev->dev);
2286         struct irq_affinity affd = {
2287                 .pre_vectors    = 1,
2288                 .calc_sets      = nvme_calc_irq_sets,
2289                 .priv           = dev,
2290         };
2291         unsigned int irq_queues, poll_queues;
2292
2293         /*
2294          * Poll queues don't need interrupts, but we need at least one I/O queue
2295          * left over for non-polled I/O.
2296          */
2297         poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
2298         dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
2299
2300         /*
2301          * Initialize for the single interrupt case, will be updated in
2302          * nvme_calc_irq_sets().
2303          */
2304         dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
2305         dev->io_queues[HCTX_TYPE_READ] = 0;
2306
2307         /*
2308          * We need interrupts for the admin queue and each non-polled I/O queue,
2309          * but some Apple controllers require all queues to use the first
2310          * vector.
2311          */
2312         irq_queues = 1;
2313         if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR))
2314                 irq_queues += (nr_io_queues - poll_queues);
2315         return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues,
2316                               PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd);
2317 }
2318
2319 static void nvme_disable_io_queues(struct nvme_dev *dev)
2320 {
2321         if (__nvme_disable_io_queues(dev, nvme_admin_delete_sq))
2322                 __nvme_disable_io_queues(dev, nvme_admin_delete_cq);
2323 }
2324
2325 static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
2326 {
2327         /*
2328          * If tags are shared with admin queue (Apple bug), then
2329          * make sure we only use one IO queue.
2330          */
2331         if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2332                 return 1;
2333         return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues;
2334 }
2335
2336 static int nvme_setup_io_queues(struct nvme_dev *dev)
2337 {
2338         struct nvme_queue *adminq = &dev->queues[0];
2339         struct pci_dev *pdev = to_pci_dev(dev->dev);
2340         unsigned int nr_io_queues;
2341         unsigned long size;
2342         int result;
2343
2344         /*
2345          * Sample the module parameters once at reset time so that we have
2346          * stable values to work with.
2347          */
2348         dev->nr_write_queues = write_queues;
2349         dev->nr_poll_queues = poll_queues;
2350
2351         nr_io_queues = dev->nr_allocated_queues - 1;
2352         result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
2353         if (result < 0)
2354                 return result;
2355
2356         if (nr_io_queues == 0)
2357                 return 0;
2358
2359         /*
2360          * Free IRQ resources as soon as NVMEQ_ENABLED bit transitions
2361          * from set to unset. If there is a window to it is truely freed,
2362          * pci_free_irq_vectors() jumping into this window will crash.
2363          * And take lock to avoid racing with pci_free_irq_vectors() in
2364          * nvme_dev_disable() path.
2365          */
2366         result = nvme_setup_io_queues_trylock(dev);
2367         if (result)
2368                 return result;
2369         if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
2370                 pci_free_irq(pdev, 0, adminq);
2371
2372         if (dev->cmb_use_sqes) {
2373                 result = nvme_cmb_qdepth(dev, nr_io_queues,
2374                                 sizeof(struct nvme_command));
2375                 if (result > 0)
2376                         dev->q_depth = result;
2377                 else
2378                         dev->cmb_use_sqes = false;
2379         }
2380
2381         do {
2382                 size = db_bar_size(dev, nr_io_queues);
2383                 result = nvme_remap_bar(dev, size);
2384                 if (!result)
2385                         break;
2386                 if (!--nr_io_queues) {
2387                         result = -ENOMEM;
2388                         goto out_unlock;
2389                 }
2390         } while (1);
2391         adminq->q_db = dev->dbs;
2392
2393  retry:
2394         /* Deregister the admin queue's interrupt */
2395         if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
2396                 pci_free_irq(pdev, 0, adminq);
2397
2398         /*
2399          * If we enable msix early due to not intx, disable it again before
2400          * setting up the full range we need.
2401          */
2402         pci_free_irq_vectors(pdev);
2403
2404         result = nvme_setup_irqs(dev, nr_io_queues);
2405         if (result <= 0) {
2406                 result = -EIO;
2407                 goto out_unlock;
2408         }
2409
2410         dev->num_vecs = result;
2411         result = max(result - 1, 1);
2412         dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
2413
2414         /*
2415          * Should investigate if there's a performance win from allocating
2416          * more queues than interrupt vectors; it might allow the submission
2417          * path to scale better, even if the receive path is limited by the
2418          * number of interrupts.
2419          */
2420         result = queue_request_irq(adminq);
2421         if (result)
2422                 goto out_unlock;
2423         set_bit(NVMEQ_ENABLED, &adminq->flags);
2424         mutex_unlock(&dev->shutdown_lock);
2425
2426         result = nvme_create_io_queues(dev);
2427         if (result || dev->online_queues < 2)
2428                 return result;
2429
2430         if (dev->online_queues - 1 < dev->max_qid) {
2431                 nr_io_queues = dev->online_queues - 1;
2432                 nvme_disable_io_queues(dev);
2433                 result = nvme_setup_io_queues_trylock(dev);
2434                 if (result)
2435                         return result;
2436                 nvme_suspend_io_queues(dev);
2437                 goto retry;
2438         }
2439         dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
2440                                         dev->io_queues[HCTX_TYPE_DEFAULT],
2441                                         dev->io_queues[HCTX_TYPE_READ],
2442                                         dev->io_queues[HCTX_TYPE_POLL]);
2443         return 0;
2444 out_unlock:
2445         mutex_unlock(&dev->shutdown_lock);
2446         return result;
2447 }
2448
2449 static void nvme_del_queue_end(struct request *req, blk_status_t error)
2450 {
2451         struct nvme_queue *nvmeq = req->end_io_data;
2452
2453         blk_mq_free_request(req);
2454         complete(&nvmeq->delete_done);
2455 }
2456
2457 static void nvme_del_cq_end(struct request *req, blk_status_t error)
2458 {
2459         struct nvme_queue *nvmeq = req->end_io_data;
2460
2461         if (error)
2462                 set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
2463
2464         nvme_del_queue_end(req, error);
2465 }
2466
2467 static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
2468 {
2469         struct request_queue *q = nvmeq->dev->ctrl.admin_q;
2470         struct request *req;
2471         struct nvme_command cmd = { };
2472
2473         cmd.delete_queue.opcode = opcode;
2474         cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2475
2476         req = blk_mq_alloc_request(q, nvme_req_op(&cmd), BLK_MQ_REQ_NOWAIT);
2477         if (IS_ERR(req))
2478                 return PTR_ERR(req);
2479         nvme_init_request(req, &cmd);
2480
2481         if (opcode == nvme_admin_delete_cq)
2482                 req->end_io = nvme_del_cq_end;
2483         else
2484                 req->end_io = nvme_del_queue_end;
2485         req->end_io_data = nvmeq;
2486
2487         init_completion(&nvmeq->delete_done);
2488         req->rq_flags |= RQF_QUIET;
2489         blk_execute_rq_nowait(req, false);
2490         return 0;
2491 }
2492
2493 static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)
2494 {
2495         int nr_queues = dev->online_queues - 1, sent = 0;
2496         unsigned long timeout;
2497
2498  retry:
2499         timeout = NVME_ADMIN_TIMEOUT;
2500         while (nr_queues > 0) {
2501                 if (nvme_delete_queue(&dev->queues[nr_queues], opcode))
2502                         break;
2503                 nr_queues--;
2504                 sent++;
2505         }
2506         while (sent) {
2507                 struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent];
2508
2509                 timeout = wait_for_completion_io_timeout(&nvmeq->delete_done,
2510                                 timeout);
2511                 if (timeout == 0)
2512                         return false;
2513
2514                 sent--;
2515                 if (nr_queues)
2516                         goto retry;
2517         }
2518         return true;
2519 }
2520
2521 static void nvme_pci_alloc_tag_set(struct nvme_dev *dev)
2522 {
2523         struct blk_mq_tag_set * set = &dev->tagset;
2524         int ret;
2525
2526         set->ops = &nvme_mq_ops;
2527         set->nr_hw_queues = dev->online_queues - 1;
2528         set->nr_maps = 2; /* default + read */
2529         if (dev->io_queues[HCTX_TYPE_POLL])
2530                 set->nr_maps++;
2531         set->timeout = NVME_IO_TIMEOUT;
2532         set->numa_node = dev->ctrl.numa_node;
2533         set->queue_depth = min_t(unsigned, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
2534         set->cmd_size = sizeof(struct nvme_iod);
2535         set->flags = BLK_MQ_F_SHOULD_MERGE;
2536         set->driver_data = dev;
2537
2538         /*
2539          * Some Apple controllers requires tags to be unique
2540          * across admin and IO queue, so reserve the first 32
2541          * tags of the IO queue.
2542          */
2543         if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2544                 set->reserved_tags = NVME_AQ_DEPTH;
2545
2546         ret = blk_mq_alloc_tag_set(set);
2547         if (ret) {
2548                 dev_warn(dev->ctrl.device,
2549                         "IO queues tagset allocation failed %d\n", ret);
2550                 return;
2551         }
2552         dev->ctrl.tagset = set;
2553 }
2554
2555 static void nvme_pci_update_nr_queues(struct nvme_dev *dev)
2556 {
2557         blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
2558         /* free previously allocated queues that are no longer usable */
2559         nvme_free_queues(dev, dev->online_queues);
2560 }
2561
2562 static int nvme_pci_enable(struct nvme_dev *dev)
2563 {
2564         int result = -ENOMEM;
2565         struct pci_dev *pdev = to_pci_dev(dev->dev);
2566         int dma_address_bits = 64;
2567
2568         if (pci_enable_device_mem(pdev))
2569                 return result;
2570
2571         pci_set_master(pdev);
2572
2573         if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48)
2574                 dma_address_bits = 48;
2575         if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(dma_address_bits)))
2576                 goto disable;
2577
2578         if (readl(dev->bar + NVME_REG_CSTS) == -1) {
2579                 result = -ENODEV;
2580                 goto disable;
2581         }
2582
2583         /*
2584          * Some devices and/or platforms don't advertise or work with INTx
2585          * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
2586          * adjust this later.
2587          */
2588         result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
2589         if (result < 0)
2590                 return result;
2591
2592         dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
2593
2594         dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1,
2595                                 io_queue_depth);
2596         dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */
2597         dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
2598         dev->dbs = dev->bar + 4096;
2599
2600         /*
2601          * Some Apple controllers require a non-standard SQE size.
2602          * Interestingly they also seem to ignore the CC:IOSQES register
2603          * so we don't bother updating it here.
2604          */
2605         if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES)
2606                 dev->io_sqes = 7;
2607         else
2608                 dev->io_sqes = NVME_NVM_IOSQES;
2609
2610         /*
2611          * Temporary fix for the Apple controller found in the MacBook8,1 and
2612          * some MacBook7,1 to avoid controller resets and data loss.
2613          */
2614         if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
2615                 dev->q_depth = 2;
2616                 dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
2617                         "set queue depth=%u to work around controller resets\n",
2618                         dev->q_depth);
2619         } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
2620                    (pdev->device == 0xa821 || pdev->device == 0xa822) &&
2621                    NVME_CAP_MQES(dev->ctrl.cap) == 0) {
2622                 dev->q_depth = 64;
2623                 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
2624                         "set queue depth=%u\n", dev->q_depth);
2625         }
2626
2627         /*
2628          * Controllers with the shared tags quirk need the IO queue to be
2629          * big enough so that we get 32 tags for the admin queue
2630          */
2631         if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) &&
2632             (dev->q_depth < (NVME_AQ_DEPTH + 2))) {
2633                 dev->q_depth = NVME_AQ_DEPTH + 2;
2634                 dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n",
2635                          dev->q_depth);
2636         }
2637
2638
2639         nvme_map_cmb(dev);
2640
2641         pci_enable_pcie_error_reporting(pdev);
2642         pci_save_state(pdev);
2643         return 0;
2644
2645  disable:
2646         pci_disable_device(pdev);
2647         return result;
2648 }
2649
2650 static void nvme_dev_unmap(struct nvme_dev *dev)
2651 {
2652         if (dev->bar)
2653                 iounmap(dev->bar);
2654         pci_release_mem_regions(to_pci_dev(dev->dev));
2655 }
2656
2657 static void nvme_pci_disable(struct nvme_dev *dev)
2658 {
2659         struct pci_dev *pdev = to_pci_dev(dev->dev);
2660
2661         pci_free_irq_vectors(pdev);
2662
2663         if (pci_is_enabled(pdev)) {
2664                 pci_disable_pcie_error_reporting(pdev);
2665                 pci_disable_device(pdev);
2666         }
2667 }
2668
2669 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
2670 {
2671         bool dead = true, freeze = false;
2672         struct pci_dev *pdev = to_pci_dev(dev->dev);
2673
2674         mutex_lock(&dev->shutdown_lock);
2675         if (pci_is_enabled(pdev)) {
2676                 u32 csts;
2677
2678                 if (pci_device_is_present(pdev))
2679                         csts = readl(dev->bar + NVME_REG_CSTS);
2680                 else
2681                         csts = ~0;
2682
2683                 if (dev->ctrl.state == NVME_CTRL_LIVE ||
2684                     dev->ctrl.state == NVME_CTRL_RESETTING) {
2685                         freeze = true;
2686                         nvme_start_freeze(&dev->ctrl);
2687                 }
2688                 dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
2689                         pdev->error_state  != pci_channel_io_normal);
2690         }
2691
2692         /*
2693          * Give the controller a chance to complete all entered requests if
2694          * doing a safe shutdown.
2695          */
2696         if (!dead && shutdown && freeze)
2697                 nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
2698
2699         nvme_stop_queues(&dev->ctrl);
2700
2701         if (!dead && dev->ctrl.queue_count > 0) {
2702                 nvme_disable_io_queues(dev);
2703                 nvme_disable_admin_queue(dev, shutdown);
2704         }
2705         nvme_suspend_io_queues(dev);
2706         nvme_suspend_queue(&dev->queues[0]);
2707         nvme_pci_disable(dev);
2708         nvme_reap_pending_cqes(dev);
2709
2710         nvme_cancel_tagset(&dev->ctrl);
2711         nvme_cancel_admin_tagset(&dev->ctrl);
2712
2713         /*
2714          * The driver will not be starting up queues again if shutting down so
2715          * must flush all entered requests to their failed completion to avoid
2716          * deadlocking blk-mq hot-cpu notifier.
2717          */
2718         if (shutdown) {
2719                 nvme_start_queues(&dev->ctrl);
2720                 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q))
2721                         nvme_start_admin_queue(&dev->ctrl);
2722         }
2723         mutex_unlock(&dev->shutdown_lock);
2724 }
2725
2726 static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
2727 {
2728         if (!nvme_wait_reset(&dev->ctrl))
2729                 return -EBUSY;
2730         nvme_dev_disable(dev, shutdown);
2731         return 0;
2732 }
2733
2734 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2735 {
2736         dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
2737                                                 NVME_CTRL_PAGE_SIZE,
2738                                                 NVME_CTRL_PAGE_SIZE, 0);
2739         if (!dev->prp_page_pool)
2740                 return -ENOMEM;
2741
2742         /* Optimisation for I/Os between 4k and 128k */
2743         dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2744                                                 256, 256, 0);
2745         if (!dev->prp_small_pool) {
2746                 dma_pool_destroy(dev->prp_page_pool);
2747                 return -ENOMEM;
2748         }
2749         return 0;
2750 }
2751
2752 static void nvme_release_prp_pools(struct nvme_dev *dev)
2753 {
2754         dma_pool_destroy(dev->prp_page_pool);
2755         dma_pool_destroy(dev->prp_small_pool);
2756 }
2757
2758 static void nvme_free_tagset(struct nvme_dev *dev)
2759 {
2760         if (dev->tagset.tags)
2761                 blk_mq_free_tag_set(&dev->tagset);
2762         dev->ctrl.tagset = NULL;
2763 }
2764
2765 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2766 {
2767         struct nvme_dev *dev = to_nvme_dev(ctrl);
2768
2769         nvme_dbbuf_dma_free(dev);
2770         nvme_free_tagset(dev);
2771         if (dev->ctrl.admin_q)
2772                 blk_put_queue(dev->ctrl.admin_q);
2773         free_opal_dev(dev->ctrl.opal_dev);
2774         mempool_destroy(dev->iod_mempool);
2775         put_device(dev->dev);
2776         kfree(dev->queues);
2777         kfree(dev);
2778 }
2779
2780 static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
2781 {
2782         /*
2783          * Set state to deleting now to avoid blocking nvme_wait_reset(), which
2784          * may be holding this pci_dev's device lock.
2785          */
2786         nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
2787         nvme_get_ctrl(&dev->ctrl);
2788         nvme_dev_disable(dev, false);
2789         nvme_kill_queues(&dev->ctrl);
2790         if (!queue_work(nvme_wq, &dev->remove_work))
2791                 nvme_put_ctrl(&dev->ctrl);
2792 }
2793
2794 static void nvme_reset_work(struct work_struct *work)
2795 {
2796         struct nvme_dev *dev =
2797                 container_of(work, struct nvme_dev, ctrl.reset_work);
2798         bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2799         int result;
2800
2801         if (dev->ctrl.state != NVME_CTRL_RESETTING) {
2802                 dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n",
2803                          dev->ctrl.state);
2804                 result = -ENODEV;
2805                 goto out;
2806         }
2807
2808         /*
2809          * If we're called to reset a live controller first shut it down before
2810          * moving on.
2811          */
2812         if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2813                 nvme_dev_disable(dev, false);
2814         nvme_sync_queues(&dev->ctrl);
2815
2816         mutex_lock(&dev->shutdown_lock);
2817         result = nvme_pci_enable(dev);
2818         if (result)
2819                 goto out_unlock;
2820
2821         result = nvme_pci_configure_admin_queue(dev);
2822         if (result)
2823                 goto out_unlock;
2824
2825         if (!dev->ctrl.admin_q) {
2826                 result = nvme_pci_alloc_admin_tag_set(dev);
2827                 if (result)
2828                         goto out_unlock;
2829         } else {
2830                 nvme_start_admin_queue(&dev->ctrl);
2831         }
2832
2833         /*
2834          * Limit the max command size to prevent iod->sg allocations going
2835          * over a single page.
2836          */
2837         dev->ctrl.max_hw_sectors = min_t(u32,
2838                 NVME_MAX_KB_SZ << 1, dma_max_mapping_size(dev->dev) >> 9);
2839         dev->ctrl.max_segments = NVME_MAX_SEGS;
2840
2841         /*
2842          * Don't limit the IOMMU merged segment size.
2843          */
2844         dma_set_max_seg_size(dev->dev, 0xffffffff);
2845         dma_set_min_align_mask(dev->dev, NVME_CTRL_PAGE_SIZE - 1);
2846
2847         mutex_unlock(&dev->shutdown_lock);
2848
2849         /*
2850          * Introduce CONNECTING state from nvme-fc/rdma transports to mark the
2851          * initializing procedure here.
2852          */
2853         if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
2854                 dev_warn(dev->ctrl.device,
2855                         "failed to mark controller CONNECTING\n");
2856                 result = -EBUSY;
2857                 goto out;
2858         }
2859
2860         /*
2861          * We do not support an SGL for metadata (yet), so we are limited to a
2862          * single integrity segment for the separate metadata pointer.
2863          */
2864         dev->ctrl.max_integrity_segments = 1;
2865
2866         result = nvme_init_ctrl_finish(&dev->ctrl);
2867         if (result)
2868                 goto out;
2869
2870         if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
2871                 if (!dev->ctrl.opal_dev)
2872                         dev->ctrl.opal_dev =
2873                                 init_opal_dev(&dev->ctrl, &nvme_sec_submit);
2874                 else if (was_suspend)
2875                         opal_unlock_from_suspend(dev->ctrl.opal_dev);
2876         } else {
2877                 free_opal_dev(dev->ctrl.opal_dev);
2878                 dev->ctrl.opal_dev = NULL;
2879         }
2880
2881         if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
2882                 result = nvme_dbbuf_dma_alloc(dev);
2883                 if (result)
2884                         dev_warn(dev->dev,
2885                                  "unable to allocate dma for dbbuf\n");
2886         }
2887
2888         if (dev->ctrl.hmpre) {
2889                 result = nvme_setup_host_mem(dev);
2890                 if (result < 0)
2891                         goto out;
2892         }
2893
2894         result = nvme_setup_io_queues(dev);
2895         if (result)
2896                 goto out;
2897
2898         /*
2899          * Keep the controller around but remove all namespaces if we don't have
2900          * any working I/O queue.
2901          */
2902         if (dev->online_queues < 2) {
2903                 dev_warn(dev->ctrl.device, "IO queues not created\n");
2904                 nvme_kill_queues(&dev->ctrl);
2905                 nvme_remove_namespaces(&dev->ctrl);
2906                 nvme_free_tagset(dev);
2907         } else {
2908                 nvme_start_queues(&dev->ctrl);
2909                 nvme_wait_freeze(&dev->ctrl);
2910                 if (!dev->ctrl.tagset)
2911                         nvme_pci_alloc_tag_set(dev);
2912                 else
2913                         nvme_pci_update_nr_queues(dev);
2914                 nvme_dbbuf_set(dev);
2915                 nvme_unfreeze(&dev->ctrl);
2916         }
2917
2918         /*
2919          * If only admin queue live, keep it to do further investigation or
2920          * recovery.
2921          */
2922         if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
2923                 dev_warn(dev->ctrl.device,
2924                         "failed to mark controller live state\n");
2925                 result = -ENODEV;
2926                 goto out;
2927         }
2928
2929         if (!dev->attrs_added && !sysfs_create_group(&dev->ctrl.device->kobj,
2930                         &nvme_pci_attr_group))
2931                 dev->attrs_added = true;
2932
2933         nvme_start_ctrl(&dev->ctrl);
2934         return;
2935
2936  out_unlock:
2937         mutex_unlock(&dev->shutdown_lock);
2938  out:
2939         if (result)
2940                 dev_warn(dev->ctrl.device,
2941                          "Removing after probe failure status: %d\n", result);
2942         nvme_remove_dead_ctrl(dev);
2943 }
2944
2945 static void nvme_remove_dead_ctrl_work(struct work_struct *work)
2946 {
2947         struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2948         struct pci_dev *pdev = to_pci_dev(dev->dev);
2949
2950         if (pci_get_drvdata(pdev))
2951                 device_release_driver(&pdev->dev);
2952         nvme_put_ctrl(&dev->ctrl);
2953 }
2954
2955 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
2956 {
2957         *val = readl(to_nvme_dev(ctrl)->bar + off);
2958         return 0;
2959 }
2960
2961 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2962 {
2963         writel(val, to_nvme_dev(ctrl)->bar + off);
2964         return 0;
2965 }
2966
2967 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
2968 {
2969         *val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off);
2970         return 0;
2971 }
2972
2973 static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
2974 {
2975         struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
2976
2977         return snprintf(buf, size, "%s\n", dev_name(&pdev->dev));
2978 }
2979
2980 static void nvme_pci_print_device_info(struct nvme_ctrl *ctrl)
2981 {
2982         struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
2983         struct nvme_subsystem *subsys = ctrl->subsys;
2984
2985         dev_err(ctrl->device,
2986                 "VID:DID %04x:%04x model:%.*s firmware:%.*s\n",
2987                 pdev->vendor, pdev->device,
2988                 nvme_strlen(subsys->model, sizeof(subsys->model)),
2989                 subsys->model, nvme_strlen(subsys->firmware_rev,
2990                                            sizeof(subsys->firmware_rev)),
2991                 subsys->firmware_rev);
2992 }
2993
2994 static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
2995 {
2996         struct nvme_dev *dev = to_nvme_dev(ctrl);
2997
2998         return dma_pci_p2pdma_supported(dev->dev);
2999 }
3000
3001 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
3002         .name                   = "pcie",
3003         .module                 = THIS_MODULE,
3004         .flags                  = NVME_F_METADATA_SUPPORTED,
3005         .reg_read32             = nvme_pci_reg_read32,
3006         .reg_write32            = nvme_pci_reg_write32,
3007         .reg_read64             = nvme_pci_reg_read64,
3008         .free_ctrl              = nvme_pci_free_ctrl,
3009         .submit_async_event     = nvme_pci_submit_async_event,
3010         .get_address            = nvme_pci_get_address,
3011         .print_device_info      = nvme_pci_print_device_info,
3012         .supports_pci_p2pdma    = nvme_pci_supports_pci_p2pdma,
3013 };
3014
3015 static int nvme_dev_map(struct nvme_dev *dev)
3016 {
3017         struct pci_dev *pdev = to_pci_dev(dev->dev);
3018
3019         if (pci_request_mem_regions(pdev, "nvme"))
3020                 return -ENODEV;
3021
3022         if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
3023                 goto release;
3024
3025         return 0;
3026   release:
3027         pci_release_mem_regions(pdev);
3028         return -ENODEV;
3029 }
3030
3031 static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
3032 {
3033         if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
3034                 /*
3035                  * Several Samsung devices seem to drop off the PCIe bus
3036                  * randomly when APST is on and uses the deepest sleep state.
3037                  * This has been observed on a Samsung "SM951 NVMe SAMSUNG
3038                  * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
3039                  * 950 PRO 256GB", but it seems to be restricted to two Dell
3040                  * laptops.
3041                  */
3042                 if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
3043                     (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
3044                      dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
3045                         return NVME_QUIRK_NO_DEEPEST_PS;
3046         } else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
3047                 /*
3048                  * Samsung SSD 960 EVO drops off the PCIe bus after system
3049                  * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
3050                  * within few minutes after bootup on a Coffee Lake board -
3051                  * ASUS PRIME Z370-A
3052                  */
3053                 if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
3054                     (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
3055                      dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
3056                         return NVME_QUIRK_NO_APST;
3057         } else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 ||
3058                     pdev->device == 0xa808 || pdev->device == 0xa809)) ||
3059                    (pdev->vendor == 0x1e0f && pdev->device == 0x0001)) {
3060                 /*
3061                  * Forcing to use host managed nvme power settings for
3062                  * lowest idle power with quick resume latency on
3063                  * Samsung and Toshiba SSDs based on suspend behavior
3064                  * on Coffee Lake board for LENOVO C640
3065                  */
3066                 if ((dmi_match(DMI_BOARD_VENDOR, "LENOVO")) &&
3067                      dmi_match(DMI_BOARD_NAME, "LNVNB161216"))
3068                         return NVME_QUIRK_SIMPLE_SUSPEND;
3069         }
3070
3071         return 0;
3072 }
3073
3074 static void nvme_async_probe(void *data, async_cookie_t cookie)
3075 {
3076         struct nvme_dev *dev = data;
3077
3078         flush_work(&dev->ctrl.reset_work);
3079         flush_work(&dev->ctrl.scan_work);
3080         nvme_put_ctrl(&dev->ctrl);
3081 }
3082
3083 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3084 {
3085         int node, result = -ENOMEM;
3086         struct nvme_dev *dev;
3087         unsigned long quirks = id->driver_data;
3088         size_t alloc_size;
3089
3090         node = dev_to_node(&pdev->dev);
3091         if (node == NUMA_NO_NODE)
3092                 set_dev_node(&pdev->dev, first_memory_node);
3093
3094         dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
3095         if (!dev)
3096                 return -ENOMEM;
3097
3098         dev->nr_write_queues = write_queues;
3099         dev->nr_poll_queues = poll_queues;
3100         dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1;
3101         dev->queues = kcalloc_node(dev->nr_allocated_queues,
3102                         sizeof(struct nvme_queue), GFP_KERNEL, node);
3103         if (!dev->queues)
3104                 goto free;
3105
3106         dev->dev = get_device(&pdev->dev);
3107         pci_set_drvdata(pdev, dev);
3108
3109         result = nvme_dev_map(dev);
3110         if (result)
3111                 goto put_pci;
3112
3113         INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
3114         INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
3115         mutex_init(&dev->shutdown_lock);
3116
3117         result = nvme_setup_prp_pools(dev);
3118         if (result)
3119                 goto unmap;
3120
3121         quirks |= check_vendor_combination_bug(pdev);
3122
3123         if (!noacpi && acpi_storage_d3(&pdev->dev)) {
3124                 /*
3125                  * Some systems use a bios work around to ask for D3 on
3126                  * platforms that support kernel managed suspend.
3127                  */
3128                 dev_info(&pdev->dev,
3129                          "platform quirk: setting simple suspend\n");
3130                 quirks |= NVME_QUIRK_SIMPLE_SUSPEND;
3131         }
3132
3133         /*
3134          * Double check that our mempool alloc size will cover the biggest
3135          * command we support.
3136          */
3137         alloc_size = nvme_pci_iod_alloc_size();
3138         WARN_ON_ONCE(alloc_size > PAGE_SIZE);
3139
3140         dev->iod_mempool = mempool_create_node(1, mempool_kmalloc,
3141                                                 mempool_kfree,
3142                                                 (void *) alloc_size,
3143                                                 GFP_KERNEL, node);
3144         if (!dev->iod_mempool) {
3145                 result = -ENOMEM;
3146                 goto release_pools;
3147         }
3148
3149         result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
3150                         quirks);
3151         if (result)
3152                 goto release_mempool;
3153
3154         dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
3155
3156         nvme_reset_ctrl(&dev->ctrl);
3157         async_schedule(nvme_async_probe, dev);
3158
3159         return 0;
3160
3161  release_mempool:
3162         mempool_destroy(dev->iod_mempool);
3163  release_pools:
3164         nvme_release_prp_pools(dev);
3165  unmap:
3166         nvme_dev_unmap(dev);
3167  put_pci:
3168         put_device(dev->dev);
3169  free:
3170         kfree(dev->queues);
3171         kfree(dev);
3172         return result;
3173 }
3174
3175 static void nvme_reset_prepare(struct pci_dev *pdev)
3176 {
3177         struct nvme_dev *dev = pci_get_drvdata(pdev);
3178
3179         /*
3180          * We don't need to check the return value from waiting for the reset
3181          * state as pci_dev device lock is held, making it impossible to race
3182          * with ->remove().
3183          */
3184         nvme_disable_prepare_reset(dev, false);
3185         nvme_sync_queues(&dev->ctrl);
3186 }
3187
3188 static void nvme_reset_done(struct pci_dev *pdev)
3189 {
3190         struct nvme_dev *dev = pci_get_drvdata(pdev);
3191
3192         if (!nvme_try_sched_reset(&dev->ctrl))
3193                 flush_work(&dev->ctrl.reset_work);
3194 }
3195
3196 static void nvme_shutdown(struct pci_dev *pdev)
3197 {
3198         struct nvme_dev *dev = pci_get_drvdata(pdev);
3199
3200         nvme_disable_prepare_reset(dev, true);
3201 }
3202
3203 static void nvme_remove_attrs(struct nvme_dev *dev)
3204 {
3205         if (dev->attrs_added)
3206                 sysfs_remove_group(&dev->ctrl.device->kobj,
3207                                    &nvme_pci_attr_group);
3208 }
3209
3210 /*
3211  * The driver's remove may be called on a device in a partially initialized
3212  * state. This function must not have any dependencies on the device state in
3213  * order to proceed.
3214  */
3215 static void nvme_remove(struct pci_dev *pdev)
3216 {
3217         struct nvme_dev *dev = pci_get_drvdata(pdev);
3218
3219         nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3220         pci_set_drvdata(pdev, NULL);
3221
3222         if (!pci_device_is_present(pdev)) {
3223                 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
3224                 nvme_dev_disable(dev, true);
3225         }
3226
3227         flush_work(&dev->ctrl.reset_work);
3228         nvme_stop_ctrl(&dev->ctrl);
3229         nvme_remove_namespaces(&dev->ctrl);
3230         nvme_dev_disable(dev, true);
3231         nvme_remove_attrs(dev);
3232         nvme_free_host_mem(dev);
3233         nvme_dev_remove_admin(dev);
3234         nvme_free_queues(dev, 0);
3235         nvme_release_prp_pools(dev);
3236         nvme_dev_unmap(dev);
3237         nvme_uninit_ctrl(&dev->ctrl);
3238 }
3239
3240 #ifdef CONFIG_PM_SLEEP
3241 static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps)
3242 {
3243         return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps);
3244 }
3245
3246 static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps)
3247 {
3248         return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL);
3249 }
3250
3251 static int nvme_resume(struct device *dev)
3252 {
3253         struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3254         struct nvme_ctrl *ctrl = &ndev->ctrl;
3255
3256         if (ndev->last_ps == U32_MAX ||
3257             nvme_set_power_state(ctrl, ndev->last_ps) != 0)
3258                 goto reset;
3259         if (ctrl->hmpre && nvme_setup_host_mem(ndev))
3260                 goto reset;
3261
3262         return 0;
3263 reset:
3264         return nvme_try_sched_reset(ctrl);
3265 }
3266
3267 static int nvme_suspend(struct device *dev)
3268 {
3269         struct pci_dev *pdev = to_pci_dev(dev);
3270         struct nvme_dev *ndev = pci_get_drvdata(pdev);
3271         struct nvme_ctrl *ctrl = &ndev->ctrl;
3272         int ret = -EBUSY;
3273
3274         ndev->last_ps = U32_MAX;
3275
3276         /*
3277          * The platform does not remove power for a kernel managed suspend so
3278          * use host managed nvme power settings for lowest idle power if
3279          * possible. This should have quicker resume latency than a full device
3280          * shutdown.  But if the firmware is involved after the suspend or the
3281          * device does not support any non-default power states, shut down the
3282          * device fully.
3283          *
3284          * If ASPM is not enabled for the device, shut down the device and allow
3285          * the PCI bus layer to put it into D3 in order to take the PCIe link
3286          * down, so as to allow the platform to achieve its minimum low-power
3287          * state (which may not be possible if the link is up).
3288          */
3289         if (pm_suspend_via_firmware() || !ctrl->npss ||
3290             !pcie_aspm_enabled(pdev) ||
3291             (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND))
3292                 return nvme_disable_prepare_reset(ndev, true);
3293
3294         nvme_start_freeze(ctrl);
3295         nvme_wait_freeze(ctrl);
3296         nvme_sync_queues(ctrl);
3297
3298         if (ctrl->state != NVME_CTRL_LIVE)
3299                 goto unfreeze;
3300
3301         /*
3302          * Host memory access may not be successful in a system suspend state,
3303          * but the specification allows the controller to access memory in a
3304          * non-operational power state.
3305          */
3306         if (ndev->hmb) {
3307                 ret = nvme_set_host_mem(ndev, 0);
3308                 if (ret < 0)
3309                         goto unfreeze;
3310         }
3311
3312         ret = nvme_get_power_state(ctrl, &ndev->last_ps);
3313         if (ret < 0)
3314                 goto unfreeze;
3315
3316         /*
3317          * A saved state prevents pci pm from generically controlling the
3318          * device's power. If we're using protocol specific settings, we don't
3319          * want pci interfering.
3320          */
3321         pci_save_state(pdev);
3322
3323         ret = nvme_set_power_state(ctrl, ctrl->npss);
3324         if (ret < 0)
3325                 goto unfreeze;
3326
3327         if (ret) {
3328                 /* discard the saved state */
3329                 pci_load_saved_state(pdev, NULL);
3330
3331                 /*
3332                  * Clearing npss forces a controller reset on resume. The
3333                  * correct value will be rediscovered then.
3334                  */
3335                 ret = nvme_disable_prepare_reset(ndev, true);
3336                 ctrl->npss = 0;
3337         }
3338 unfreeze:
3339         nvme_unfreeze(ctrl);
3340         return ret;
3341 }
3342
3343 static int nvme_simple_suspend(struct device *dev)
3344 {
3345         struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3346
3347         return nvme_disable_prepare_reset(ndev, true);
3348 }
3349
3350 static int nvme_simple_resume(struct device *dev)
3351 {
3352         struct pci_dev *pdev = to_pci_dev(dev);
3353         struct nvme_dev *ndev = pci_get_drvdata(pdev);
3354
3355         return nvme_try_sched_reset(&ndev->ctrl);
3356 }
3357
3358 static const struct dev_pm_ops nvme_dev_pm_ops = {
3359         .suspend        = nvme_suspend,
3360         .resume         = nvme_resume,
3361         .freeze         = nvme_simple_suspend,
3362         .thaw           = nvme_simple_resume,
3363         .poweroff       = nvme_simple_suspend,
3364         .restore        = nvme_simple_resume,
3365 };
3366 #endif /* CONFIG_PM_SLEEP */
3367
3368 static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
3369                                                 pci_channel_state_t state)
3370 {
3371         struct nvme_dev *dev = pci_get_drvdata(pdev);
3372
3373         /*
3374          * A frozen channel requires a reset. When detected, this method will
3375          * shutdown the controller to quiesce. The controller will be restarted
3376          * after the slot reset through driver's slot_reset callback.
3377          */
3378         switch (state) {
3379         case pci_channel_io_normal:
3380                 return PCI_ERS_RESULT_CAN_RECOVER;
3381         case pci_channel_io_frozen:
3382                 dev_warn(dev->ctrl.device,
3383                         "frozen state error detected, reset controller\n");
3384                 nvme_dev_disable(dev, false);
3385                 return PCI_ERS_RESULT_NEED_RESET;
3386         case pci_channel_io_perm_failure:
3387                 dev_warn(dev->ctrl.device,
3388                         "failure state error detected, request disconnect\n");
3389                 return PCI_ERS_RESULT_DISCONNECT;
3390         }
3391         return PCI_ERS_RESULT_NEED_RESET;
3392 }
3393
3394 static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
3395 {
3396         struct nvme_dev *dev = pci_get_drvdata(pdev);
3397
3398         dev_info(dev->ctrl.device, "restart after slot reset\n");
3399         pci_restore_state(pdev);
3400         nvme_reset_ctrl(&dev->ctrl);
3401         return PCI_ERS_RESULT_RECOVERED;
3402 }
3403
3404 static void nvme_error_resume(struct pci_dev *pdev)
3405 {
3406         struct nvme_dev *dev = pci_get_drvdata(pdev);
3407
3408         flush_work(&dev->ctrl.reset_work);
3409 }
3410
3411 static const struct pci_error_handlers nvme_err_handler = {
3412         .error_detected = nvme_error_detected,
3413         .slot_reset     = nvme_slot_reset,
3414         .resume         = nvme_error_resume,
3415         .reset_prepare  = nvme_reset_prepare,
3416         .reset_done     = nvme_reset_done,
3417 };
3418
3419 static const struct pci_device_id nvme_id_table[] = {
3420         { PCI_VDEVICE(INTEL, 0x0953),   /* Intel 750/P3500/P3600/P3700 */
3421                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3422                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
3423         { PCI_VDEVICE(INTEL, 0x0a53),   /* Intel P3520 */
3424                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3425                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
3426         { PCI_VDEVICE(INTEL, 0x0a54),   /* Intel P4500/P4600 */
3427                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3428                                 NVME_QUIRK_DEALLOCATE_ZEROES |
3429                                 NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3430         { PCI_VDEVICE(INTEL, 0x0a55),   /* Dell Express Flash P4600 */
3431                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3432                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
3433         { PCI_VDEVICE(INTEL, 0xf1a5),   /* Intel 600P/P3100 */
3434                 .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3435                                 NVME_QUIRK_MEDIUM_PRIO_SQ |
3436                                 NVME_QUIRK_NO_TEMP_THRESH_CHANGE |
3437                                 NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3438         { PCI_VDEVICE(INTEL, 0xf1a6),   /* Intel 760p/Pro 7600p */
3439                 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3440         { PCI_VDEVICE(INTEL, 0x5845),   /* Qemu emulated controller */
3441                 .driver_data = NVME_QUIRK_IDENTIFY_CNS |
3442                                 NVME_QUIRK_DISABLE_WRITE_ZEROES |
3443                                 NVME_QUIRK_BOGUS_NID, },
3444         { PCI_VDEVICE(REDHAT, 0x0010),  /* Qemu emulated controller */
3445                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3446         { PCI_DEVICE(0x126f, 0x2263),   /* Silicon Motion unidentified */
3447                 .driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3448                                 NVME_QUIRK_BOGUS_NID, },
3449         { PCI_DEVICE(0x1bb1, 0x0100),   /* Seagate Nytro Flash Storage */
3450                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3451                                 NVME_QUIRK_NO_NS_DESC_LIST, },
3452         { PCI_DEVICE(0x1c58, 0x0003),   /* HGST adapter */
3453                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3454         { PCI_DEVICE(0x1c58, 0x0023),   /* WDC SN200 adapter */
3455                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3456         { PCI_DEVICE(0x1c5f, 0x0540),   /* Memblaze Pblaze4 adapter */
3457                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3458         { PCI_DEVICE(0x144d, 0xa821),   /* Samsung PM1725 */
3459                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3460         { PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
3461                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3462                                 NVME_QUIRK_DISABLE_WRITE_ZEROES|
3463                                 NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3464         { PCI_DEVICE(0x1987, 0x5012),   /* Phison E12 */
3465                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3466         { PCI_DEVICE(0x1987, 0x5016),   /* Phison E16 */
3467                 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
3468                                 NVME_QUIRK_BOGUS_NID, },
3469         { PCI_DEVICE(0x1b4b, 0x1092),   /* Lexar 256 GB SSD */
3470                 .driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3471                                 NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3472         { PCI_DEVICE(0x1cc1, 0x33f8),   /* ADATA IM2P33F8ABR1 1 TB */
3473                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3474         { PCI_DEVICE(0x10ec, 0x5762),   /* ADATA SX6000LNP */
3475                 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
3476                                 NVME_QUIRK_BOGUS_NID, },
3477         { PCI_DEVICE(0x1cc1, 0x8201),   /* ADATA SX8200PNP 512GB */
3478                 .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3479                                 NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3480          { PCI_DEVICE(0x1344, 0x5407), /* Micron Technology Inc NVMe SSD */
3481                 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN },
3482         { PCI_DEVICE(0x1c5c, 0x1504),   /* SK Hynix PC400 */
3483                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3484         { PCI_DEVICE(0x1c5c, 0x174a),   /* SK Hynix P31 SSD */
3485                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3486         { PCI_DEVICE(0x15b7, 0x2001),   /*  Sandisk Skyhawk */
3487                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3488         { PCI_DEVICE(0x1d97, 0x2263),   /* SPCC */
3489                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3490         { PCI_DEVICE(0x144d, 0xa80b),   /* Samsung PM9B1 256G and 512G */
3491                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3492         { PCI_DEVICE(0x144d, 0xa809),   /* Samsung MZALQ256HBJD 256G */
3493                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3494         { PCI_DEVICE(0x1cc4, 0x6303),   /* UMIS RPJTJ512MGE1QDY 512G */
3495                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3496         { PCI_DEVICE(0x1cc4, 0x6302),   /* UMIS RPJTJ256MGE1QDY 256G */
3497                 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3498         { PCI_DEVICE(0x2646, 0x2262),   /* KINGSTON SKC2000 NVMe SSD */
3499                 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3500         { PCI_DEVICE(0x2646, 0x2263),   /* KINGSTON A2000 NVMe SSD  */
3501                 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3502         { PCI_DEVICE(0x1e4B, 0x1001),   /* MAXIO MAP1001 */
3503                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3504         { PCI_DEVICE(0x1e4B, 0x1002),   /* MAXIO MAP1002 */
3505                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3506         { PCI_DEVICE(0x1e4B, 0x1202),   /* MAXIO MAP1202 */
3507                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3508         { PCI_DEVICE(0x1cc1, 0x5350),   /* ADATA XPG GAMMIX S50 */
3509                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3510         { PCI_DEVICE(0x1dbe, 0x5236),   /* ADATA XPG GAMMIX S70 */
3511                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3512         { PCI_DEVICE(0x1e49, 0x0041),   /* ZHITAI TiPro7000 NVMe SSD */
3513                 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3514         { PCI_DEVICE(0xc0a9, 0x540a),   /* Crucial P2 */
3515                 .driver_data = NVME_QUIRK_BOGUS_NID, },
3516         { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061),
3517                 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
3518         { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065),
3519                 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
3520         { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x8061),
3521                 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
3522         { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd00),
3523                 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
3524         { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd01),
3525                 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
3526         { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd02),
3527                 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
3528         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001),
3529                 .driver_data = NVME_QUIRK_SINGLE_VECTOR },
3530         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
3531         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2005),
3532                 .driver_data = NVME_QUIRK_SINGLE_VECTOR |
3533                                 NVME_QUIRK_128_BYTES_SQES |
3534                                 NVME_QUIRK_SHARED_TAGS |
3535                                 NVME_QUIRK_SKIP_CID_GEN },
3536         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
3537         { 0, }
3538 };
3539 MODULE_DEVICE_TABLE(pci, nvme_id_table);
3540
3541 static struct pci_driver nvme_driver = {
3542         .name           = "nvme",
3543         .id_table       = nvme_id_table,
3544         .probe          = nvme_probe,
3545         .remove         = nvme_remove,
3546         .shutdown       = nvme_shutdown,
3547 #ifdef CONFIG_PM_SLEEP
3548         .driver         = {
3549                 .pm     = &nvme_dev_pm_ops,
3550         },
3551 #endif
3552         .sriov_configure = pci_sriov_configure_simple,
3553         .err_handler    = &nvme_err_handler,
3554 };
3555
3556 static int __init nvme_init(void)
3557 {
3558         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
3559         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
3560         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
3561         BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
3562
3563         return pci_register_driver(&nvme_driver);
3564 }
3565
3566 static void __exit nvme_exit(void)
3567 {
3568         pci_unregister_driver(&nvme_driver);
3569         flush_workqueue(nvme_wq);
3570 }
3571
3572 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
3573 MODULE_LICENSE("GPL");
3574 MODULE_VERSION("1.0");
3575 module_init(nvme_init);
3576 module_exit(nvme_exit);