drivers/nvme/nvme.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (C) 2017 NXP Semiconductors
   4  * Copyright (C) 2017 Bin Meng <bmeng.cn@gmail.com>
   5  */
   6
   7 #include <common.h>
   8 #include <dm.h>
   9 #include <errno.h>
  10 #include <memalign.h>
  11 #include <pci.h>
  12 #include <dm/device-internal.h>
  13 #include "nvme.h"
  14
  15 #define NVME_Q_DEPTH            2
  16 #define NVME_AQ_DEPTH           2
  17 #define NVME_SQ_SIZE(depth)     (depth * sizeof(struct nvme_command))
  18 #define NVME_CQ_SIZE(depth)     (depth * sizeof(struct nvme_completion))
  19 #define ADMIN_TIMEOUT           60
  20 #define IO_TIMEOUT              30
  21 #define MAX_PRP_POOL            512
  22
  23 enum nvme_queue_id {
  24         NVME_ADMIN_Q,
  25         NVME_IO_Q,
  26         NVME_Q_NUM,
  27 };
  28
  29 /*
  30  * An NVM Express queue. Each device has at least two (one for admin
  31  * commands and one for I/O commands).
  32  */
  33 struct nvme_queue {
  34         struct nvme_dev *dev;
  35         struct nvme_command *sq_cmds;
  36         struct nvme_completion *cqes;
  37         wait_queue_head_t sq_full;
  38         u32 __iomem *q_db;
  39         u16 q_depth;
  40         s16 cq_vector;
  41         u16 sq_head;
  42         u16 sq_tail;
  43         u16 cq_head;
  44         u16 qid;
  45         u8 cq_phase;
  46         u8 cqe_seen;
  47         unsigned long cmdid_data[];
  48 };
  49
  50 static int nvme_wait_ready(struct nvme_dev *dev, bool enabled)
  51 {
  52         u32 bit = enabled ? NVME_CSTS_RDY : 0;
  53         int timeout;
  54         ulong start;
  55
  56         /* Timeout field in the CAP register is in 500 millisecond units */
  57         timeout = NVME_CAP_TIMEOUT(dev->cap) * 500;
  58
  59         start = get_timer(0);
  60         while (get_timer(start) < timeout) {
  61                 if ((readl(&dev->bar->csts) & NVME_CSTS_RDY) == bit)
  62                         return 0;
  63         }
  64
  65         return -ETIME;
  66 }
  67
  68 static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
  69                            int total_len, u64 dma_addr)
  70 {
  71         u32 page_size = dev->page_size;
  72         int offset = dma_addr & (page_size - 1);
  73         u64 *prp_pool;
  74         int length = total_len;
  75         int i, nprps;
  76         u32 prps_per_page = (page_size >> 3) - 1;
  77         u32 num_pages;
  78
  79         length -= (page_size - offset);
  80
  81         if (length <= 0) {
  82                 *prp2 = 0;
  83                 return 0;
  84         }
  85
  86         if (length)
  87                 dma_addr += (page_size - offset);
  88
  89         if (length <= page_size) {
  90                 *prp2 = dma_addr;
  91                 return 0;
  92         }
  93
  94         nprps = DIV_ROUND_UP(length, page_size);
  95         num_pages = DIV_ROUND_UP(nprps, prps_per_page);
  96
  97         if (nprps > dev->prp_entry_num) {
  98                 free(dev->prp_pool);
  99                 /*
 100                  * Always increase in increments of pages.  It doesn't waste
 101                  * much memory and reduces the number of allocations.
 102                  */
 103                 dev->prp_pool = memalign(page_size, num_pages * page_size);
 104                 if (!dev->prp_pool) {
 105                         printf("Error: malloc prp_pool fail\n");
 106                         return -ENOMEM;
 107                 }
 108                 dev->prp_entry_num = prps_per_page * num_pages;
 109         }
 110
 111         prp_pool = dev->prp_pool;
 112         i = 0;
 113         while (nprps) {
 114                 if (i == ((page_size >> 3) - 1)) {
 115                         *(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
 116                                         page_size);
 117                         i = 0;
 118                         prp_pool += page_size;
 119                 }
 120                 *(prp_pool + i++) = cpu_to_le64(dma_addr);
 121                 dma_addr += page_size;
 122                 nprps--;
 123         }
 124         *prp2 = (ulong)dev->prp_pool;
 125
 126         flush_dcache_range((ulong)dev->prp_pool, (ulong)dev->prp_pool +
 127                            dev->prp_entry_num * sizeof(u64));
 128
 129         return 0;
 130 }
 131
 132 static __le16 nvme_get_cmd_id(void)
 133 {
 134         static unsigned short cmdid;
 135
 136         return cpu_to_le16((cmdid < USHRT_MAX) ? cmdid++ : 0);
 137 }
 138
 139 static u16 nvme_read_completion_status(struct nvme_queue *nvmeq, u16 index)
 140 {
 141         u64 start = (ulong)&nvmeq->cqes[index];
 142         u64 stop = start + sizeof(struct nvme_completion);
 143
 144         invalidate_dcache_range(start, stop);
 145
 146         return le16_to_cpu(readw(&(nvmeq->cqes[index].status)));
 147 }
 148
 149 /**
 150  * nvme_submit_cmd() - copy a command into a queue and ring the doorbell
 151  *
 152  * @nvmeq:      The queue to use
 153  * @cmd:        The command to send
 154  */
 155 static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
 156 {
 157         u16 tail = nvmeq->sq_tail;
 158
 159         memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
 160         flush_dcache_range((ulong)&nvmeq->sq_cmds[tail],
 161                            (ulong)&nvmeq->sq_cmds[tail] + sizeof(*cmd));
 162
 163         if (++tail == nvmeq->q_depth)
 164                 tail = 0;
 165         writel(tail, nvmeq->q_db);
 166         nvmeq->sq_tail = tail;
 167 }
 168
 169 static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq,
 170                                 struct nvme_command *cmd,
 171                                 u32 *result, unsigned timeout)
 172 {
 173         u16 head = nvmeq->cq_head;
 174         u16 phase = nvmeq->cq_phase;
 175         u16 status;
 176         ulong start_time;
 177         ulong timeout_us = timeout * 100000;
 178
 179         cmd->common.command_id = nvme_get_cmd_id();
 180         nvme_submit_cmd(nvmeq, cmd);
 181
 182         start_time = timer_get_us();
 183
 184         for (;;) {
 185                 status = nvme_read_completion_status(nvmeq, head);
 186                 if ((status & 0x01) == phase)
 187                         break;
 188                 if (timeout_us > 0 && (timer_get_us() - start_time)
 189                     >= timeout_us)
 190                         return -ETIMEDOUT;
 191         }
 192
 193         status >>= 1;
 194         if (status) {
 195                 printf("ERROR: status = %x, phase = %d, head = %d\n",
 196                        status, phase, head);
 197                 status = 0;
 198                 if (++head == nvmeq->q_depth) {
 199                         head = 0;
 200                         phase = !phase;
 201                 }
 202                 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
 203                 nvmeq->cq_head = head;
 204                 nvmeq->cq_phase = phase;
 205
 206                 return -EIO;
 207         }
 208
 209         if (result)
 210                 *result = le32_to_cpu(readl(&(nvmeq->cqes[head].result)));
 211
 212         if (++head == nvmeq->q_depth) {
 213                 head = 0;
 214                 phase = !phase;
 215         }
 216         writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
 217         nvmeq->cq_head = head;
 218         nvmeq->cq_phase = phase;
 219
 220         return status;
 221 }
 222
 223 static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
 224                                  u32 *result)
 225 {
 226         return nvme_submit_sync_cmd(dev->queues[NVME_ADMIN_Q], cmd,
 227                                     result, ADMIN_TIMEOUT);
 228 }
 229
 230 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
 231                                            int qid, int depth)
 232 {
 233         struct nvme_queue *nvmeq = malloc(sizeof(*nvmeq));
 234         if (!nvmeq)
 235                 return NULL;
 236         memset(nvmeq, 0, sizeof(*nvmeq));
 237
 238         nvmeq->cqes = (void *)memalign(4096, NVME_CQ_SIZE(depth));
 239         if (!nvmeq->cqes)
 240                 goto free_nvmeq;
 241         memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(depth));
 242
 243         nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
 244         if (!nvmeq->sq_cmds)
 245                 goto free_queue;
 246         memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));
 247
 248         nvmeq->dev = dev;
 249
 250         nvmeq->cq_head = 0;
 251         nvmeq->cq_phase = 1;
 252         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
 253         nvmeq->q_depth = depth;
 254         nvmeq->qid = qid;
 255         dev->queue_count++;
 256         dev->queues[qid] = nvmeq;
 257
 258         return nvmeq;
 259
 260  free_queue:
 261         free((void *)nvmeq->cqes);
 262  free_nvmeq:
 263         free(nvmeq);
 264
 265         return NULL;
 266 }
 267
 268 static int nvme_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 269 {
 270         struct nvme_command c;
 271
 272         memset(&c, 0, sizeof(c));
 273         c.delete_queue.opcode = opcode;
 274         c.delete_queue.qid = cpu_to_le16(id);
 275
 276         return nvme_submit_admin_cmd(dev, &c, NULL);
 277 }
 278
 279 static int nvme_delete_sq(struct nvme_dev *dev, u16 sqid)
 280 {
 281         return nvme_delete_queue(dev, nvme_admin_delete_sq, sqid);
 282 }
 283
 284 static int nvme_delete_cq(struct nvme_dev *dev, u16 cqid)
 285 {
 286         return nvme_delete_queue(dev, nvme_admin_delete_cq, cqid);
 287 }
 288
 289 static int nvme_enable_ctrl(struct nvme_dev *dev)
 290 {
 291         dev->ctrl_config &= ~NVME_CC_SHN_MASK;
 292         dev->ctrl_config |= NVME_CC_ENABLE;
 293         writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
 294
 295         return nvme_wait_ready(dev, true);
 296 }
 297
 298 static int nvme_disable_ctrl(struct nvme_dev *dev)
 299 {
 300         dev->ctrl_config &= ~NVME_CC_SHN_MASK;
 301         dev->ctrl_config &= ~NVME_CC_ENABLE;
 302         writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
 303
 304         return nvme_wait_ready(dev, false);
 305 }
 306
 307 static void nvme_free_queue(struct nvme_queue *nvmeq)
 308 {
 309         free((void *)nvmeq->cqes);
 310         free(nvmeq->sq_cmds);
 311         free(nvmeq);
 312 }
 313
 314 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
 315 {
 316         int i;
 317
 318         for (i = dev->queue_count - 1; i >= lowest; i--) {
 319                 struct nvme_queue *nvmeq = dev->queues[i];
 320                 dev->queue_count--;
 321                 dev->queues[i] = NULL;
 322                 nvme_free_queue(nvmeq);
 323         }
 324 }
 325
 326 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 327 {
 328         struct nvme_dev *dev = nvmeq->dev;
 329
 330         nvmeq->sq_tail = 0;
 331         nvmeq->cq_head = 0;
 332         nvmeq->cq_phase = 1;
 333         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
 334         memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(nvmeq->q_depth));
 335         flush_dcache_range((ulong)nvmeq->cqes,
 336                            (ulong)nvmeq->cqes + NVME_CQ_SIZE(nvmeq->q_depth));
 337         dev->online_queues++;
 338 }
 339
 340 static int nvme_configure_admin_queue(struct nvme_dev *dev)
 341 {
 342         int result;
 343         u32 aqa;
 344         u64 cap = dev->cap;
 345         struct nvme_queue *nvmeq;
 346         /* most architectures use 4KB as the page size */
 347         unsigned page_shift = 12;
 348         unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
 349         unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;
 350
 351         if (page_shift < dev_page_min) {
 352                 debug("Device minimum page size (%u) too large for host (%u)\n",
 353                       1 << dev_page_min, 1 << page_shift);
 354                 return -ENODEV;
 355         }
 356
 357         if (page_shift > dev_page_max) {
 358                 debug("Device maximum page size (%u) smaller than host (%u)\n",
 359                       1 << dev_page_max, 1 << page_shift);
 360                 page_shift = dev_page_max;
 361         }
 362
 363         result = nvme_disable_ctrl(dev);
 364         if (result < 0)
 365                 return result;
 366
 367         nvmeq = dev->queues[NVME_ADMIN_Q];
 368         if (!nvmeq) {
 369                 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
 370                 if (!nvmeq)
 371                         return -ENOMEM;
 372         }
 373
 374         aqa = nvmeq->q_depth - 1;
 375         aqa |= aqa << 16;
 376         aqa |= aqa << 16;
 377
 378         dev->page_size = 1 << page_shift;
 379
 380         dev->ctrl_config = NVME_CC_CSS_NVM;
 381         dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
 382         dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
 383         dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
 384
 385         writel(aqa, &dev->bar->aqa);
 386         nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
 387         nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
 388
 389         result = nvme_enable_ctrl(dev);
 390         if (result)
 391                 goto free_nvmeq;
 392
 393         nvmeq->cq_vector = 0;
 394
 395         nvme_init_queue(dev->queues[NVME_ADMIN_Q], 0);
 396
 397         return result;
 398
 399  free_nvmeq:
 400         nvme_free_queues(dev, 0);
 401
 402         return result;
 403 }
 404
 405 static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
 406                             struct nvme_queue *nvmeq)
 407 {
 408         struct nvme_command c;
 409         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
 410
 411         memset(&c, 0, sizeof(c));
 412         c.create_cq.opcode = nvme_admin_create_cq;
 413         c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
 414         c.create_cq.cqid = cpu_to_le16(qid);
 415         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 416         c.create_cq.cq_flags = cpu_to_le16(flags);
 417         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 418
 419         return nvme_submit_admin_cmd(dev, &c, NULL);
 420 }
 421
 422 static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
 423                             struct nvme_queue *nvmeq)
 424 {
 425         struct nvme_command c;
 426         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
 427
 428         memset(&c, 0, sizeof(c));
 429         c.create_sq.opcode = nvme_admin_create_sq;
 430         c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
 431         c.create_sq.sqid = cpu_to_le16(qid);
 432         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 433         c.create_sq.sq_flags = cpu_to_le16(flags);
 434         c.create_sq.cqid = cpu_to_le16(qid);
 435
 436         return nvme_submit_admin_cmd(dev, &c, NULL);
 437 }
 438
 439 int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 440                   unsigned cns, dma_addr_t dma_addr)
 441 {
 442         struct nvme_command c;
 443         u32 page_size = dev->page_size;
 444         int offset = dma_addr & (page_size - 1);
 445         int length = sizeof(struct nvme_id_ctrl);
 446         int ret;
 447
 448         memset(&c, 0, sizeof(c));
 449         c.identify.opcode = nvme_admin_identify;
 450         c.identify.nsid = cpu_to_le32(nsid);
 451         c.identify.prp1 = cpu_to_le64(dma_addr);
 452
 453         length -= (page_size - offset);
 454         if (length <= 0) {
 455                 c.identify.prp2 = 0;
 456         } else {
 457                 dma_addr += (page_size - offset);
 458                 c.identify.prp2 = cpu_to_le64(dma_addr);
 459         }
 460
 461         c.identify.cns = cpu_to_le32(cns);
 462
 463         ret = nvme_submit_admin_cmd(dev, &c, NULL);
 464         if (!ret)
 465                 invalidate_dcache_range(dma_addr,
 466                                         dma_addr + sizeof(struct nvme_id_ctrl));
 467
 468         return ret;
 469 }
 470
 471 int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
 472                       dma_addr_t dma_addr, u32 *result)
 473 {
 474         struct nvme_command c;
 475
 476         memset(&c, 0, sizeof(c));
 477         c.features.opcode = nvme_admin_get_features;
 478         c.features.nsid = cpu_to_le32(nsid);
 479         c.features.prp1 = cpu_to_le64(dma_addr);
 480         c.features.fid = cpu_to_le32(fid);
 481
 482         /*
 483          * TODO: add cache invalidate operation when the size of
 484          * the DMA buffer is known
 485          */
 486
 487         return nvme_submit_admin_cmd(dev, &c, result);
 488 }
 489
 490 int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
 491                       dma_addr_t dma_addr, u32 *result)
 492 {
 493         struct nvme_command c;
 494
 495         memset(&c, 0, sizeof(c));
 496         c.features.opcode = nvme_admin_set_features;
 497         c.features.prp1 = cpu_to_le64(dma_addr);
 498         c.features.fid = cpu_to_le32(fid);
 499         c.features.dword11 = cpu_to_le32(dword11);
 500
 501         /*
 502          * TODO: add cache flush operation when the size of
 503          * the DMA buffer is known
 504          */
 505
 506         return nvme_submit_admin_cmd(dev, &c, result);
 507 }
 508
 509 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 510 {
 511         struct nvme_dev *dev = nvmeq->dev;
 512         int result;
 513
 514         nvmeq->cq_vector = qid - 1;
 515         result = nvme_alloc_cq(dev, qid, nvmeq);
 516         if (result < 0)
 517                 goto release_cq;
 518
 519         result = nvme_alloc_sq(dev, qid, nvmeq);
 520         if (result < 0)
 521                 goto release_sq;
 522
 523         nvme_init_queue(nvmeq, qid);
 524
 525         return result;
 526
 527  release_sq:
 528         nvme_delete_sq(dev, qid);
 529  release_cq:
 530         nvme_delete_cq(dev, qid);
 531
 532         return result;
 533 }
 534
 535 static int nvme_set_queue_count(struct nvme_dev *dev, int count)
 536 {
 537         int status;
 538         u32 result;
 539         u32 q_count = (count - 1) | ((count - 1) << 16);
 540
 541         status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES,
 542                         q_count, 0, &result);
 543
 544         if (status < 0)
 545                 return status;
 546         if (status > 1)
 547                 return 0;
 548
 549         return min(result & 0xffff, result >> 16) + 1;
 550 }
 551
 552 static void nvme_create_io_queues(struct nvme_dev *dev)
 553 {
 554         unsigned int i;
 555
 556         for (i = dev->queue_count; i <= dev->max_qid; i++)
 557                 if (!nvme_alloc_queue(dev, i, dev->q_depth))
 558                         break;
 559
 560         for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
 561                 if (nvme_create_queue(dev->queues[i], i))
 562                         break;
 563 }
 564
 565 static int nvme_setup_io_queues(struct nvme_dev *dev)
 566 {
 567         int nr_io_queues;
 568         int result;
 569
 570         nr_io_queues = 1;
 571         result = nvme_set_queue_count(dev, nr_io_queues);
 572         if (result <= 0)
 573                 return result;
 574
 575         dev->max_qid = nr_io_queues;
 576
 577         /* Free previously allocated queues */
 578         nvme_free_queues(dev, nr_io_queues + 1);
 579         nvme_create_io_queues(dev);
 580
 581         return 0;
 582 }
 583
 584 static int nvme_get_info_from_identify(struct nvme_dev *dev)
 585 {
 586         struct nvme_id_ctrl *ctrl;
 587         int ret;
 588         int shift = NVME_CAP_MPSMIN(dev->cap) + 12;
 589
 590         ctrl = memalign(dev->page_size, sizeof(struct nvme_id_ctrl));
 591         if (!ctrl)
 592                 return -ENOMEM;
 593
 594         ret = nvme_identify(dev, 0, 1, (dma_addr_t)(long)ctrl);
 595         if (ret) {
 596                 free(ctrl);
 597                 return -EIO;
 598         }
 599
 600         dev->nn = le32_to_cpu(ctrl->nn);
 601         dev->vwc = ctrl->vwc;
 602         memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
 603         memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
 604         memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
 605         if (ctrl->mdts)
 606                 dev->max_transfer_shift = (ctrl->mdts + shift);
 607         else {
 608                 /*
 609                  * Maximum Data Transfer Size (MDTS) field indicates the maximum
 610                  * data transfer size between the host and the controller. The
 611                  * host should not submit a command that exceeds this transfer
 612                  * size. The value is in units of the minimum memory page size
 613                  * and is reported as a power of two (2^n).
 614                  *
 615                  * The spec also says: a value of 0h indicates no restrictions
 616                  * on transfer size. But in nvme_blk_read/write() below we have
 617                  * the following algorithm for maximum number of logic blocks
 618                  * per transfer:
 619                  *
 620                  * u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
 621                  *
 622                  * In order for lbas not to overflow, the maximum number is 15
 623                  * which means dev->max_transfer_shift = 15 + 9 (ns->lba_shift).
 624                  * Let's use 20 which provides 1MB size.
 625                  */
 626                 dev->max_transfer_shift = 20;
 627         }
 628
 629         free(ctrl);
 630         return 0;
 631 }
 632
 633 int nvme_get_namespace_id(struct udevice *udev, u32 *ns_id, u8 *eui64)
 634 {
 635         struct nvme_ns *ns = dev_get_priv(udev);
 636
 637         if (ns_id)
 638                 *ns_id = ns->ns_id;
 639         if (eui64)
 640                 memcpy(eui64, ns->eui64, sizeof(ns->eui64));
 641
 642         return 0;
 643 }
 644
 645 int nvme_scan_namespace(void)
 646 {
 647         struct uclass *uc;
 648         struct udevice *dev;
 649         int ret;
 650
 651         ret = uclass_get(UCLASS_NVME, &uc);
 652         if (ret)
 653                 return ret;
 654
 655         uclass_foreach_dev(dev, uc) {
 656                 ret = device_probe(dev);
 657                 if (ret)
 658                         return ret;
 659         }
 660
 661         return 0;
 662 }
 663
 664 static int nvme_blk_probe(struct udevice *udev)
 665 {
 666         struct nvme_dev *ndev = dev_get_priv(udev->parent);
 667         struct blk_desc *desc = dev_get_uclass_platdata(udev);
 668         struct nvme_ns *ns = dev_get_priv(udev);
 669         u8 flbas;
 670         struct pci_child_platdata *pplat;
 671         struct nvme_id_ns *id;
 672
 673         id = memalign(ndev->page_size, sizeof(struct nvme_id_ns));
 674         if (!id)
 675                 return -ENOMEM;
 676
 677         memset(ns, 0, sizeof(*ns));
 678         ns->dev = ndev;
 679         /* extract the namespace id from the block device name */
 680         ns->ns_id = trailing_strtol(udev->name) + 1;
 681         if (nvme_identify(ndev, ns->ns_id, 0, (dma_addr_t)(long)id)) {
 682                 free(id);
 683                 return -EIO;
 684         }
 685
 686         memcpy(&ns->eui64, &id->eui64, sizeof(id->eui64));
 687         flbas = id->flbas & NVME_NS_FLBAS_LBA_MASK;
 688         ns->flbas = flbas;
 689         ns->lba_shift = id->lbaf[flbas].ds;
 690         ns->mode_select_num_blocks = le64_to_cpu(id->nsze);
 691         ns->mode_select_block_len = 1 << ns->lba_shift;
 692         list_add(&ns->list, &ndev->namespaces);
 693
 694         desc->lba = ns->mode_select_num_blocks;
 695         desc->log2blksz = ns->lba_shift;
 696         desc->blksz = 1 << ns->lba_shift;
 697         desc->bdev = udev;
 698         pplat = dev_get_parent_platdata(udev->parent);
 699         sprintf(desc->vendor, "0x%.4x", pplat->vendor);
 700         memcpy(desc->product, ndev->serial, sizeof(ndev->serial));
 701         memcpy(desc->revision, ndev->firmware_rev, sizeof(ndev->firmware_rev));
 702
 703         free(id);
 704         return 0;
 705 }
 706
 707 static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
 708                          lbaint_t blkcnt, void *buffer, bool read)
 709 {
 710         struct nvme_ns *ns = dev_get_priv(udev);
 711         struct nvme_dev *dev = ns->dev;
 712         struct nvme_command c;
 713         struct blk_desc *desc = dev_get_uclass_platdata(udev);
 714         int status;
 715         u64 prp2;
 716         u64 total_len = blkcnt << desc->log2blksz;
 717         u64 temp_len = total_len;
 718
 719         u64 slba = blknr;
 720         u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
 721         u64 total_lbas = blkcnt;
 722
 723         flush_dcache_range((unsigned long)buffer,
 724                            (unsigned long)buffer + total_len);
 725
 726         c.rw.opcode = read ? nvme_cmd_read : nvme_cmd_write;
 727         c.rw.flags = 0;
 728         c.rw.nsid = cpu_to_le32(ns->ns_id);
 729         c.rw.control = 0;
 730         c.rw.dsmgmt = 0;
 731         c.rw.reftag = 0;
 732         c.rw.apptag = 0;
 733         c.rw.appmask = 0;
 734         c.rw.metadata = 0;
 735
 736         while (total_lbas) {
 737                 if (total_lbas < lbas) {
 738                         lbas = (u16)total_lbas;
 739                         total_lbas = 0;
 740                 } else {
 741                         total_lbas -= lbas;
 742                 }
 743
 744                 if (nvme_setup_prps(dev, &prp2,
 745                                     lbas << ns->lba_shift, (ulong)buffer))
 746                         return -EIO;
 747                 c.rw.slba = cpu_to_le64(slba);
 748                 slba += lbas;
 749                 c.rw.length = cpu_to_le16(lbas - 1);
 750                 c.rw.prp1 = cpu_to_le64((ulong)buffer);
 751                 c.rw.prp2 = cpu_to_le64(prp2);
 752                 status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
 753                                 &c, NULL, IO_TIMEOUT);
 754                 if (status)
 755                         break;
 756                 temp_len -= (u32)lbas << ns->lba_shift;
 757                 buffer += lbas << ns->lba_shift;
 758         }
 759
 760         if (read)
 761                 invalidate_dcache_range((unsigned long)buffer,
 762                                         (unsigned long)buffer + total_len);
 763
 764         return (total_len - temp_len) >> desc->log2blksz;
 765 }
 766
 767 static ulong nvme_blk_read(struct udevice *udev, lbaint_t blknr,
 768                            lbaint_t blkcnt, void *buffer)
 769 {
 770         return nvme_blk_rw(udev, blknr, blkcnt, buffer, true);
 771 }
 772
 773 static ulong nvme_blk_write(struct udevice *udev, lbaint_t blknr,
 774                             lbaint_t blkcnt, const void *buffer)
 775 {
 776         return nvme_blk_rw(udev, blknr, blkcnt, (void *)buffer, false);
 777 }
 778
 779 static const struct blk_ops nvme_blk_ops = {
 780         .read   = nvme_blk_read,
 781         .write  = nvme_blk_write,
 782 };
 783
 784 U_BOOT_DRIVER(nvme_blk) = {
 785         .name   = "nvme-blk",
 786         .id     = UCLASS_BLK,
 787         .probe  = nvme_blk_probe,
 788         .ops    = &nvme_blk_ops,
 789         .priv_auto_alloc_size = sizeof(struct nvme_ns),
 790 };
 791
 792 static int nvme_bind(struct udevice *udev)
 793 {
 794         static int ndev_num;
 795         char name[20];
 796
 797         sprintf(name, "nvme#%d", ndev_num++);
 798
 799         return device_set_name(udev, name);
 800 }
 801
 802 static int nvme_probe(struct udevice *udev)
 803 {
 804         int ret;
 805         struct nvme_dev *ndev = dev_get_priv(udev);
 806
 807         ndev->instance = trailing_strtol(udev->name);
 808
 809         INIT_LIST_HEAD(&ndev->namespaces);
 810         ndev->bar = dm_pci_map_bar(udev, PCI_BASE_ADDRESS_0,
 811                         PCI_REGION_MEM);
 812         if (readl(&ndev->bar->csts) == -1) {
 813                 ret = -ENODEV;
 814                 printf("Error: %s: Out of memory!\n", udev->name);
 815                 goto free_nvme;
 816         }
 817
 818         ndev->queues = malloc(NVME_Q_NUM * sizeof(struct nvme_queue *));
 819         if (!ndev->queues) {
 820                 ret = -ENOMEM;
 821                 printf("Error: %s: Out of memory!\n", udev->name);
 822                 goto free_nvme;
 823         }
 824         memset(ndev->queues, 0, NVME_Q_NUM * sizeof(struct nvme_queue *));
 825
 826         ndev->cap = nvme_readq(&ndev->bar->cap);
 827         ndev->q_depth = min_t(int, NVME_CAP_MQES(ndev->cap) + 1, NVME_Q_DEPTH);
 828         ndev->db_stride = 1 << NVME_CAP_STRIDE(ndev->cap);
 829         ndev->dbs = ((void __iomem *)ndev->bar) + 4096;
 830
 831         ret = nvme_configure_admin_queue(ndev);
 832         if (ret)
 833                 goto free_queue;
 834
 835         /* Allocate after the page size is known */
 836         ndev->prp_pool = memalign(ndev->page_size, MAX_PRP_POOL);
 837         if (!ndev->prp_pool) {
 838                 ret = -ENOMEM;
 839                 printf("Error: %s: Out of memory!\n", udev->name);
 840                 goto free_nvme;
 841         }
 842         ndev->prp_entry_num = MAX_PRP_POOL >> 3;
 843
 844         ret = nvme_setup_io_queues(ndev);
 845         if (ret)
 846                 goto free_queue;
 847
 848         nvme_get_info_from_identify(ndev);
 849
 850         return 0;
 851
 852 free_queue:
 853         free((void *)ndev->queues);
 854 free_nvme:
 855         return ret;
 856 }
 857
 858 U_BOOT_DRIVER(nvme) = {
 859         .name   = "nvme",
 860         .id     = UCLASS_NVME,
 861         .bind   = nvme_bind,
 862         .probe  = nvme_probe,
 863         .priv_auto_alloc_size = sizeof(struct nvme_dev),
 864 };
 865
 866 struct pci_device_id nvme_supported[] = {
 867         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, ~0) },
 868         {}
 869 };
 870
 871 U_BOOT_PCI_DEVICE(nvme, nvme_supported);