drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/blk-integrity.h>
  10 #include <linux/compat.h>
  11 #include <linux/delay.h>
  12 #include <linux/errno.h>
  13 #include <linux/hdreg.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <asm/unaligned.h>
  24
  25 #include "nvme.h"
  26 #include "fabrics.h"
  27 #include <linux/nvme-auth.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include "trace.h"
  31
  32 #define NVME_MINORS             (1U << MINORBITS)
  33
  34 struct nvme_ns_info {
  35         struct nvme_ns_ids ids;
  36         u32 nsid;
  37         __le32 anagrpid;
  38         bool is_shared;
  39         bool is_readonly;
  40         bool is_ready;
  41         bool is_removed;
  42 };
  43
  44 unsigned int admin_timeout = 60;
  45 module_param(admin_timeout, uint, 0644);
  46 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  47 EXPORT_SYMBOL_GPL(admin_timeout);
  48
  49 unsigned int nvme_io_timeout = 30;
  50 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  51 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  52 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  53
  54 static unsigned char shutdown_timeout = 5;
  55 module_param(shutdown_timeout, byte, 0644);
  56 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  57
  58 static u8 nvme_max_retries = 5;
  59 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  60 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  61
  62 static unsigned long default_ps_max_latency_us = 100000;
  63 module_param(default_ps_max_latency_us, ulong, 0644);
  64 MODULE_PARM_DESC(default_ps_max_latency_us,
  65                  "max power saving latency for new devices; use PM QOS to change per device");
  66
  67 static bool force_apst;
  68 module_param(force_apst, bool, 0644);
  69 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  70
  71 static unsigned long apst_primary_timeout_ms = 100;
  72 module_param(apst_primary_timeout_ms, ulong, 0644);
  73 MODULE_PARM_DESC(apst_primary_timeout_ms,
  74         "primary APST timeout in ms");
  75
  76 static unsigned long apst_secondary_timeout_ms = 2000;
  77 module_param(apst_secondary_timeout_ms, ulong, 0644);
  78 MODULE_PARM_DESC(apst_secondary_timeout_ms,
  79         "secondary APST timeout in ms");
  80
  81 static unsigned long apst_primary_latency_tol_us = 15000;
  82 module_param(apst_primary_latency_tol_us, ulong, 0644);
  83 MODULE_PARM_DESC(apst_primary_latency_tol_us,
  84         "primary APST latency tolerance in us");
  85
  86 static unsigned long apst_secondary_latency_tol_us = 100000;
  87 module_param(apst_secondary_latency_tol_us, ulong, 0644);
  88 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
  89         "secondary APST latency tolerance in us");
  90
  91 /*
  92  * nvme_wq - hosts nvme related works that are not reset or delete
  93  * nvme_reset_wq - hosts nvme reset works
  94  * nvme_delete_wq - hosts nvme delete works
  95  *
  96  * nvme_wq will host works such as scan, aen handling, fw activation,
  97  * keep-alive, periodic reconnects etc. nvme_reset_wq
  98  * runs reset works which also flush works hosted on nvme_wq for
  99  * serialization purposes. nvme_delete_wq host controller deletion
 100  * works which flush reset works for serialization.
 101  */
 102 struct workqueue_struct *nvme_wq;
 103 EXPORT_SYMBOL_GPL(nvme_wq);
 104
 105 struct workqueue_struct *nvme_reset_wq;
 106 EXPORT_SYMBOL_GPL(nvme_reset_wq);
 107
 108 struct workqueue_struct *nvme_delete_wq;
 109 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 110
 111 static LIST_HEAD(nvme_subsystems);
 112 static DEFINE_MUTEX(nvme_subsystems_lock);
 113
 114 static DEFINE_IDA(nvme_instance_ida);
 115 static dev_t nvme_ctrl_base_chr_devt;
 116 static struct class *nvme_class;
 117 static struct class *nvme_subsys_class;
 118
 119 static DEFINE_IDA(nvme_ns_chr_minor_ida);
 120 static dev_t nvme_ns_chr_devt;
 121 static struct class *nvme_ns_chr_class;
 122
 123 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
 124 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
 125                                            unsigned nsid);
 126 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
 127                                    struct nvme_command *cmd);
 128
 129 void nvme_queue_scan(struct nvme_ctrl *ctrl)
 130 {
 131         /*
 132          * Only new queue scan work when admin and IO queues are both alive
 133          */
 134         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 135                 queue_work(nvme_wq, &ctrl->scan_work);
 136 }
 137
 138 /*
 139  * Use this function to proceed with scheduling reset_work for a controller
 140  * that had previously been set to the resetting state. This is intended for
 141  * code paths that can't be interrupted by other reset attempts. A hot removal
 142  * may prevent this from succeeding.
 143  */
 144 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 145 {
 146         if (ctrl->state != NVME_CTRL_RESETTING)
 147                 return -EBUSY;
 148         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 149                 return -EBUSY;
 150         return 0;
 151 }
 152 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 153
 154 static void nvme_failfast_work(struct work_struct *work)
 155 {
 156         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
 157                         struct nvme_ctrl, failfast_work);
 158
 159         if (ctrl->state != NVME_CTRL_CONNECTING)
 160                 return;
 161
 162         set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 163         dev_info(ctrl->device, "failfast expired\n");
 164         nvme_kick_requeue_lists(ctrl);
 165 }
 166
 167 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
 168 {
 169         if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
 170                 return;
 171
 172         schedule_delayed_work(&ctrl->failfast_work,
 173                               ctrl->opts->fast_io_fail_tmo * HZ);
 174 }
 175
 176 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
 177 {
 178         if (!ctrl->opts)
 179                 return;
 180
 181         cancel_delayed_work_sync(&ctrl->failfast_work);
 182         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 183 }
 184
 185
 186 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 187 {
 188         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 189                 return -EBUSY;
 190         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 191                 return -EBUSY;
 192         return 0;
 193 }
 194 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 195
 196 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 197 {
 198         int ret;
 199
 200         ret = nvme_reset_ctrl(ctrl);
 201         if (!ret) {
 202                 flush_work(&ctrl->reset_work);
 203                 if (ctrl->state != NVME_CTRL_LIVE)
 204                         ret = -ENETRESET;
 205         }
 206
 207         return ret;
 208 }
 209
 210 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 211 {
 212         dev_info(ctrl->device,
 213                  "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
 214
 215         flush_work(&ctrl->reset_work);
 216         nvme_stop_ctrl(ctrl);
 217         nvme_remove_namespaces(ctrl);
 218         ctrl->ops->delete_ctrl(ctrl);
 219         nvme_uninit_ctrl(ctrl);
 220 }
 221
 222 static void nvme_delete_ctrl_work(struct work_struct *work)
 223 {
 224         struct nvme_ctrl *ctrl =
 225                 container_of(work, struct nvme_ctrl, delete_work);
 226
 227         nvme_do_delete_ctrl(ctrl);
 228 }
 229
 230 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 231 {
 232         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 233                 return -EBUSY;
 234         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 235                 return -EBUSY;
 236         return 0;
 237 }
 238 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 239
 240 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 241 {
 242         /*
 243          * Keep a reference until nvme_do_delete_ctrl() complete,
 244          * since ->delete_ctrl can free the controller.
 245          */
 246         nvme_get_ctrl(ctrl);
 247         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 248                 nvme_do_delete_ctrl(ctrl);
 249         nvme_put_ctrl(ctrl);
 250 }
 251
 252 static blk_status_t nvme_error_status(u16 status)
 253 {
 254         switch (status & 0x7ff) {
 255         case NVME_SC_SUCCESS:
 256                 return BLK_STS_OK;
 257         case NVME_SC_CAP_EXCEEDED:
 258                 return BLK_STS_NOSPC;
 259         case NVME_SC_LBA_RANGE:
 260         case NVME_SC_CMD_INTERRUPTED:
 261         case NVME_SC_NS_NOT_READY:
 262                 return BLK_STS_TARGET;
 263         case NVME_SC_BAD_ATTRIBUTES:
 264         case NVME_SC_ONCS_NOT_SUPPORTED:
 265         case NVME_SC_INVALID_OPCODE:
 266         case NVME_SC_INVALID_FIELD:
 267         case NVME_SC_INVALID_NS:
 268                 return BLK_STS_NOTSUPP;
 269         case NVME_SC_WRITE_FAULT:
 270         case NVME_SC_READ_ERROR:
 271         case NVME_SC_UNWRITTEN_BLOCK:
 272         case NVME_SC_ACCESS_DENIED:
 273         case NVME_SC_READ_ONLY:
 274         case NVME_SC_COMPARE_FAILED:
 275                 return BLK_STS_MEDIUM;
 276         case NVME_SC_GUARD_CHECK:
 277         case NVME_SC_APPTAG_CHECK:
 278         case NVME_SC_REFTAG_CHECK:
 279         case NVME_SC_INVALID_PI:
 280                 return BLK_STS_PROTECTION;
 281         case NVME_SC_RESERVATION_CONFLICT:
 282                 return BLK_STS_NEXUS;
 283         case NVME_SC_HOST_PATH_ERROR:
 284                 return BLK_STS_TRANSPORT;
 285         case NVME_SC_ZONE_TOO_MANY_ACTIVE:
 286                 return BLK_STS_ZONE_ACTIVE_RESOURCE;
 287         case NVME_SC_ZONE_TOO_MANY_OPEN:
 288                 return BLK_STS_ZONE_OPEN_RESOURCE;
 289         default:
 290                 return BLK_STS_IOERR;
 291         }
 292 }
 293
 294 static void nvme_retry_req(struct request *req)
 295 {
 296         unsigned long delay = 0;
 297         u16 crd;
 298
 299         /* The mask and shift result must be <= 3 */
 300         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 301         if (crd)
 302                 delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
 303
 304         nvme_req(req)->retries++;
 305         blk_mq_requeue_request(req, false);
 306         blk_mq_delay_kick_requeue_list(req->q, delay);
 307 }
 308
 309 static void nvme_log_error(struct request *req)
 310 {
 311         struct nvme_ns *ns = req->q->queuedata;
 312         struct nvme_request *nr = nvme_req(req);
 313
 314         if (ns) {
 315                 pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
 316                        ns->disk ? ns->disk->disk_name : "?",
 317                        nvme_get_opcode_str(nr->cmd->common.opcode),
 318                        nr->cmd->common.opcode,
 319                        (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)),
 320                        (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
 321                        nvme_get_error_status_str(nr->status),
 322                        nr->status >> 8 & 7,     /* Status Code Type */
 323                        nr->status & 0xff,       /* Status Code */
 324                        nr->status & NVME_SC_MORE ? "MORE " : "",
 325                        nr->status & NVME_SC_DNR  ? "DNR "  : "");
 326                 return;
 327         }
 328
 329         pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
 330                            dev_name(nr->ctrl->device),
 331                            nvme_get_admin_opcode_str(nr->cmd->common.opcode),
 332                            nr->cmd->common.opcode,
 333                            nvme_get_error_status_str(nr->status),
 334                            nr->status >> 8 & 7, /* Status Code Type */
 335                            nr->status & 0xff,   /* Status Code */
 336                            nr->status & NVME_SC_MORE ? "MORE " : "",
 337                            nr->status & NVME_SC_DNR  ? "DNR "  : "");
 338 }
 339
 340 enum nvme_disposition {
 341         COMPLETE,
 342         RETRY,
 343         FAILOVER,
 344         AUTHENTICATE,
 345 };
 346
 347 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 348 {
 349         if (likely(nvme_req(req)->status == 0))
 350                 return COMPLETE;
 351
 352         if ((nvme_req(req)->status & 0x7ff) == NVME_SC_AUTH_REQUIRED)
 353                 return AUTHENTICATE;
 354
 355         if (blk_noretry_request(req) ||
 356             (nvme_req(req)->status & NVME_SC_DNR) ||
 357             nvme_req(req)->retries >= nvme_max_retries)
 358                 return COMPLETE;
 359
 360         if (req->cmd_flags & REQ_NVME_MPATH) {
 361                 if (nvme_is_path_error(nvme_req(req)->status) ||
 362                     blk_queue_dying(req->q))
 363                         return FAILOVER;
 364         } else {
 365                 if (blk_queue_dying(req->q))
 366                         return COMPLETE;
 367         }
 368
 369         return RETRY;
 370 }
 371
 372 static inline void nvme_end_req_zoned(struct request *req)
 373 {
 374         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 375             req_op(req) == REQ_OP_ZONE_APPEND)
 376                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 377                         le64_to_cpu(nvme_req(req)->result.u64));
 378 }
 379
 380 static inline void nvme_end_req(struct request *req)
 381 {
 382         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 383
 384         if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET)))
 385                 nvme_log_error(req);
 386         nvme_end_req_zoned(req);
 387         nvme_trace_bio_complete(req);
 388         if (req->cmd_flags & REQ_NVME_MPATH)
 389                 nvme_mpath_end_request(req);
 390         blk_mq_end_request(req, status);
 391 }
 392
 393 void nvme_complete_rq(struct request *req)
 394 {
 395         struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 396
 397         trace_nvme_complete_rq(req);
 398         nvme_cleanup_cmd(req);
 399
 400         if (ctrl->kas)
 401                 ctrl->comp_seen = true;
 402
 403         switch (nvme_decide_disposition(req)) {
 404         case COMPLETE:
 405                 nvme_end_req(req);
 406                 return;
 407         case RETRY:
 408                 nvme_retry_req(req);
 409                 return;
 410         case FAILOVER:
 411                 nvme_failover_req(req);
 412                 return;
 413         case AUTHENTICATE:
 414 #ifdef CONFIG_NVME_AUTH
 415                 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
 416                 nvme_retry_req(req);
 417 #else
 418                 nvme_end_req(req);
 419 #endif
 420                 return;
 421         }
 422 }
 423 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 424
 425 void nvme_complete_batch_req(struct request *req)
 426 {
 427         trace_nvme_complete_rq(req);
 428         nvme_cleanup_cmd(req);
 429         nvme_end_req_zoned(req);
 430 }
 431 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
 432
 433 /*
 434  * Called to unwind from ->queue_rq on a failed command submission so that the
 435  * multipathing code gets called to potentially failover to another path.
 436  * The caller needs to unwind all transport specific resource allocations and
 437  * must return propagate the return value.
 438  */
 439 blk_status_t nvme_host_path_error(struct request *req)
 440 {
 441         nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
 442         blk_mq_set_request_complete(req);
 443         nvme_complete_rq(req);
 444         return BLK_STS_OK;
 445 }
 446 EXPORT_SYMBOL_GPL(nvme_host_path_error);
 447
 448 bool nvme_cancel_request(struct request *req, void *data)
 449 {
 450         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 451                                 "Cancelling I/O %d", req->tag);
 452
 453         /* don't abort one completed or idle request */
 454         if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
 455                 return true;
 456
 457         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 458         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 459         blk_mq_complete_request(req);
 460         return true;
 461 }
 462 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 463
 464 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
 465 {
 466         if (ctrl->tagset) {
 467                 blk_mq_tagset_busy_iter(ctrl->tagset,
 468                                 nvme_cancel_request, ctrl);
 469                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
 470         }
 471 }
 472 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
 473
 474 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 475 {
 476         if (ctrl->admin_tagset) {
 477                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
 478                                 nvme_cancel_request, ctrl);
 479                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
 480         }
 481 }
 482 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 483
 484 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 485                 enum nvme_ctrl_state new_state)
 486 {
 487         enum nvme_ctrl_state old_state;
 488         unsigned long flags;
 489         bool changed = false;
 490
 491         spin_lock_irqsave(&ctrl->lock, flags);
 492
 493         old_state = ctrl->state;
 494         switch (new_state) {
 495         case NVME_CTRL_LIVE:
 496                 switch (old_state) {
 497                 case NVME_CTRL_NEW:
 498                 case NVME_CTRL_RESETTING:
 499                 case NVME_CTRL_CONNECTING:
 500                         changed = true;
 501                         fallthrough;
 502                 default:
 503                         break;
 504                 }
 505                 break;
 506         case NVME_CTRL_RESETTING:
 507                 switch (old_state) {
 508                 case NVME_CTRL_NEW:
 509                 case NVME_CTRL_LIVE:
 510                         changed = true;
 511                         fallthrough;
 512                 default:
 513                         break;
 514                 }
 515                 break;
 516         case NVME_CTRL_CONNECTING:
 517                 switch (old_state) {
 518                 case NVME_CTRL_NEW:
 519                 case NVME_CTRL_RESETTING:
 520                         changed = true;
 521                         fallthrough;
 522                 default:
 523                         break;
 524                 }
 525                 break;
 526         case NVME_CTRL_DELETING:
 527                 switch (old_state) {
 528                 case NVME_CTRL_LIVE:
 529                 case NVME_CTRL_RESETTING:
 530                 case NVME_CTRL_CONNECTING:
 531                         changed = true;
 532                         fallthrough;
 533                 default:
 534                         break;
 535                 }
 536                 break;
 537         case NVME_CTRL_DELETING_NOIO:
 538                 switch (old_state) {
 539                 case NVME_CTRL_DELETING:
 540                 case NVME_CTRL_DEAD:
 541                         changed = true;
 542                         fallthrough;
 543                 default:
 544                         break;
 545                 }
 546                 break;
 547         case NVME_CTRL_DEAD:
 548                 switch (old_state) {
 549                 case NVME_CTRL_DELETING:
 550                         changed = true;
 551                         fallthrough;
 552                 default:
 553                         break;
 554                 }
 555                 break;
 556         default:
 557                 break;
 558         }
 559
 560         if (changed) {
 561                 ctrl->state = new_state;
 562                 wake_up_all(&ctrl->state_wq);
 563         }
 564
 565         spin_unlock_irqrestore(&ctrl->lock, flags);
 566         if (!changed)
 567                 return false;
 568
 569         if (ctrl->state == NVME_CTRL_LIVE) {
 570                 if (old_state == NVME_CTRL_CONNECTING)
 571                         nvme_stop_failfast_work(ctrl);
 572                 nvme_kick_requeue_lists(ctrl);
 573         } else if (ctrl->state == NVME_CTRL_CONNECTING &&
 574                 old_state == NVME_CTRL_RESETTING) {
 575                 nvme_start_failfast_work(ctrl);
 576         }
 577         return changed;
 578 }
 579 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 580
 581 /*
 582  * Returns true for sink states that can't ever transition back to live.
 583  */
 584 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 585 {
 586         switch (ctrl->state) {
 587         case NVME_CTRL_NEW:
 588         case NVME_CTRL_LIVE:
 589         case NVME_CTRL_RESETTING:
 590         case NVME_CTRL_CONNECTING:
 591                 return false;
 592         case NVME_CTRL_DELETING:
 593         case NVME_CTRL_DELETING_NOIO:
 594         case NVME_CTRL_DEAD:
 595                 return true;
 596         default:
 597                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 598                 return true;
 599         }
 600 }
 601
 602 /*
 603  * Waits for the controller state to be resetting, or returns false if it is
 604  * not possible to ever transition to that state.
 605  */
 606 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 607 {
 608         wait_event(ctrl->state_wq,
 609                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 610                    nvme_state_terminal(ctrl));
 611         return ctrl->state == NVME_CTRL_RESETTING;
 612 }
 613 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 614
 615 static void nvme_free_ns_head(struct kref *ref)
 616 {
 617         struct nvme_ns_head *head =
 618                 container_of(ref, struct nvme_ns_head, ref);
 619
 620         nvme_mpath_remove_disk(head);
 621         ida_free(&head->subsys->ns_ida, head->instance);
 622         cleanup_srcu_struct(&head->srcu);
 623         nvme_put_subsystem(head->subsys);
 624         kfree(head);
 625 }
 626
 627 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
 628 {
 629         return kref_get_unless_zero(&head->ref);
 630 }
 631
 632 void nvme_put_ns_head(struct nvme_ns_head *head)
 633 {
 634         kref_put(&head->ref, nvme_free_ns_head);
 635 }
 636
 637 static void nvme_free_ns(struct kref *kref)
 638 {
 639         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 640
 641         put_disk(ns->disk);
 642         nvme_put_ns_head(ns->head);
 643         nvme_put_ctrl(ns->ctrl);
 644         kfree(ns);
 645 }
 646
 647 static inline bool nvme_get_ns(struct nvme_ns *ns)
 648 {
 649         return kref_get_unless_zero(&ns->kref);
 650 }
 651
 652 void nvme_put_ns(struct nvme_ns *ns)
 653 {
 654         kref_put(&ns->kref, nvme_free_ns);
 655 }
 656 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 657
 658 static inline void nvme_clear_nvme_request(struct request *req)
 659 {
 660         nvme_req(req)->status = 0;
 661         nvme_req(req)->retries = 0;
 662         nvme_req(req)->flags = 0;
 663         req->rq_flags |= RQF_DONTPREP;
 664 }
 665
 666 /* initialize a passthrough request */
 667 void nvme_init_request(struct request *req, struct nvme_command *cmd)
 668 {
 669         if (req->q->queuedata)
 670                 req->timeout = NVME_IO_TIMEOUT;
 671         else /* no queuedata implies admin queue */
 672                 req->timeout = NVME_ADMIN_TIMEOUT;
 673
 674         /* passthru commands should let the driver set the SGL flags */
 675         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 676
 677         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 678         if (req->mq_hctx->type == HCTX_TYPE_POLL)
 679                 req->cmd_flags |= REQ_POLLED;
 680         nvme_clear_nvme_request(req);
 681         req->rq_flags |= RQF_QUIET;
 682         memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
 683 }
 684 EXPORT_SYMBOL_GPL(nvme_init_request);
 685
 686 /*
 687  * For something we're not in a state to send to the device the default action
 688  * is to busy it and retry it after the controller state is recovered.  However,
 689  * if the controller is deleting or if anything is marked for failfast or
 690  * nvme multipath it is immediately failed.
 691  *
 692  * Note: commands used to initialize the controller will be marked for failfast.
 693  * Note: nvme cli/ioctl commands are marked for failfast.
 694  */
 695 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
 696                 struct request *rq)
 697 {
 698         if (ctrl->state != NVME_CTRL_DELETING_NOIO &&
 699             ctrl->state != NVME_CTRL_DELETING &&
 700             ctrl->state != NVME_CTRL_DEAD &&
 701             !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
 702             !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
 703                 return BLK_STS_RESOURCE;
 704         return nvme_host_path_error(rq);
 705 }
 706 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
 707
 708 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
 709                 bool queue_live)
 710 {
 711         struct nvme_request *req = nvme_req(rq);
 712
 713         /*
 714          * currently we have a problem sending passthru commands
 715          * on the admin_q if the controller is not LIVE because we can't
 716          * make sure that they are going out after the admin connect,
 717          * controller enable and/or other commands in the initialization
 718          * sequence. until the controller will be LIVE, fail with
 719          * BLK_STS_RESOURCE so that they will be rescheduled.
 720          */
 721         if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
 722                 return false;
 723
 724         if (ctrl->ops->flags & NVME_F_FABRICS) {
 725                 /*
 726                  * Only allow commands on a live queue, except for the connect
 727                  * command, which is require to set the queue live in the
 728                  * appropinquate states.
 729                  */
 730                 switch (ctrl->state) {
 731                 case NVME_CTRL_CONNECTING:
 732                         if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
 733                             (req->cmd->fabrics.fctype == nvme_fabrics_type_connect ||
 734                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_send ||
 735                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive))
 736                                 return true;
 737                         break;
 738                 default:
 739                         break;
 740                 case NVME_CTRL_DEAD:
 741                         return false;
 742                 }
 743         }
 744
 745         return queue_live;
 746 }
 747 EXPORT_SYMBOL_GPL(__nvme_check_ready);
 748
 749 static inline void nvme_setup_flush(struct nvme_ns *ns,
 750                 struct nvme_command *cmnd)
 751 {
 752         memset(cmnd, 0, sizeof(*cmnd));
 753         cmnd->common.opcode = nvme_cmd_flush;
 754         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 755 }
 756
 757 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 758                 struct nvme_command *cmnd)
 759 {
 760         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 761         struct nvme_dsm_range *range;
 762         struct bio *bio;
 763
 764         /*
 765          * Some devices do not consider the DSM 'Number of Ranges' field when
 766          * determining how much data to DMA. Always allocate memory for maximum
 767          * number of segments to prevent device reading beyond end of buffer.
 768          */
 769         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 770
 771         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 772         if (!range) {
 773                 /*
 774                  * If we fail allocation our range, fallback to the controller
 775                  * discard page. If that's also busy, it's safe to return
 776                  * busy, as we know we can make progress once that's freed.
 777                  */
 778                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 779                         return BLK_STS_RESOURCE;
 780
 781                 range = page_address(ns->ctrl->discard_page);
 782         }
 783
 784         __rq_for_each_bio(bio, req) {
 785                 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 786                 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 787
 788                 if (n < segments) {
 789                         range[n].cattr = cpu_to_le32(0);
 790                         range[n].nlb = cpu_to_le32(nlb);
 791                         range[n].slba = cpu_to_le64(slba);
 792                 }
 793                 n++;
 794         }
 795
 796         if (WARN_ON_ONCE(n != segments)) {
 797                 if (virt_to_page(range) == ns->ctrl->discard_page)
 798                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 799                 else
 800                         kfree(range);
 801                 return BLK_STS_IOERR;
 802         }
 803
 804         memset(cmnd, 0, sizeof(*cmnd));
 805         cmnd->dsm.opcode = nvme_cmd_dsm;
 806         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 807         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 808         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 809
 810         bvec_set_virt(&req->special_vec, range, alloc_size);
 811         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 812
 813         return BLK_STS_OK;
 814 }
 815
 816 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
 817                               struct request *req)
 818 {
 819         u32 upper, lower;
 820         u64 ref48;
 821
 822         /* both rw and write zeroes share the same reftag format */
 823         switch (ns->guard_type) {
 824         case NVME_NVM_NS_16B_GUARD:
 825                 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 826                 break;
 827         case NVME_NVM_NS_64B_GUARD:
 828                 ref48 = ext_pi_ref_tag(req);
 829                 lower = lower_32_bits(ref48);
 830                 upper = upper_32_bits(ref48);
 831
 832                 cmnd->rw.reftag = cpu_to_le32(lower);
 833                 cmnd->rw.cdw3 = cpu_to_le32(upper);
 834                 break;
 835         default:
 836                 break;
 837         }
 838 }
 839
 840 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 841                 struct request *req, struct nvme_command *cmnd)
 842 {
 843         memset(cmnd, 0, sizeof(*cmnd));
 844
 845         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 846                 return nvme_setup_discard(ns, req, cmnd);
 847
 848         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 849         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 850         cmnd->write_zeroes.slba =
 851                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 852         cmnd->write_zeroes.length =
 853                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 854
 855         if (!(req->cmd_flags & REQ_NOUNMAP) && (ns->features & NVME_NS_DEAC))
 856                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC);
 857
 858         if (nvme_ns_has_pi(ns)) {
 859                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT);
 860
 861                 switch (ns->pi_type) {
 862                 case NVME_NS_DPS_PI_TYPE1:
 863                 case NVME_NS_DPS_PI_TYPE2:
 864                         nvme_set_ref_tag(ns, cmnd, req);
 865                         break;
 866                 }
 867         }
 868
 869         return BLK_STS_OK;
 870 }
 871
 872 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 873                 struct request *req, struct nvme_command *cmnd,
 874                 enum nvme_opcode op)
 875 {
 876         u16 control = 0;
 877         u32 dsmgmt = 0;
 878
 879         if (req->cmd_flags & REQ_FUA)
 880                 control |= NVME_RW_FUA;
 881         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 882                 control |= NVME_RW_LR;
 883
 884         if (req->cmd_flags & REQ_RAHEAD)
 885                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 886
 887         cmnd->rw.opcode = op;
 888         cmnd->rw.flags = 0;
 889         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 890         cmnd->rw.cdw2 = 0;
 891         cmnd->rw.cdw3 = 0;
 892         cmnd->rw.metadata = 0;
 893         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 894         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 895         cmnd->rw.reftag = 0;
 896         cmnd->rw.apptag = 0;
 897         cmnd->rw.appmask = 0;
 898
 899         if (ns->ms) {
 900                 /*
 901                  * If formated with metadata, the block layer always provides a
 902                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 903                  * we enable the PRACT bit for protection information or set the
 904                  * namespace capacity to zero to prevent any I/O.
 905                  */
 906                 if (!blk_integrity_rq(req)) {
 907                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 908                                 return BLK_STS_NOTSUPP;
 909                         control |= NVME_RW_PRINFO_PRACT;
 910                 }
 911
 912                 switch (ns->pi_type) {
 913                 case NVME_NS_DPS_PI_TYPE3:
 914                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 915                         break;
 916                 case NVME_NS_DPS_PI_TYPE1:
 917                 case NVME_NS_DPS_PI_TYPE2:
 918                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 919                                         NVME_RW_PRINFO_PRCHK_REF;
 920                         if (op == nvme_cmd_zone_append)
 921                                 control |= NVME_RW_APPEND_PIREMAP;
 922                         nvme_set_ref_tag(ns, cmnd, req);
 923                         break;
 924                 }
 925         }
 926
 927         cmnd->rw.control = cpu_to_le16(control);
 928         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 929         return 0;
 930 }
 931
 932 void nvme_cleanup_cmd(struct request *req)
 933 {
 934         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 935                 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 936
 937                 if (req->special_vec.bv_page == ctrl->discard_page)
 938                         clear_bit_unlock(0, &ctrl->discard_page_busy);
 939                 else
 940                         kfree(bvec_virt(&req->special_vec));
 941         }
 942 }
 943 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 944
 945 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
 946 {
 947         struct nvme_command *cmd = nvme_req(req)->cmd;
 948         blk_status_t ret = BLK_STS_OK;
 949
 950         if (!(req->rq_flags & RQF_DONTPREP))
 951                 nvme_clear_nvme_request(req);
 952
 953         switch (req_op(req)) {
 954         case REQ_OP_DRV_IN:
 955         case REQ_OP_DRV_OUT:
 956                 /* these are setup prior to execution in nvme_init_request() */
 957                 break;
 958         case REQ_OP_FLUSH:
 959                 nvme_setup_flush(ns, cmd);
 960                 break;
 961         case REQ_OP_ZONE_RESET_ALL:
 962         case REQ_OP_ZONE_RESET:
 963                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 964                 break;
 965         case REQ_OP_ZONE_OPEN:
 966                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 967                 break;
 968         case REQ_OP_ZONE_CLOSE:
 969                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 970                 break;
 971         case REQ_OP_ZONE_FINISH:
 972                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 973                 break;
 974         case REQ_OP_WRITE_ZEROES:
 975                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 976                 break;
 977         case REQ_OP_DISCARD:
 978                 ret = nvme_setup_discard(ns, req, cmd);
 979                 break;
 980         case REQ_OP_READ:
 981                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 982                 break;
 983         case REQ_OP_WRITE:
 984                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 985                 break;
 986         case REQ_OP_ZONE_APPEND:
 987                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 988                 break;
 989         default:
 990                 WARN_ON_ONCE(1);
 991                 return BLK_STS_IOERR;
 992         }
 993
 994         cmd->common.command_id = nvme_cid(req);
 995         trace_nvme_setup_cmd(req, cmd);
 996         return ret;
 997 }
 998 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
 999
1000 /*
1001  * Return values:
1002  * 0:  success
1003  * >0: nvme controller's cqe status response
1004  * <0: kernel error in lieu of controller response
1005  */
1006 int nvme_execute_rq(struct request *rq, bool at_head)
1007 {
1008         blk_status_t status;
1009
1010         status = blk_execute_rq(rq, at_head);
1011         if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
1012                 return -EINTR;
1013         if (nvme_req(rq)->status)
1014                 return nvme_req(rq)->status;
1015         return blk_status_to_errno(status);
1016 }
1017 EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);
1018
1019 /*
1020  * Returns 0 on success.  If the result is negative, it's a Linux error code;
1021  * if the result is positive, it's an NVM Express status code
1022  */
1023 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1024                 union nvme_result *result, void *buffer, unsigned bufflen,
1025                 int qid, int at_head, blk_mq_req_flags_t flags)
1026 {
1027         struct request *req;
1028         int ret;
1029
1030         if (qid == NVME_QID_ANY)
1031                 req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
1032         else
1033                 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
1034                                                 qid - 1);
1035
1036         if (IS_ERR(req))
1037                 return PTR_ERR(req);
1038         nvme_init_request(req, cmd);
1039
1040         if (buffer && bufflen) {
1041                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1042                 if (ret)
1043                         goto out;
1044         }
1045
1046         ret = nvme_execute_rq(req, at_head);
1047         if (result && ret >= 0)
1048                 *result = nvme_req(req)->result;
1049  out:
1050         blk_mq_free_request(req);
1051         return ret;
1052 }
1053 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1054
1055 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1056                 void *buffer, unsigned bufflen)
1057 {
1058         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
1059                         NVME_QID_ANY, 0, 0);
1060 }
1061 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1062
1063 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1064 {
1065         u32 effects = 0;
1066
1067         if (ns) {
1068                 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1069                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1070                         dev_warn_once(ctrl->device,
1071                                 "IO command:%02x has unusual effects:%08x\n",
1072                                 opcode, effects);
1073
1074                 /*
1075                  * NVME_CMD_EFFECTS_CSE_MASK causes a freeze all I/O queues,
1076                  * which would deadlock when done on an I/O command.  Note that
1077                  * We already warn about an unusual effect above.
1078                  */
1079                 effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1080         } else {
1081                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1082         }
1083
1084         return effects;
1085 }
1086 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1087
1088 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1089 {
1090         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1091
1092         /*
1093          * For simplicity, IO to all namespaces is quiesced even if the command
1094          * effects say only one namespace is affected.
1095          */
1096         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1097                 mutex_lock(&ctrl->scan_lock);
1098                 mutex_lock(&ctrl->subsys->lock);
1099                 nvme_mpath_start_freeze(ctrl->subsys);
1100                 nvme_mpath_wait_freeze(ctrl->subsys);
1101                 nvme_start_freeze(ctrl);
1102                 nvme_wait_freeze(ctrl);
1103         }
1104         return effects;
1105 }
1106 EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, NVME_TARGET_PASSTHRU);
1107
1108 void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,
1109                        struct nvme_command *cmd, int status)
1110 {
1111         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1112                 nvme_unfreeze(ctrl);
1113                 nvme_mpath_unfreeze(ctrl->subsys);
1114                 mutex_unlock(&ctrl->subsys->lock);
1115                 mutex_unlock(&ctrl->scan_lock);
1116         }
1117         if (effects & NVME_CMD_EFFECTS_CCC) {
1118                 dev_info(ctrl->device,
1119 "controller capabilities changed, reset may be required to take effect.\n");
1120         }
1121         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1122                 nvme_queue_scan(ctrl);
1123                 flush_work(&ctrl->scan_work);
1124         }
1125
1126         switch (cmd->common.opcode) {
1127         case nvme_admin_set_features:
1128                 switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1129                 case NVME_FEAT_KATO:
1130                         /*
1131                          * Keep alive commands interval on the host should be
1132                          * updated when KATO is modified by Set Features
1133                          * commands.
1134                          */
1135                         if (!status)
1136                                 nvme_update_keep_alive(ctrl, cmd);
1137                         break;
1138                 default:
1139                         break;
1140                 }
1141                 break;
1142         default:
1143                 break;
1144         }
1145 }
1146 EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU);
1147
1148 /*
1149  * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1150  *
1151  *   The host should send Keep Alive commands at half of the Keep Alive Timeout
1152  *   accounting for transport roundtrip times [..].
1153  */
1154 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1155 {
1156         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2);
1157 }
1158
1159 static enum rq_end_io_ret nvme_keep_alive_end_io(struct request *rq,
1160                                                  blk_status_t status)
1161 {
1162         struct nvme_ctrl *ctrl = rq->end_io_data;
1163         unsigned long flags;
1164         bool startka = false;
1165
1166         blk_mq_free_request(rq);
1167
1168         if (status) {
1169                 dev_err(ctrl->device,
1170                         "failed nvme_keep_alive_end_io error=%d\n",
1171                                 status);
1172                 return RQ_END_IO_NONE;
1173         }
1174
1175         ctrl->comp_seen = false;
1176         spin_lock_irqsave(&ctrl->lock, flags);
1177         if (ctrl->state == NVME_CTRL_LIVE ||
1178             ctrl->state == NVME_CTRL_CONNECTING)
1179                 startka = true;
1180         spin_unlock_irqrestore(&ctrl->lock, flags);
1181         if (startka)
1182                 nvme_queue_keep_alive_work(ctrl);
1183         return RQ_END_IO_NONE;
1184 }
1185
1186 static void nvme_keep_alive_work(struct work_struct *work)
1187 {
1188         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1189                         struct nvme_ctrl, ka_work);
1190         bool comp_seen = ctrl->comp_seen;
1191         struct request *rq;
1192
1193         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1194                 dev_dbg(ctrl->device,
1195                         "reschedule traffic based keep-alive timer\n");
1196                 ctrl->comp_seen = false;
1197                 nvme_queue_keep_alive_work(ctrl);
1198                 return;
1199         }
1200
1201         rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1202                                   BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1203         if (IS_ERR(rq)) {
1204                 /* allocation failure, reset the controller */
1205                 dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1206                 nvme_reset_ctrl(ctrl);
1207                 return;
1208         }
1209         nvme_init_request(rq, &ctrl->ka_cmd);
1210
1211         rq->timeout = ctrl->kato * HZ;
1212         rq->end_io = nvme_keep_alive_end_io;
1213         rq->end_io_data = ctrl;
1214         blk_execute_rq_nowait(rq, false);
1215 }
1216
1217 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1218 {
1219         if (unlikely(ctrl->kato == 0))
1220                 return;
1221
1222         nvme_queue_keep_alive_work(ctrl);
1223 }
1224
1225 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1226 {
1227         if (unlikely(ctrl->kato == 0))
1228                 return;
1229
1230         cancel_delayed_work_sync(&ctrl->ka_work);
1231 }
1232 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1233
1234 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1235                                    struct nvme_command *cmd)
1236 {
1237         unsigned int new_kato =
1238                 DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1239
1240         dev_info(ctrl->device,
1241                  "keep alive interval updated from %u ms to %u ms\n",
1242                  ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1243
1244         nvme_stop_keep_alive(ctrl);
1245         ctrl->kato = new_kato;
1246         nvme_start_keep_alive(ctrl);
1247 }
1248
1249 /*
1250  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1251  * flag, thus sending any new CNS opcodes has a big chance of not working.
1252  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1253  * (but not for any later version).
1254  */
1255 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1256 {
1257         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1258                 return ctrl->vs < NVME_VS(1, 2, 0);
1259         return ctrl->vs < NVME_VS(1, 1, 0);
1260 }
1261
1262 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1263 {
1264         struct nvme_command c = { };
1265         int error;
1266
1267         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1268         c.identify.opcode = nvme_admin_identify;
1269         c.identify.cns = NVME_ID_CNS_CTRL;
1270
1271         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1272         if (!*id)
1273                 return -ENOMEM;
1274
1275         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1276                         sizeof(struct nvme_id_ctrl));
1277         if (error)
1278                 kfree(*id);
1279         return error;
1280 }
1281
1282 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1283                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1284 {
1285         const char *warn_str = "ctrl returned bogus length:";
1286         void *data = cur;
1287
1288         switch (cur->nidt) {
1289         case NVME_NIDT_EUI64:
1290                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1291                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1292                                  warn_str, cur->nidl);
1293                         return -1;
1294                 }
1295                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1296                         return NVME_NIDT_EUI64_LEN;
1297                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1298                 return NVME_NIDT_EUI64_LEN;
1299         case NVME_NIDT_NGUID:
1300                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1301                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1302                                  warn_str, cur->nidl);
1303                         return -1;
1304                 }
1305                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1306                         return NVME_NIDT_NGUID_LEN;
1307                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1308                 return NVME_NIDT_NGUID_LEN;
1309         case NVME_NIDT_UUID:
1310                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1311                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1312                                  warn_str, cur->nidl);
1313                         return -1;
1314                 }
1315                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1316                         return NVME_NIDT_UUID_LEN;
1317                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1318                 return NVME_NIDT_UUID_LEN;
1319         case NVME_NIDT_CSI:
1320                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1321                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1322                                  warn_str, cur->nidl);
1323                         return -1;
1324                 }
1325                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1326                 *csi_seen = true;
1327                 return NVME_NIDT_CSI_LEN;
1328         default:
1329                 /* Skip unknown types */
1330                 return cur->nidl;
1331         }
1332 }
1333
1334 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,
1335                 struct nvme_ns_info *info)
1336 {
1337         struct nvme_command c = { };
1338         bool csi_seen = false;
1339         int status, pos, len;
1340         void *data;
1341
1342         if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1343                 return 0;
1344         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1345                 return 0;
1346
1347         c.identify.opcode = nvme_admin_identify;
1348         c.identify.nsid = cpu_to_le32(info->nsid);
1349         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1350
1351         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1352         if (!data)
1353                 return -ENOMEM;
1354
1355         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1356                                       NVME_IDENTIFY_DATA_SIZE);
1357         if (status) {
1358                 dev_warn(ctrl->device,
1359                         "Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1360                         info->nsid, status);
1361                 goto free_data;
1362         }
1363
1364         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1365                 struct nvme_ns_id_desc *cur = data + pos;
1366
1367                 if (cur->nidl == 0)
1368                         break;
1369
1370                 len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen);
1371                 if (len < 0)
1372                         break;
1373
1374                 len += sizeof(*cur);
1375         }
1376
1377         if (nvme_multi_css(ctrl) && !csi_seen) {
1378                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1379                          info->nsid);
1380                 status = -EINVAL;
1381         }
1382
1383 free_data:
1384         kfree(data);
1385         return status;
1386 }
1387
1388 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1389                         struct nvme_id_ns **id)
1390 {
1391         struct nvme_command c = { };
1392         int error;
1393
1394         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1395         c.identify.opcode = nvme_admin_identify;
1396         c.identify.nsid = cpu_to_le32(nsid);
1397         c.identify.cns = NVME_ID_CNS_NS;
1398
1399         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1400         if (!*id)
1401                 return -ENOMEM;
1402
1403         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1404         if (error) {
1405                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1406                 kfree(*id);
1407         }
1408         return error;
1409 }
1410
1411 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,
1412                 struct nvme_ns_info *info)
1413 {
1414         struct nvme_ns_ids *ids = &info->ids;
1415         struct nvme_id_ns *id;
1416         int ret;
1417
1418         ret = nvme_identify_ns(ctrl, info->nsid, &id);
1419         if (ret)
1420                 return ret;
1421
1422         if (id->ncap == 0) {
1423                 /* namespace not allocated or attached */
1424                 info->is_removed = true;
1425                 return -ENODEV;
1426         }
1427
1428         info->anagrpid = id->anagrpid;
1429         info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1430         info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1431         info->is_ready = true;
1432         if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1433                 dev_info(ctrl->device,
1434                          "Ignoring bogus Namespace Identifiers\n");
1435         } else {
1436                 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1437                     !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1438                         memcpy(ids->eui64, id->eui64, sizeof(ids->eui64));
1439                 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1440                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1441                         memcpy(ids->nguid, id->nguid, sizeof(ids->nguid));
1442         }
1443         kfree(id);
1444         return 0;
1445 }
1446
1447 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,
1448                 struct nvme_ns_info *info)
1449 {
1450         struct nvme_id_ns_cs_indep *id;
1451         struct nvme_command c = {
1452                 .identify.opcode        = nvme_admin_identify,
1453                 .identify.nsid          = cpu_to_le32(info->nsid),
1454                 .identify.cns           = NVME_ID_CNS_NS_CS_INDEP,
1455         };
1456         int ret;
1457
1458         id = kmalloc(sizeof(*id), GFP_KERNEL);
1459         if (!id)
1460                 return -ENOMEM;
1461
1462         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1463         if (!ret) {
1464                 info->anagrpid = id->anagrpid;
1465                 info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1466                 info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1467                 info->is_ready = id->nstat & NVME_NSTAT_NRDY;
1468         }
1469         kfree(id);
1470         return ret;
1471 }
1472
1473 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1474                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1475 {
1476         union nvme_result res = { 0 };
1477         struct nvme_command c = { };
1478         int ret;
1479
1480         c.features.opcode = op;
1481         c.features.fid = cpu_to_le32(fid);
1482         c.features.dword11 = cpu_to_le32(dword11);
1483
1484         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1485                         buffer, buflen, NVME_QID_ANY, 0, 0);
1486         if (ret >= 0 && result)
1487                 *result = le32_to_cpu(res.u32);
1488         return ret;
1489 }
1490
1491 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1492                       unsigned int dword11, void *buffer, size_t buflen,
1493                       u32 *result)
1494 {
1495         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1496                              buflen, result);
1497 }
1498 EXPORT_SYMBOL_GPL(nvme_set_features);
1499
1500 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1501                       unsigned int dword11, void *buffer, size_t buflen,
1502                       u32 *result)
1503 {
1504         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1505                              buflen, result);
1506 }
1507 EXPORT_SYMBOL_GPL(nvme_get_features);
1508
1509 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1510 {
1511         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1512         u32 result;
1513         int status, nr_io_queues;
1514
1515         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1516                         &result);
1517         if (status < 0)
1518                 return status;
1519
1520         /*
1521          * Degraded controllers might return an error when setting the queue
1522          * count.  We still want to be able to bring them online and offer
1523          * access to the admin queue, as that might be only way to fix them up.
1524          */
1525         if (status > 0) {
1526                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1527                 *count = 0;
1528         } else {
1529                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1530                 *count = min(*count, nr_io_queues);
1531         }
1532
1533         return 0;
1534 }
1535 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1536
1537 #define NVME_AEN_SUPPORTED \
1538         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1539          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1540
1541 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1542 {
1543         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1544         int status;
1545
1546         if (!supported_aens)
1547                 return;
1548
1549         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1550                         NULL, 0, &result);
1551         if (status)
1552                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1553                          supported_aens);
1554
1555         queue_work(nvme_wq, &ctrl->async_event_work);
1556 }
1557
1558 static int nvme_ns_open(struct nvme_ns *ns)
1559 {
1560
1561         /* should never be called due to GENHD_FL_HIDDEN */
1562         if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1563                 goto fail;
1564         if (!nvme_get_ns(ns))
1565                 goto fail;
1566         if (!try_module_get(ns->ctrl->ops->module))
1567                 goto fail_put_ns;
1568
1569         return 0;
1570
1571 fail_put_ns:
1572         nvme_put_ns(ns);
1573 fail:
1574         return -ENXIO;
1575 }
1576
1577 static void nvme_ns_release(struct nvme_ns *ns)
1578 {
1579
1580         module_put(ns->ctrl->ops->module);
1581         nvme_put_ns(ns);
1582 }
1583
1584 static int nvme_open(struct block_device *bdev, fmode_t mode)
1585 {
1586         return nvme_ns_open(bdev->bd_disk->private_data);
1587 }
1588
1589 static void nvme_release(struct gendisk *disk, fmode_t mode)
1590 {
1591         nvme_ns_release(disk->private_data);
1592 }
1593
1594 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1595 {
1596         /* some standard values */
1597         geo->heads = 1 << 6;
1598         geo->sectors = 1 << 5;
1599         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1600         return 0;
1601 }
1602
1603 #ifdef CONFIG_BLK_DEV_INTEGRITY
1604 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1605                                 u32 max_integrity_segments)
1606 {
1607         struct blk_integrity integrity = { };
1608
1609         switch (ns->pi_type) {
1610         case NVME_NS_DPS_PI_TYPE3:
1611                 switch (ns->guard_type) {
1612                 case NVME_NVM_NS_16B_GUARD:
1613                         integrity.profile = &t10_pi_type3_crc;
1614                         integrity.tag_size = sizeof(u16) + sizeof(u32);
1615                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1616                         break;
1617                 case NVME_NVM_NS_64B_GUARD:
1618                         integrity.profile = &ext_pi_type3_crc64;
1619                         integrity.tag_size = sizeof(u16) + 6;
1620                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1621                         break;
1622                 default:
1623                         integrity.profile = NULL;
1624                         break;
1625                 }
1626                 break;
1627         case NVME_NS_DPS_PI_TYPE1:
1628         case NVME_NS_DPS_PI_TYPE2:
1629                 switch (ns->guard_type) {
1630                 case NVME_NVM_NS_16B_GUARD:
1631                         integrity.profile = &t10_pi_type1_crc;
1632                         integrity.tag_size = sizeof(u16);
1633                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1634                         break;
1635                 case NVME_NVM_NS_64B_GUARD:
1636                         integrity.profile = &ext_pi_type1_crc64;
1637                         integrity.tag_size = sizeof(u16);
1638                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1639                         break;
1640                 default:
1641                         integrity.profile = NULL;
1642                         break;
1643                 }
1644                 break;
1645         default:
1646                 integrity.profile = NULL;
1647                 break;
1648         }
1649
1650         integrity.tuple_size = ns->ms;
1651         blk_integrity_register(disk, &integrity);
1652         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1653 }
1654 #else
1655 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1656                                 u32 max_integrity_segments)
1657 {
1658 }
1659 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1660
1661 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1662 {
1663         struct nvme_ctrl *ctrl = ns->ctrl;
1664         struct request_queue *queue = disk->queue;
1665         u32 size = queue_logical_block_size(queue);
1666
1667         if (ctrl->max_discard_sectors == 0) {
1668                 blk_queue_max_discard_sectors(queue, 0);
1669                 return;
1670         }
1671
1672         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1673                         NVME_DSM_MAX_RANGES);
1674
1675         queue->limits.discard_granularity = size;
1676
1677         /* If discard is already enabled, don't reset queue limits */
1678         if (queue->limits.max_discard_sectors)
1679                 return;
1680
1681         if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns, UINT_MAX))
1682                 ctrl->max_discard_sectors = nvme_lba_to_sect(ns, ctrl->dmrsl);
1683
1684         blk_queue_max_discard_sectors(queue, ctrl->max_discard_sectors);
1685         blk_queue_max_discard_segments(queue, ctrl->max_discard_segments);
1686
1687         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1688                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1689 }
1690
1691 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1692 {
1693         return uuid_equal(&a->uuid, &b->uuid) &&
1694                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1695                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1696                 a->csi == b->csi;
1697 }
1698
1699 static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id)
1700 {
1701         bool first = id->dps & NVME_NS_DPS_PI_FIRST;
1702         unsigned lbaf = nvme_lbaf_index(id->flbas);
1703         struct nvme_ctrl *ctrl = ns->ctrl;
1704         struct nvme_command c = { };
1705         struct nvme_id_ns_nvm *nvm;
1706         int ret = 0;
1707         u32 elbaf;
1708
1709         ns->pi_size = 0;
1710         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1711         if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1712                 ns->pi_size = sizeof(struct t10_pi_tuple);
1713                 ns->guard_type = NVME_NVM_NS_16B_GUARD;
1714                 goto set_pi;
1715         }
1716
1717         nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1718         if (!nvm)
1719                 return -ENOMEM;
1720
1721         c.identify.opcode = nvme_admin_identify;
1722         c.identify.nsid = cpu_to_le32(ns->head->ns_id);
1723         c.identify.cns = NVME_ID_CNS_CS_NS;
1724         c.identify.csi = NVME_CSI_NVM;
1725
1726         ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm));
1727         if (ret)
1728                 goto free_data;
1729
1730         elbaf = le32_to_cpu(nvm->elbaf[lbaf]);
1731
1732         /* no support for storage tag formats right now */
1733         if (nvme_elbaf_sts(elbaf))
1734                 goto free_data;
1735
1736         ns->guard_type = nvme_elbaf_guard_type(elbaf);
1737         switch (ns->guard_type) {
1738         case NVME_NVM_NS_64B_GUARD:
1739                 ns->pi_size = sizeof(struct crc64_pi_tuple);
1740                 break;
1741         case NVME_NVM_NS_16B_GUARD:
1742                 ns->pi_size = sizeof(struct t10_pi_tuple);
1743                 break;
1744         default:
1745                 break;
1746         }
1747
1748 free_data:
1749         kfree(nvm);
1750 set_pi:
1751         if (ns->pi_size && (first || ns->ms == ns->pi_size))
1752                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1753         else
1754                 ns->pi_type = 0;
1755
1756         return ret;
1757 }
1758
1759 static void nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1760 {
1761         struct nvme_ctrl *ctrl = ns->ctrl;
1762
1763         if (nvme_init_ms(ns, id))
1764                 return;
1765
1766         ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1767         if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1768                 return;
1769
1770         if (ctrl->ops->flags & NVME_F_FABRICS) {
1771                 /*
1772                  * The NVMe over Fabrics specification only supports metadata as
1773                  * part of the extended data LBA.  We rely on HCA/HBA support to
1774                  * remap the separate metadata buffer from the block layer.
1775                  */
1776                 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1777                         return;
1778
1779                 ns->features |= NVME_NS_EXT_LBAS;
1780
1781                 /*
1782                  * The current fabrics transport drivers support namespace
1783                  * metadata formats only if nvme_ns_has_pi() returns true.
1784                  * Suppress support for all other formats so the namespace will
1785                  * have a 0 capacity and not be usable through the block stack.
1786                  *
1787                  * Note, this check will need to be modified if any drivers
1788                  * gain the ability to use other metadata formats.
1789                  */
1790                 if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns))
1791                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1792         } else {
1793                 /*
1794                  * For PCIe controllers, we can't easily remap the separate
1795                  * metadata buffer from the block layer and thus require a
1796                  * separate metadata buffer for block layer metadata/PI support.
1797                  * We allow extended LBAs for the passthrough interface, though.
1798                  */
1799                 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1800                         ns->features |= NVME_NS_EXT_LBAS;
1801                 else
1802                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1803         }
1804 }
1805
1806 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1807                 struct request_queue *q)
1808 {
1809         bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1810
1811         if (ctrl->max_hw_sectors) {
1812                 u32 max_segments =
1813                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1814
1815                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1816                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1817                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1818         }
1819         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1820         blk_queue_dma_alignment(q, 3);
1821         blk_queue_write_cache(q, vwc, vwc);
1822 }
1823
1824 static void nvme_update_disk_info(struct gendisk *disk,
1825                 struct nvme_ns *ns, struct nvme_id_ns *id)
1826 {
1827         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1828         unsigned short bs = 1 << ns->lba_shift;
1829         u32 atomic_bs, phys_bs, io_opt = 0;
1830
1831         /*
1832          * The block layer can't support LBA sizes larger than the page size
1833          * yet, so catch this early and don't allow block I/O.
1834          */
1835         if (ns->lba_shift > PAGE_SHIFT) {
1836                 capacity = 0;
1837                 bs = (1 << 9);
1838         }
1839
1840         blk_integrity_unregister(disk);
1841
1842         atomic_bs = phys_bs = bs;
1843         if (id->nabo == 0) {
1844                 /*
1845                  * Bit 1 indicates whether NAWUPF is defined for this namespace
1846                  * and whether it should be used instead of AWUPF. If NAWUPF ==
1847                  * 0 then AWUPF must be used instead.
1848                  */
1849                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1850                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1851                 else
1852                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1853         }
1854
1855         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1856                 /* NPWG = Namespace Preferred Write Granularity */
1857                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
1858                 /* NOWS = Namespace Optimal Write Size */
1859                 io_opt = bs * (1 + le16_to_cpu(id->nows));
1860         }
1861
1862         blk_queue_logical_block_size(disk->queue, bs);
1863         /*
1864          * Linux filesystems assume writing a single physical block is
1865          * an atomic operation. Hence limit the physical block size to the
1866          * value of the Atomic Write Unit Power Fail parameter.
1867          */
1868         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1869         blk_queue_io_min(disk->queue, phys_bs);
1870         blk_queue_io_opt(disk->queue, io_opt);
1871
1872         /*
1873          * Register a metadata profile for PI, or the plain non-integrity NVMe
1874          * metadata masquerading as Type 0 if supported, otherwise reject block
1875          * I/O to namespaces with metadata except when the namespace supports
1876          * PI, as it can strip/insert in that case.
1877          */
1878         if (ns->ms) {
1879                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1880                     (ns->features & NVME_NS_METADATA_SUPPORTED))
1881                         nvme_init_integrity(disk, ns,
1882                                             ns->ctrl->max_integrity_segments);
1883                 else if (!nvme_ns_has_pi(ns))
1884                         capacity = 0;
1885         }
1886
1887         set_capacity_and_notify(disk, capacity);
1888
1889         nvme_config_discard(disk, ns);
1890         blk_queue_max_write_zeroes_sectors(disk->queue,
1891                                            ns->ctrl->max_zeroes_sectors);
1892 }
1893
1894 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info)
1895 {
1896         return info->is_readonly || test_bit(NVME_NS_FORCE_RO, &ns->flags);
1897 }
1898
1899 static inline bool nvme_first_scan(struct gendisk *disk)
1900 {
1901         /* nvme_alloc_ns() scans the disk prior to adding it */
1902         return !disk_live(disk);
1903 }
1904
1905 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
1906 {
1907         struct nvme_ctrl *ctrl = ns->ctrl;
1908         u32 iob;
1909
1910         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1911             is_power_of_2(ctrl->max_hw_sectors))
1912                 iob = ctrl->max_hw_sectors;
1913         else
1914                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
1915
1916         if (!iob)
1917                 return;
1918
1919         if (!is_power_of_2(iob)) {
1920                 if (nvme_first_scan(ns->disk))
1921                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
1922                                 ns->disk->disk_name, iob);
1923                 return;
1924         }
1925
1926         if (blk_queue_is_zoned(ns->disk->queue)) {
1927                 if (nvme_first_scan(ns->disk))
1928                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
1929                                 ns->disk->disk_name);
1930                 return;
1931         }
1932
1933         blk_queue_chunk_sectors(ns->queue, iob);
1934 }
1935
1936 static int nvme_update_ns_info_generic(struct nvme_ns *ns,
1937                 struct nvme_ns_info *info)
1938 {
1939         blk_mq_freeze_queue(ns->disk->queue);
1940         nvme_set_queue_limits(ns->ctrl, ns->queue);
1941         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
1942         blk_mq_unfreeze_queue(ns->disk->queue);
1943
1944         if (nvme_ns_head_multipath(ns->head)) {
1945                 blk_mq_freeze_queue(ns->head->disk->queue);
1946                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
1947                 nvme_mpath_revalidate_paths(ns);
1948                 blk_stack_limits(&ns->head->disk->queue->limits,
1949                                  &ns->queue->limits, 0);
1950                 ns->head->disk->flags |= GENHD_FL_HIDDEN;
1951                 blk_mq_unfreeze_queue(ns->head->disk->queue);
1952         }
1953
1954         /* Hide the block-interface for these devices */
1955         ns->disk->flags |= GENHD_FL_HIDDEN;
1956         set_bit(NVME_NS_READY, &ns->flags);
1957
1958         return 0;
1959 }
1960
1961 static int nvme_update_ns_info_block(struct nvme_ns *ns,
1962                 struct nvme_ns_info *info)
1963 {
1964         struct nvme_id_ns *id;
1965         unsigned lbaf;
1966         int ret;
1967
1968         ret = nvme_identify_ns(ns->ctrl, info->nsid, &id);
1969         if (ret)
1970                 return ret;
1971
1972         blk_mq_freeze_queue(ns->disk->queue);
1973         lbaf = nvme_lbaf_index(id->flbas);
1974         ns->lba_shift = id->lbaf[lbaf].ds;
1975         nvme_set_queue_limits(ns->ctrl, ns->queue);
1976
1977         nvme_configure_metadata(ns, id);
1978         nvme_set_chunk_sectors(ns, id);
1979         nvme_update_disk_info(ns->disk, ns, id);
1980
1981         if (ns->head->ids.csi == NVME_CSI_ZNS) {
1982                 ret = nvme_update_zone_info(ns, lbaf);
1983                 if (ret) {
1984                         blk_mq_unfreeze_queue(ns->disk->queue);
1985                         goto out;
1986                 }
1987         }
1988
1989         /*
1990          * Only set the DEAC bit if the device guarantees that reads from
1991          * deallocated data return zeroes.  While the DEAC bit does not
1992          * require that, it must be a no-op if reads from deallocated data
1993          * do not return zeroes.
1994          */
1995         if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3)))
1996                 ns->features |= NVME_NS_DEAC;
1997         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
1998         set_bit(NVME_NS_READY, &ns->flags);
1999         blk_mq_unfreeze_queue(ns->disk->queue);
2000
2001         if (blk_queue_is_zoned(ns->queue)) {
2002                 ret = nvme_revalidate_zones(ns);
2003                 if (ret && !nvme_first_scan(ns->disk))
2004                         goto out;
2005         }
2006
2007         if (nvme_ns_head_multipath(ns->head)) {
2008                 blk_mq_freeze_queue(ns->head->disk->queue);
2009                 nvme_update_disk_info(ns->head->disk, ns, id);
2010                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
2011                 nvme_mpath_revalidate_paths(ns);
2012                 blk_stack_limits(&ns->head->disk->queue->limits,
2013                                  &ns->queue->limits, 0);
2014                 disk_update_readahead(ns->head->disk);
2015                 blk_mq_unfreeze_queue(ns->head->disk->queue);
2016         }
2017
2018         ret = 0;
2019 out:
2020         /*
2021          * If probing fails due an unsupported feature, hide the block device,
2022          * but still allow other access.
2023          */
2024         if (ret == -ENODEV) {
2025                 ns->disk->flags |= GENHD_FL_HIDDEN;
2026                 set_bit(NVME_NS_READY, &ns->flags);
2027                 ret = 0;
2028         }
2029         kfree(id);
2030         return ret;
2031 }
2032
2033 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
2034 {
2035         switch (info->ids.csi) {
2036         case NVME_CSI_ZNS:
2037                 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2038                         dev_info(ns->ctrl->device,
2039         "block device for nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
2040                                 info->nsid);
2041                         return nvme_update_ns_info_generic(ns, info);
2042                 }
2043                 return nvme_update_ns_info_block(ns, info);
2044         case NVME_CSI_NVM:
2045                 return nvme_update_ns_info_block(ns, info);
2046         default:
2047                 dev_info(ns->ctrl->device,
2048                         "block device for nsid %u not supported (csi %u)\n",
2049                         info->nsid, info->ids.csi);
2050                 return nvme_update_ns_info_generic(ns, info);
2051         }
2052 }
2053
2054 static char nvme_pr_type(enum pr_type type)
2055 {
2056         switch (type) {
2057         case PR_WRITE_EXCLUSIVE:
2058                 return 1;
2059         case PR_EXCLUSIVE_ACCESS:
2060                 return 2;
2061         case PR_WRITE_EXCLUSIVE_REG_ONLY:
2062                 return 3;
2063         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2064                 return 4;
2065         case PR_WRITE_EXCLUSIVE_ALL_REGS:
2066                 return 5;
2067         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2068                 return 6;
2069         default:
2070                 return 0;
2071         }
2072 }
2073
2074 static int nvme_send_ns_head_pr_command(struct block_device *bdev,
2075                 struct nvme_command *c, u8 data[16])
2076 {
2077         struct nvme_ns_head *head = bdev->bd_disk->private_data;
2078         int srcu_idx = srcu_read_lock(&head->srcu);
2079         struct nvme_ns *ns = nvme_find_path(head);
2080         int ret = -EWOULDBLOCK;
2081
2082         if (ns) {
2083                 c->common.nsid = cpu_to_le32(ns->head->ns_id);
2084                 ret = nvme_submit_sync_cmd(ns->queue, c, data, 16);
2085         }
2086         srcu_read_unlock(&head->srcu, srcu_idx);
2087         return ret;
2088 }
2089
2090 static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c,
2091                 u8 data[16])
2092 {
2093         c->common.nsid = cpu_to_le32(ns->head->ns_id);
2094         return nvme_submit_sync_cmd(ns->queue, c, data, 16);
2095 }
2096
2097 static int nvme_sc_to_pr_err(int nvme_sc)
2098 {
2099         if (nvme_is_path_error(nvme_sc))
2100                 return PR_STS_PATH_FAILED;
2101
2102         switch (nvme_sc) {
2103         case NVME_SC_SUCCESS:
2104                 return PR_STS_SUCCESS;
2105         case NVME_SC_RESERVATION_CONFLICT:
2106                 return PR_STS_RESERVATION_CONFLICT;
2107         case NVME_SC_ONCS_NOT_SUPPORTED:
2108                 return -EOPNOTSUPP;
2109         case NVME_SC_BAD_ATTRIBUTES:
2110         case NVME_SC_INVALID_OPCODE:
2111         case NVME_SC_INVALID_FIELD:
2112         case NVME_SC_INVALID_NS:
2113                 return -EINVAL;
2114         default:
2115                 return PR_STS_IOERR;
2116         }
2117 }
2118
2119 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2120                                 u64 key, u64 sa_key, u8 op)
2121 {
2122         struct nvme_command c = { };
2123         u8 data[16] = { 0, };
2124         int ret;
2125
2126         put_unaligned_le64(key, &data[0]);
2127         put_unaligned_le64(sa_key, &data[8]);
2128
2129         c.common.opcode = op;
2130         c.common.cdw10 = cpu_to_le32(cdw10);
2131
2132         if (IS_ENABLED(CONFIG_NVME_MULTIPATH) &&
2133             bdev->bd_disk->fops == &nvme_ns_head_ops)
2134                 ret = nvme_send_ns_head_pr_command(bdev, &c, data);
2135         else
2136                 ret = nvme_send_ns_pr_command(bdev->bd_disk->private_data, &c,
2137                                               data);
2138         if (ret < 0)
2139                 return ret;
2140
2141         return nvme_sc_to_pr_err(ret);
2142 }
2143
2144 static int nvme_pr_register(struct block_device *bdev, u64 old,
2145                 u64 new, unsigned flags)
2146 {
2147         u32 cdw10;
2148
2149         if (flags & ~PR_FL_IGNORE_KEY)
2150                 return -EOPNOTSUPP;
2151
2152         cdw10 = old ? 2 : 0;
2153         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2154         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2155         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2156 }
2157
2158 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2159                 enum pr_type type, unsigned flags)
2160 {
2161         u32 cdw10;
2162
2163         if (flags & ~PR_FL_IGNORE_KEY)
2164                 return -EOPNOTSUPP;
2165
2166         cdw10 = nvme_pr_type(type) << 8;
2167         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2168         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2169 }
2170
2171 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2172                 enum pr_type type, bool abort)
2173 {
2174         u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2175
2176         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2177 }
2178
2179 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2180 {
2181         u32 cdw10 = 1 | (key ? 0 : 1 << 3);
2182
2183         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2184 }
2185
2186 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2187 {
2188         u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 0 : 1 << 3);
2189
2190         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2191 }
2192
2193 const struct pr_ops nvme_pr_ops = {
2194         .pr_register    = nvme_pr_register,
2195         .pr_reserve     = nvme_pr_reserve,
2196         .pr_release     = nvme_pr_release,
2197         .pr_preempt     = nvme_pr_preempt,
2198         .pr_clear       = nvme_pr_clear,
2199 };
2200
2201 #ifdef CONFIG_BLK_SED_OPAL
2202 static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2203                 bool send)
2204 {
2205         struct nvme_ctrl *ctrl = data;
2206         struct nvme_command cmd = { };
2207
2208         if (send)
2209                 cmd.common.opcode = nvme_admin_security_send;
2210         else
2211                 cmd.common.opcode = nvme_admin_security_recv;
2212         cmd.common.nsid = 0;
2213         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2214         cmd.common.cdw11 = cpu_to_le32(len);
2215
2216         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2217                         NVME_QID_ANY, 1, 0);
2218 }
2219
2220 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2221 {
2222         if (ctrl->oacs & NVME_CTRL_OACS_SEC_SUPP) {
2223                 if (!ctrl->opal_dev)
2224                         ctrl->opal_dev = init_opal_dev(ctrl, &nvme_sec_submit);
2225                 else if (was_suspended)
2226                         opal_unlock_from_suspend(ctrl->opal_dev);
2227         } else {
2228                 free_opal_dev(ctrl->opal_dev);
2229                 ctrl->opal_dev = NULL;
2230         }
2231 }
2232 #else
2233 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2234 {
2235 }
2236 #endif /* CONFIG_BLK_SED_OPAL */
2237
2238 #ifdef CONFIG_BLK_DEV_ZONED
2239 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2240                 unsigned int nr_zones, report_zones_cb cb, void *data)
2241 {
2242         return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2243                         data);
2244 }
2245 #else
2246 #define nvme_report_zones       NULL
2247 #endif /* CONFIG_BLK_DEV_ZONED */
2248
2249 static const struct block_device_operations nvme_bdev_ops = {
2250         .owner          = THIS_MODULE,
2251         .ioctl          = nvme_ioctl,
2252         .compat_ioctl   = blkdev_compat_ptr_ioctl,
2253         .open           = nvme_open,
2254         .release        = nvme_release,
2255         .getgeo         = nvme_getgeo,
2256         .report_zones   = nvme_report_zones,
2257         .pr_ops         = &nvme_pr_ops,
2258 };
2259
2260 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
2261                 u32 timeout, const char *op)
2262 {
2263         unsigned long timeout_jiffies = jiffies + timeout * HZ;
2264         u32 csts;
2265         int ret;
2266
2267         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2268                 if (csts == ~0)
2269                         return -ENODEV;
2270                 if ((csts & mask) == val)
2271                         break;
2272
2273                 usleep_range(1000, 2000);
2274                 if (fatal_signal_pending(current))
2275                         return -EINTR;
2276                 if (time_after(jiffies, timeout_jiffies)) {
2277                         dev_err(ctrl->device,
2278                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2279                                 op, csts);
2280                         return -ENODEV;
2281                 }
2282         }
2283
2284         return ret;
2285 }
2286
2287 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2288 {
2289         int ret;
2290
2291         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2292         if (shutdown)
2293                 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2294         else
2295                 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2296
2297         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2298         if (ret)
2299                 return ret;
2300
2301         if (shutdown) {
2302                 return nvme_wait_ready(ctrl, NVME_CSTS_SHST_MASK,
2303                                        NVME_CSTS_SHST_CMPLT,
2304                                        ctrl->shutdown_timeout, "shutdown");
2305         }
2306         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2307                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2308         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, 0,
2309                                (NVME_CAP_TIMEOUT(ctrl->cap) + 1) / 2, "reset");
2310 }
2311 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2312
2313 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2314 {
2315         unsigned dev_page_min;
2316         u32 timeout;
2317         int ret;
2318
2319         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2320         if (ret) {
2321                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2322                 return ret;
2323         }
2324         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2325
2326         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2327                 dev_err(ctrl->device,
2328                         "Minimum device page size %u too large for host (%u)\n",
2329                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2330                 return -ENODEV;
2331         }
2332
2333         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2334                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2335         else
2336                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2337
2338         if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2339                 u32 crto;
2340
2341                 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2342                 if (ret) {
2343                         dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2344                                 ret);
2345                         return ret;
2346                 }
2347
2348                 if (ctrl->cap & NVME_CAP_CRMS_CRIMS) {
2349                         ctrl->ctrl_config |= NVME_CC_CRIME;
2350                         timeout = NVME_CRTO_CRIMT(crto);
2351                 } else {
2352                         timeout = NVME_CRTO_CRWMT(crto);
2353                 }
2354         } else {
2355                 timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2356         }
2357
2358         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2359         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2360         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2361         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2362         if (ret)
2363                 return ret;
2364
2365         /* Flush write to device (required if transport is PCI) */
2366         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CC, &ctrl->ctrl_config);
2367         if (ret)
2368                 return ret;
2369
2370         ctrl->ctrl_config |= NVME_CC_ENABLE;
2371         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2372         if (ret)
2373                 return ret;
2374         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, NVME_CSTS_RDY,
2375                                (timeout + 1) / 2, "initialisation");
2376 }
2377 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2378
2379 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2380 {
2381         __le64 ts;
2382         int ret;
2383
2384         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2385                 return 0;
2386
2387         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2388         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2389                         NULL);
2390         if (ret)
2391                 dev_warn_once(ctrl->device,
2392                         "could not set timestamp (%d)\n", ret);
2393         return ret;
2394 }
2395
2396 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2397 {
2398         struct nvme_feat_host_behavior *host;
2399         u8 acre = 0, lbafee = 0;
2400         int ret;
2401
2402         /* Don't bother enabling the feature if retry delay is not reported */
2403         if (ctrl->crdt[0])
2404                 acre = NVME_ENABLE_ACRE;
2405         if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2406                 lbafee = NVME_ENABLE_LBAFEE;
2407
2408         if (!acre && !lbafee)
2409                 return 0;
2410
2411         host = kzalloc(sizeof(*host), GFP_KERNEL);
2412         if (!host)
2413                 return 0;
2414
2415         host->acre = acre;
2416         host->lbafee = lbafee;
2417         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2418                                 host, sizeof(*host), NULL);
2419         kfree(host);
2420         return ret;
2421 }
2422
2423 /*
2424  * The function checks whether the given total (exlat + enlat) latency of
2425  * a power state allows the latter to be used as an APST transition target.
2426  * It does so by comparing the latency to the primary and secondary latency
2427  * tolerances defined by module params. If there's a match, the corresponding
2428  * timeout value is returned and the matching tolerance index (1 or 2) is
2429  * reported.
2430  */
2431 static bool nvme_apst_get_transition_time(u64 total_latency,
2432                 u64 *transition_time, unsigned *last_index)
2433 {
2434         if (total_latency <= apst_primary_latency_tol_us) {
2435                 if (*last_index == 1)
2436                         return false;
2437                 *last_index = 1;
2438                 *transition_time = apst_primary_timeout_ms;
2439                 return true;
2440         }
2441         if (apst_secondary_timeout_ms &&
2442                 total_latency <= apst_secondary_latency_tol_us) {
2443                 if (*last_index <= 2)
2444                         return false;
2445                 *last_index = 2;
2446                 *transition_time = apst_secondary_timeout_ms;
2447                 return true;
2448         }
2449         return false;
2450 }
2451
2452 /*
2453  * APST (Autonomous Power State Transition) lets us program a table of power
2454  * state transitions that the controller will perform automatically.
2455  *
2456  * Depending on module params, one of the two supported techniques will be used:
2457  *
2458  * - If the parameters provide explicit timeouts and tolerances, they will be
2459  *   used to build a table with up to 2 non-operational states to transition to.
2460  *   The default parameter values were selected based on the values used by
2461  *   Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2462  *   regeneration of the APST table in the event of switching between external
2463  *   and battery power, the timeouts and tolerances reflect a compromise
2464  *   between values used by Microsoft for AC and battery scenarios.
2465  * - If not, we'll configure the table with a simple heuristic: we are willing
2466  *   to spend at most 2% of the time transitioning between power states.
2467  *   Therefore, when running in any given state, we will enter the next
2468  *   lower-power non-operational state after waiting 50 * (enlat + exlat)
2469  *   microseconds, as long as that state's exit latency is under the requested
2470  *   maximum latency.
2471  *
2472  * We will not autonomously enter any non-operational state for which the total
2473  * latency exceeds ps_max_latency_us.
2474  *
2475  * Users can set ps_max_latency_us to zero to turn off APST.
2476  */
2477 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2478 {
2479         struct nvme_feat_auto_pst *table;
2480         unsigned apste = 0;
2481         u64 max_lat_us = 0;
2482         __le64 target = 0;
2483         int max_ps = -1;
2484         int state;
2485         int ret;
2486         unsigned last_lt_index = UINT_MAX;
2487
2488         /*
2489          * If APST isn't supported or if we haven't been initialized yet,
2490          * then don't do anything.
2491          */
2492         if (!ctrl->apsta)
2493                 return 0;
2494
2495         if (ctrl->npss > 31) {
2496                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2497                 return 0;
2498         }
2499
2500         table = kzalloc(sizeof(*table), GFP_KERNEL);
2501         if (!table)
2502                 return 0;
2503
2504         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2505                 /* Turn off APST. */
2506                 dev_dbg(ctrl->device, "APST disabled\n");
2507                 goto done;
2508         }
2509
2510         /*
2511          * Walk through all states from lowest- to highest-power.
2512          * According to the spec, lower-numbered states use more power.  NPSS,
2513          * despite the name, is the index of the lowest-power state, not the
2514          * number of states.
2515          */
2516         for (state = (int)ctrl->npss; state >= 0; state--) {
2517                 u64 total_latency_us, exit_latency_us, transition_ms;
2518
2519                 if (target)
2520                         table->entries[state] = target;
2521
2522                 /*
2523                  * Don't allow transitions to the deepest state if it's quirked
2524                  * off.
2525                  */
2526                 if (state == ctrl->npss &&
2527                     (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2528                         continue;
2529
2530                 /*
2531                  * Is this state a useful non-operational state for higher-power
2532                  * states to autonomously transition to?
2533                  */
2534                 if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2535                         continue;
2536
2537                 exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2538                 if (exit_latency_us > ctrl->ps_max_latency_us)
2539                         continue;
2540
2541                 total_latency_us = exit_latency_us +
2542                         le32_to_cpu(ctrl->psd[state].entry_lat);
2543
2544                 /*
2545                  * This state is good. It can be used as the APST idle target
2546                  * for higher power states.
2547                  */
2548                 if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2549                         if (!nvme_apst_get_transition_time(total_latency_us,
2550                                         &transition_ms, &last_lt_index))
2551                                 continue;
2552                 } else {
2553                         transition_ms = total_latency_us + 19;
2554                         do_div(transition_ms, 20);
2555                         if (transition_ms > (1 << 24) - 1)
2556                                 transition_ms = (1 << 24) - 1;
2557                 }
2558
2559                 target = cpu_to_le64((state << 3) | (transition_ms << 8));
2560                 if (max_ps == -1)
2561                         max_ps = state;
2562                 if (total_latency_us > max_lat_us)
2563                         max_lat_us = total_latency_us;
2564         }
2565
2566         if (max_ps == -1)
2567                 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2568         else
2569                 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2570                         max_ps, max_lat_us, (int)sizeof(*table), table);
2571         apste = 1;
2572
2573 done:
2574         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2575                                 table, sizeof(*table), NULL);
2576         if (ret)
2577                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2578         kfree(table);
2579         return ret;
2580 }
2581
2582 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2583 {
2584         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2585         u64 latency;
2586
2587         switch (val) {
2588         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2589         case PM_QOS_LATENCY_ANY:
2590                 latency = U64_MAX;
2591                 break;
2592
2593         default:
2594                 latency = val;
2595         }
2596
2597         if (ctrl->ps_max_latency_us != latency) {
2598                 ctrl->ps_max_latency_us = latency;
2599                 if (ctrl->state == NVME_CTRL_LIVE)
2600                         nvme_configure_apst(ctrl);
2601         }
2602 }
2603
2604 struct nvme_core_quirk_entry {
2605         /*
2606          * NVMe model and firmware strings are padded with spaces.  For
2607          * simplicity, strings in the quirk table are padded with NULLs
2608          * instead.
2609          */
2610         u16 vid;
2611         const char *mn;
2612         const char *fr;
2613         unsigned long quirks;
2614 };
2615
2616 static const struct nvme_core_quirk_entry core_quirks[] = {
2617         {
2618                 /*
2619                  * This Toshiba device seems to die using any APST states.  See:
2620                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2621                  */
2622                 .vid = 0x1179,
2623                 .mn = "THNSF5256GPUK TOSHIBA",
2624                 .quirks = NVME_QUIRK_NO_APST,
2625         },
2626         {
2627                 /*
2628                  * This LiteON CL1-3D*-Q11 firmware version has a race
2629                  * condition associated with actions related to suspend to idle
2630                  * LiteON has resolved the problem in future firmware
2631                  */
2632                 .vid = 0x14a4,
2633                 .fr = "22301111",
2634                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2635         },
2636         {
2637                 /*
2638                  * This Kioxia CD6-V Series / HPE PE8030 device times out and
2639                  * aborts I/O during any load, but more easily reproducible
2640                  * with discards (fstrim).
2641                  *
2642                  * The device is left in a state where it is also not possible
2643                  * to use "nvme set-feature" to disable APST, but booting with
2644                  * nvme_core.default_ps_max_latency=0 works.
2645                  */
2646                 .vid = 0x1e0f,
2647                 .mn = "KCD6XVUL6T40",
2648                 .quirks = NVME_QUIRK_NO_APST,
2649         },
2650         {
2651                 /*
2652                  * The external Samsung X5 SSD fails initialization without a
2653                  * delay before checking if it is ready and has a whole set of
2654                  * other problems.  To make this even more interesting, it
2655                  * shares the PCI ID with internal Samsung 970 Evo Plus that
2656                  * does not need or want these quirks.
2657                  */
2658                 .vid = 0x144d,
2659                 .mn = "Samsung Portable SSD X5",
2660                 .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2661                           NVME_QUIRK_NO_DEEPEST_PS |
2662                           NVME_QUIRK_IGNORE_DEV_SUBNQN,
2663         }
2664 };
2665
2666 /* match is null-terminated but idstr is space-padded. */
2667 static bool string_matches(const char *idstr, const char *match, size_t len)
2668 {
2669         size_t matchlen;
2670
2671         if (!match)
2672                 return true;
2673
2674         matchlen = strlen(match);
2675         WARN_ON_ONCE(matchlen > len);
2676
2677         if (memcmp(idstr, match, matchlen))
2678                 return false;
2679
2680         for (; matchlen < len; matchlen++)
2681                 if (idstr[matchlen] != ' ')
2682                         return false;
2683
2684         return true;
2685 }
2686
2687 static bool quirk_matches(const struct nvme_id_ctrl *id,
2688                           const struct nvme_core_quirk_entry *q)
2689 {
2690         return q->vid == le16_to_cpu(id->vid) &&
2691                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2692                 string_matches(id->fr, q->fr, sizeof(id->fr));
2693 }
2694
2695 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2696                 struct nvme_id_ctrl *id)
2697 {
2698         size_t nqnlen;
2699         int off;
2700
2701         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2702                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2703                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2704                         strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2705                         return;
2706                 }
2707
2708                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2709                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2710         }
2711
2712         /*
2713          * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe
2714          * Base Specification 2.0.  It is slightly different from the format
2715          * specified there due to historic reasons, and we can't change it now.
2716          */
2717         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2718                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2719                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2720         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2721         off += sizeof(id->sn);
2722         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2723         off += sizeof(id->mn);
2724         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2725 }
2726
2727 static void nvme_release_subsystem(struct device *dev)
2728 {
2729         struct nvme_subsystem *subsys =
2730                 container_of(dev, struct nvme_subsystem, dev);
2731
2732         if (subsys->instance >= 0)
2733                 ida_free(&nvme_instance_ida, subsys->instance);
2734         kfree(subsys);
2735 }
2736
2737 static void nvme_destroy_subsystem(struct kref *ref)
2738 {
2739         struct nvme_subsystem *subsys =
2740                         container_of(ref, struct nvme_subsystem, ref);
2741
2742         mutex_lock(&nvme_subsystems_lock);
2743         list_del(&subsys->entry);
2744         mutex_unlock(&nvme_subsystems_lock);
2745
2746         ida_destroy(&subsys->ns_ida);
2747         device_del(&subsys->dev);
2748         put_device(&subsys->dev);
2749 }
2750
2751 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2752 {
2753         kref_put(&subsys->ref, nvme_destroy_subsystem);
2754 }
2755
2756 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2757 {
2758         struct nvme_subsystem *subsys;
2759
2760         lockdep_assert_held(&nvme_subsystems_lock);
2761
2762         /*
2763          * Fail matches for discovery subsystems. This results
2764          * in each discovery controller bound to a unique subsystem.
2765          * This avoids issues with validating controller values
2766          * that can only be true when there is a single unique subsystem.
2767          * There may be multiple and completely independent entities
2768          * that provide discovery controllers.
2769          */
2770         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2771                 return NULL;
2772
2773         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2774                 if (strcmp(subsys->subnqn, subsysnqn))
2775                         continue;
2776                 if (!kref_get_unless_zero(&subsys->ref))
2777                         continue;
2778                 return subsys;
2779         }
2780
2781         return NULL;
2782 }
2783
2784 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2785         struct device_attribute subsys_attr_##_name = \
2786                 __ATTR(_name, _mode, _show, NULL)
2787
2788 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2789                                     struct device_attribute *attr,
2790                                     char *buf)
2791 {
2792         struct nvme_subsystem *subsys =
2793                 container_of(dev, struct nvme_subsystem, dev);
2794
2795         return sysfs_emit(buf, "%s\n", subsys->subnqn);
2796 }
2797 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2798
2799 static ssize_t nvme_subsys_show_type(struct device *dev,
2800                                     struct device_attribute *attr,
2801                                     char *buf)
2802 {
2803         struct nvme_subsystem *subsys =
2804                 container_of(dev, struct nvme_subsystem, dev);
2805
2806         switch (subsys->subtype) {
2807         case NVME_NQN_DISC:
2808                 return sysfs_emit(buf, "discovery\n");
2809         case NVME_NQN_NVME:
2810                 return sysfs_emit(buf, "nvm\n");
2811         default:
2812                 return sysfs_emit(buf, "reserved\n");
2813         }
2814 }
2815 static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type);
2816
2817 #define nvme_subsys_show_str_function(field)                            \
2818 static ssize_t subsys_##field##_show(struct device *dev,                \
2819                             struct device_attribute *attr, char *buf)   \
2820 {                                                                       \
2821         struct nvme_subsystem *subsys =                                 \
2822                 container_of(dev, struct nvme_subsystem, dev);          \
2823         return sysfs_emit(buf, "%.*s\n",                                \
2824                            (int)sizeof(subsys->field), subsys->field);  \
2825 }                                                                       \
2826 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2827
2828 nvme_subsys_show_str_function(model);
2829 nvme_subsys_show_str_function(serial);
2830 nvme_subsys_show_str_function(firmware_rev);
2831
2832 static struct attribute *nvme_subsys_attrs[] = {
2833         &subsys_attr_model.attr,
2834         &subsys_attr_serial.attr,
2835         &subsys_attr_firmware_rev.attr,
2836         &subsys_attr_subsysnqn.attr,
2837         &subsys_attr_subsystype.attr,
2838 #ifdef CONFIG_NVME_MULTIPATH
2839         &subsys_attr_iopolicy.attr,
2840 #endif
2841         NULL,
2842 };
2843
2844 static const struct attribute_group nvme_subsys_attrs_group = {
2845         .attrs = nvme_subsys_attrs,
2846 };
2847
2848 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2849         &nvme_subsys_attrs_group,
2850         NULL,
2851 };
2852
2853 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2854 {
2855         return ctrl->opts && ctrl->opts->discovery_nqn;
2856 }
2857
2858 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2859                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2860 {
2861         struct nvme_ctrl *tmp;
2862
2863         lockdep_assert_held(&nvme_subsystems_lock);
2864
2865         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2866                 if (nvme_state_terminal(tmp))
2867                         continue;
2868
2869                 if (tmp->cntlid == ctrl->cntlid) {
2870                         dev_err(ctrl->device,
2871                                 "Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2872                                 ctrl->cntlid, dev_name(tmp->device),
2873                                 subsys->subnqn);
2874                         return false;
2875                 }
2876
2877                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2878                     nvme_discovery_ctrl(ctrl))
2879                         continue;
2880
2881                 dev_err(ctrl->device,
2882                         "Subsystem does not support multiple controllers\n");
2883                 return false;
2884         }
2885
2886         return true;
2887 }
2888
2889 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2890 {
2891         struct nvme_subsystem *subsys, *found;
2892         int ret;
2893
2894         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2895         if (!subsys)
2896                 return -ENOMEM;
2897
2898         subsys->instance = -1;
2899         mutex_init(&subsys->lock);
2900         kref_init(&subsys->ref);
2901         INIT_LIST_HEAD(&subsys->ctrls);
2902         INIT_LIST_HEAD(&subsys->nsheads);
2903         nvme_init_subnqn(subsys, ctrl, id);
2904         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2905         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2906         subsys->vendor_id = le16_to_cpu(id->vid);
2907         subsys->cmic = id->cmic;
2908
2909         /* Versions prior to 1.4 don't necessarily report a valid type */
2910         if (id->cntrltype == NVME_CTRL_DISC ||
2911             !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2912                 subsys->subtype = NVME_NQN_DISC;
2913         else
2914                 subsys->subtype = NVME_NQN_NVME;
2915
2916         if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2917                 dev_err(ctrl->device,
2918                         "Subsystem %s is not a discovery controller",
2919                         subsys->subnqn);
2920                 kfree(subsys);
2921                 return -EINVAL;
2922         }
2923         subsys->awupf = le16_to_cpu(id->awupf);
2924         nvme_mpath_default_iopolicy(subsys);
2925
2926         subsys->dev.class = nvme_subsys_class;
2927         subsys->dev.release = nvme_release_subsystem;
2928         subsys->dev.groups = nvme_subsys_attrs_groups;
2929         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2930         device_initialize(&subsys->dev);
2931
2932         mutex_lock(&nvme_subsystems_lock);
2933         found = __nvme_find_get_subsystem(subsys->subnqn);
2934         if (found) {
2935                 put_device(&subsys->dev);
2936                 subsys = found;
2937
2938                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2939                         ret = -EINVAL;
2940                         goto out_put_subsystem;
2941                 }
2942         } else {
2943                 ret = device_add(&subsys->dev);
2944                 if (ret) {
2945                         dev_err(ctrl->device,
2946                                 "failed to register subsystem device.\n");
2947                         put_device(&subsys->dev);
2948                         goto out_unlock;
2949                 }
2950                 ida_init(&subsys->ns_ida);
2951                 list_add_tail(&subsys->entry, &nvme_subsystems);
2952         }
2953
2954         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2955                                 dev_name(ctrl->device));
2956         if (ret) {
2957                 dev_err(ctrl->device,
2958                         "failed to create sysfs link from subsystem.\n");
2959                 goto out_put_subsystem;
2960         }
2961
2962         if (!found)
2963                 subsys->instance = ctrl->instance;
2964         ctrl->subsys = subsys;
2965         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2966         mutex_unlock(&nvme_subsystems_lock);
2967         return 0;
2968
2969 out_put_subsystem:
2970         nvme_put_subsystem(subsys);
2971 out_unlock:
2972         mutex_unlock(&nvme_subsystems_lock);
2973         return ret;
2974 }
2975
2976 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2977                 void *log, size_t size, u64 offset)
2978 {
2979         struct nvme_command c = { };
2980         u32 dwlen = nvme_bytes_to_numd(size);
2981
2982         c.get_log_page.opcode = nvme_admin_get_log_page;
2983         c.get_log_page.nsid = cpu_to_le32(nsid);
2984         c.get_log_page.lid = log_page;
2985         c.get_log_page.lsp = lsp;
2986         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2987         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2988         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2989         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2990         c.get_log_page.csi = csi;
2991
2992         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2993 }
2994
2995 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2996                                 struct nvme_effects_log **log)
2997 {
2998         struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
2999         int ret;
3000
3001         if (cel)
3002                 goto out;
3003
3004         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3005         if (!cel)
3006                 return -ENOMEM;
3007
3008         ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
3009                         cel, sizeof(*cel), 0);
3010         if (ret) {
3011                 kfree(cel);
3012                 return ret;
3013         }
3014
3015         xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
3016 out:
3017         *log = cel;
3018         return 0;
3019 }
3020
3021 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
3022 {
3023         u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
3024
3025         if (check_shl_overflow(1U, units + page_shift - 9, &val))
3026                 return UINT_MAX;
3027         return val;
3028 }
3029
3030 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
3031 {
3032         struct nvme_command c = { };
3033         struct nvme_id_ctrl_nvm *id;
3034         int ret;
3035
3036         if (ctrl->oncs & NVME_CTRL_ONCS_DSM) {
3037                 ctrl->max_discard_sectors = UINT_MAX;
3038                 ctrl->max_discard_segments = NVME_DSM_MAX_RANGES;
3039         } else {
3040                 ctrl->max_discard_sectors = 0;
3041                 ctrl->max_discard_segments = 0;
3042         }
3043
3044         /*
3045          * Even though NVMe spec explicitly states that MDTS is not applicable
3046          * to the write-zeroes, we are cautious and limit the size to the
3047          * controllers max_hw_sectors value, which is based on the MDTS field
3048          * and possibly other limiting factors.
3049          */
3050         if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
3051             !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
3052                 ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
3053         else
3054                 ctrl->max_zeroes_sectors = 0;
3055
3056         if (nvme_ctrl_limited_cns(ctrl))
3057                 return 0;
3058
3059         id = kzalloc(sizeof(*id), GFP_KERNEL);
3060         if (!id)
3061                 return -ENOMEM;
3062
3063         c.identify.opcode = nvme_admin_identify;
3064         c.identify.cns = NVME_ID_CNS_CS_CTRL;
3065         c.identify.csi = NVME_CSI_NVM;
3066
3067         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
3068         if (ret)
3069                 goto free_data;
3070
3071         if (id->dmrl)
3072                 ctrl->max_discard_segments = id->dmrl;
3073         ctrl->dmrsl = le32_to_cpu(id->dmrsl);
3074         if (id->wzsl)
3075                 ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
3076
3077 free_data:
3078         kfree(id);
3079         return ret;
3080 }
3081
3082 static void nvme_init_known_nvm_effects(struct nvme_ctrl *ctrl)
3083 {
3084         struct nvme_effects_log *log = ctrl->effects;
3085
3086         log->acs[nvme_admin_format_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3087                                                 NVME_CMD_EFFECTS_NCC |
3088                                                 NVME_CMD_EFFECTS_CSE_MASK);
3089         log->acs[nvme_admin_sanitize_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3090                                                 NVME_CMD_EFFECTS_CSE_MASK);
3091
3092         /*
3093          * The spec says the result of a security receive command depends on
3094          * the previous security send command. As such, many vendors log this
3095          * command as one to submitted only when no other commands to the same
3096          * namespace are outstanding. The intention is to tell the host to
3097          * prevent mixing security send and receive.
3098          *
3099          * This driver can only enforce such exclusive access against IO
3100          * queues, though. We are not readily able to enforce such a rule for
3101          * two commands to the admin queue, which is the only queue that
3102          * matters for this command.
3103          *
3104          * Rather than blindly freezing the IO queues for this effect that
3105          * doesn't even apply to IO, mask it off.
3106          */
3107         log->acs[nvme_admin_security_recv] &= cpu_to_le32(~NVME_CMD_EFFECTS_CSE_MASK);
3108
3109         log->iocs[nvme_cmd_write] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3110         log->iocs[nvme_cmd_write_zeroes] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3111         log->iocs[nvme_cmd_write_uncor] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3112 }
3113
3114 static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
3115 {
3116         int ret = 0;
3117
3118         if (ctrl->effects)
3119                 return 0;
3120
3121         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3122                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3123                 if (ret < 0)
3124                         return ret;
3125         }
3126
3127         if (!ctrl->effects) {
3128                 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
3129                 if (!ctrl->effects)
3130                         return -ENOMEM;
3131                 xa_store(&ctrl->cels, NVME_CSI_NVM, ctrl->effects, GFP_KERNEL);
3132         }
3133
3134         nvme_init_known_nvm_effects(ctrl);
3135         return 0;
3136 }
3137
3138 static int nvme_init_identify(struct nvme_ctrl *ctrl)
3139 {
3140         struct nvme_id_ctrl *id;
3141         u32 max_hw_sectors;
3142         bool prev_apst_enabled;
3143         int ret;
3144
3145         ret = nvme_identify_ctrl(ctrl, &id);
3146         if (ret) {
3147                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3148                 return -EIO;
3149         }
3150
3151         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3152                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3153
3154         if (!ctrl->identified) {
3155                 unsigned int i;
3156
3157                 /*
3158                  * Check for quirks.  Quirk can depend on firmware version,
3159                  * so, in principle, the set of quirks present can change
3160                  * across a reset.  As a possible future enhancement, we
3161                  * could re-scan for quirks every time we reinitialize
3162                  * the device, but we'd have to make sure that the driver
3163                  * behaves intelligently if the quirks change.
3164                  */
3165                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3166                         if (quirk_matches(id, &core_quirks[i]))
3167                                 ctrl->quirks |= core_quirks[i].quirks;
3168                 }
3169
3170                 ret = nvme_init_subsystem(ctrl, id);
3171                 if (ret)
3172                         goto out_free;
3173
3174                 ret = nvme_init_effects(ctrl, id);
3175                 if (ret)
3176                         goto out_free;
3177         }
3178         memcpy(ctrl->subsys->firmware_rev, id->fr,
3179                sizeof(ctrl->subsys->firmware_rev));
3180
3181         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3182                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3183                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3184         }
3185
3186         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3187         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3188         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3189
3190         ctrl->oacs = le16_to_cpu(id->oacs);
3191         ctrl->oncs = le16_to_cpu(id->oncs);
3192         ctrl->mtfa = le16_to_cpu(id->mtfa);
3193         ctrl->oaes = le32_to_cpu(id->oaes);
3194         ctrl->wctemp = le16_to_cpu(id->wctemp);
3195         ctrl->cctemp = le16_to_cpu(id->cctemp);
3196
3197         atomic_set(&ctrl->abort_limit, id->acl + 1);
3198         ctrl->vwc = id->vwc;
3199         if (id->mdts)
3200                 max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3201         else
3202                 max_hw_sectors = UINT_MAX;
3203         ctrl->max_hw_sectors =
3204                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3205
3206         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3207         ctrl->sgls = le32_to_cpu(id->sgls);
3208         ctrl->kas = le16_to_cpu(id->kas);
3209         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3210         ctrl->ctratt = le32_to_cpu(id->ctratt);
3211
3212         ctrl->cntrltype = id->cntrltype;
3213         ctrl->dctype = id->dctype;
3214
3215         if (id->rtd3e) {
3216                 /* us -> s */
3217                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3218
3219                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3220                                                  shutdown_timeout, 60);
3221
3222                 if (ctrl->shutdown_timeout != shutdown_timeout)
3223                         dev_info(ctrl->device,
3224                                  "Shutdown timeout set to %u seconds\n",
3225                                  ctrl->shutdown_timeout);
3226         } else
3227                 ctrl->shutdown_timeout = shutdown_timeout;
3228
3229         ctrl->npss = id->npss;
3230         ctrl->apsta = id->apsta;
3231         prev_apst_enabled = ctrl->apst_enabled;
3232         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3233                 if (force_apst && id->apsta) {
3234                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3235                         ctrl->apst_enabled = true;
3236                 } else {
3237                         ctrl->apst_enabled = false;
3238                 }
3239         } else {
3240                 ctrl->apst_enabled = id->apsta;
3241         }
3242         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3243
3244         if (ctrl->ops->flags & NVME_F_FABRICS) {
3245                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3246                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3247                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3248                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3249
3250                 /*
3251                  * In fabrics we need to verify the cntlid matches the
3252                  * admin connect
3253                  */
3254                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3255                         dev_err(ctrl->device,
3256                                 "Mismatching cntlid: Connect %u vs Identify "
3257                                 "%u, rejecting\n",
3258                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3259                         ret = -EINVAL;
3260                         goto out_free;
3261                 }
3262
3263                 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3264                         dev_err(ctrl->device,
3265                                 "keep-alive support is mandatory for fabrics\n");
3266                         ret = -EINVAL;
3267                         goto out_free;
3268                 }
3269         } else {
3270                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3271                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3272                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3273                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3274         }
3275
3276         ret = nvme_mpath_init_identify(ctrl, id);
3277         if (ret < 0)
3278                 goto out_free;
3279
3280         if (ctrl->apst_enabled && !prev_apst_enabled)
3281                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3282         else if (!ctrl->apst_enabled && prev_apst_enabled)
3283                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3284
3285 out_free:
3286         kfree(id);
3287         return ret;
3288 }
3289
3290 /*
3291  * Initialize the cached copies of the Identify data and various controller
3292  * register in our nvme_ctrl structure.  This should be called as soon as
3293  * the admin queue is fully up and running.
3294  */
3295 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
3296 {
3297         int ret;
3298
3299         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3300         if (ret) {
3301                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3302                 return ret;
3303         }
3304
3305         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3306
3307         if (ctrl->vs >= NVME_VS(1, 1, 0))
3308                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3309
3310         ret = nvme_init_identify(ctrl);
3311         if (ret)
3312                 return ret;
3313
3314         ret = nvme_configure_apst(ctrl);
3315         if (ret < 0)
3316                 return ret;
3317
3318         ret = nvme_configure_timestamp(ctrl);
3319         if (ret < 0)
3320                 return ret;
3321
3322         ret = nvme_configure_host_options(ctrl);
3323         if (ret < 0)
3324                 return ret;
3325
3326         nvme_configure_opal(ctrl, was_suspended);
3327
3328         if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3329                 /*
3330                  * Do not return errors unless we are in a controller reset,
3331                  * the controller works perfectly fine without hwmon.
3332                  */
3333                 ret = nvme_hwmon_init(ctrl);
3334                 if (ret == -EINTR)
3335                         return ret;
3336         }
3337
3338         ctrl->identified = true;
3339
3340         return 0;
3341 }
3342 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3343
3344 static int nvme_dev_open(struct inode *inode, struct file *file)
3345 {
3346         struct nvme_ctrl *ctrl =
3347                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3348
3349         switch (ctrl->state) {
3350         case NVME_CTRL_LIVE:
3351                 break;
3352         default:
3353                 return -EWOULDBLOCK;
3354         }
3355
3356         nvme_get_ctrl(ctrl);
3357         if (!try_module_get(ctrl->ops->module)) {
3358                 nvme_put_ctrl(ctrl);
3359                 return -EINVAL;
3360         }
3361
3362         file->private_data = ctrl;
3363         return 0;
3364 }
3365
3366 static int nvme_dev_release(struct inode *inode, struct file *file)
3367 {
3368         struct nvme_ctrl *ctrl =
3369                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3370
3371         module_put(ctrl->ops->module);
3372         nvme_put_ctrl(ctrl);
3373         return 0;
3374 }
3375
3376 static const struct file_operations nvme_dev_fops = {
3377         .owner          = THIS_MODULE,
3378         .open           = nvme_dev_open,
3379         .release        = nvme_dev_release,
3380         .unlocked_ioctl = nvme_dev_ioctl,
3381         .compat_ioctl   = compat_ptr_ioctl,
3382         .uring_cmd      = nvme_dev_uring_cmd,
3383 };
3384
3385 static ssize_t nvme_sysfs_reset(struct device *dev,
3386                                 struct device_attribute *attr, const char *buf,
3387                                 size_t count)
3388 {
3389         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3390         int ret;
3391
3392         ret = nvme_reset_ctrl_sync(ctrl);
3393         if (ret < 0)
3394                 return ret;
3395         return count;
3396 }
3397 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3398
3399 static ssize_t nvme_sysfs_rescan(struct device *dev,
3400                                 struct device_attribute *attr, const char *buf,
3401                                 size_t count)
3402 {
3403         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3404
3405         nvme_queue_scan(ctrl);
3406         return count;
3407 }
3408 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3409
3410 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3411 {
3412         struct gendisk *disk = dev_to_disk(dev);
3413
3414         if (disk->fops == &nvme_bdev_ops)
3415                 return nvme_get_ns_from_dev(dev)->head;
3416         else
3417                 return disk->private_data;
3418 }
3419
3420 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3421                 char *buf)
3422 {
3423         struct nvme_ns_head *head = dev_to_ns_head(dev);
3424         struct nvme_ns_ids *ids = &head->ids;
3425         struct nvme_subsystem *subsys = head->subsys;
3426         int serial_len = sizeof(subsys->serial);
3427         int model_len = sizeof(subsys->model);
3428
3429         if (!uuid_is_null(&ids->uuid))
3430                 return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
3431
3432         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3433                 return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
3434
3435         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3436                 return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
3437
3438         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3439                                   subsys->serial[serial_len - 1] == '\0'))
3440                 serial_len--;
3441         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3442                                  subsys->model[model_len - 1] == '\0'))
3443                 model_len--;
3444
3445         return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3446                 serial_len, subsys->serial, model_len, subsys->model,
3447                 head->ns_id);
3448 }
3449 static DEVICE_ATTR_RO(wwid);
3450
3451 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3452                 char *buf)
3453 {
3454         return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3455 }
3456 static DEVICE_ATTR_RO(nguid);
3457
3458 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3459                 char *buf)
3460 {
3461         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3462
3463         /* For backward compatibility expose the NGUID to userspace if
3464          * we have no UUID set
3465          */
3466         if (uuid_is_null(&ids->uuid)) {
3467                 dev_warn_ratelimited(dev,
3468                         "No UUID available providing old NGUID\n");
3469                 return sysfs_emit(buf, "%pU\n", ids->nguid);
3470         }
3471         return sysfs_emit(buf, "%pU\n", &ids->uuid);
3472 }
3473 static DEVICE_ATTR_RO(uuid);
3474
3475 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3476                 char *buf)
3477 {
3478         return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3479 }
3480 static DEVICE_ATTR_RO(eui);
3481
3482 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3483                 char *buf)
3484 {
3485         return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3486 }
3487 static DEVICE_ATTR_RO(nsid);
3488
3489 static struct attribute *nvme_ns_id_attrs[] = {
3490         &dev_attr_wwid.attr,
3491         &dev_attr_uuid.attr,
3492         &dev_attr_nguid.attr,
3493         &dev_attr_eui.attr,
3494         &dev_attr_nsid.attr,
3495 #ifdef CONFIG_NVME_MULTIPATH
3496         &dev_attr_ana_grpid.attr,
3497         &dev_attr_ana_state.attr,
3498 #endif
3499         NULL,
3500 };
3501
3502 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3503                 struct attribute *a, int n)
3504 {
3505         struct device *dev = container_of(kobj, struct device, kobj);
3506         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3507
3508         if (a == &dev_attr_uuid.attr) {
3509                 if (uuid_is_null(&ids->uuid) &&
3510                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3511                         return 0;
3512         }
3513         if (a == &dev_attr_nguid.attr) {
3514                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3515                         return 0;
3516         }
3517         if (a == &dev_attr_eui.attr) {
3518                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3519                         return 0;
3520         }
3521 #ifdef CONFIG_NVME_MULTIPATH
3522         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3523                 if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */
3524                         return 0;
3525                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3526                         return 0;
3527         }
3528 #endif
3529         return a->mode;
3530 }
3531
3532 static const struct attribute_group nvme_ns_id_attr_group = {
3533         .attrs          = nvme_ns_id_attrs,
3534         .is_visible     = nvme_ns_id_attrs_are_visible,
3535 };
3536
3537 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3538         &nvme_ns_id_attr_group,
3539         NULL,
3540 };
3541
3542 #define nvme_show_str_function(field)                                           \
3543 static ssize_t  field##_show(struct device *dev,                                \
3544                             struct device_attribute *attr, char *buf)           \
3545 {                                                                               \
3546         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3547         return sysfs_emit(buf, "%.*s\n",                                        \
3548                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3549 }                                                                               \
3550 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3551
3552 nvme_show_str_function(model);
3553 nvme_show_str_function(serial);
3554 nvme_show_str_function(firmware_rev);
3555
3556 #define nvme_show_int_function(field)                                           \
3557 static ssize_t  field##_show(struct device *dev,                                \
3558                             struct device_attribute *attr, char *buf)           \
3559 {                                                                               \
3560         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3561         return sysfs_emit(buf, "%d\n", ctrl->field);                            \
3562 }                                                                               \
3563 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3564
3565 nvme_show_int_function(cntlid);
3566 nvme_show_int_function(numa_node);
3567 nvme_show_int_function(queue_count);
3568 nvme_show_int_function(sqsize);
3569 nvme_show_int_function(kato);
3570
3571 static ssize_t nvme_sysfs_delete(struct device *dev,
3572                                 struct device_attribute *attr, const char *buf,
3573                                 size_t count)
3574 {
3575         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3576
3577         if (device_remove_file_self(dev, attr))
3578                 nvme_delete_ctrl_sync(ctrl);
3579         return count;
3580 }
3581 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3582
3583 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3584                                          struct device_attribute *attr,
3585                                          char *buf)
3586 {
3587         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3588
3589         return sysfs_emit(buf, "%s\n", ctrl->ops->name);
3590 }
3591 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3592
3593 static ssize_t nvme_sysfs_show_state(struct device *dev,
3594                                      struct device_attribute *attr,
3595                                      char *buf)
3596 {
3597         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3598         static const char *const state_name[] = {
3599                 [NVME_CTRL_NEW]         = "new",
3600                 [NVME_CTRL_LIVE]        = "live",
3601                 [NVME_CTRL_RESETTING]   = "resetting",
3602                 [NVME_CTRL_CONNECTING]  = "connecting",
3603                 [NVME_CTRL_DELETING]    = "deleting",
3604                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3605                 [NVME_CTRL_DEAD]        = "dead",
3606         };
3607
3608         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3609             state_name[ctrl->state])
3610                 return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
3611
3612         return sysfs_emit(buf, "unknown state\n");
3613 }
3614
3615 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3616
3617 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3618                                          struct device_attribute *attr,
3619                                          char *buf)
3620 {
3621         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3622
3623         return sysfs_emit(buf, "%s\n", ctrl->subsys->subnqn);
3624 }
3625 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3626
3627 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3628                                         struct device_attribute *attr,
3629                                         char *buf)
3630 {
3631         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3632
3633         return sysfs_emit(buf, "%s\n", ctrl->opts->host->nqn);
3634 }
3635 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3636
3637 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3638                                         struct device_attribute *attr,
3639                                         char *buf)
3640 {
3641         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3642
3643         return sysfs_emit(buf, "%pU\n", &ctrl->opts->host->id);
3644 }
3645 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3646
3647 static ssize_t nvme_sysfs_show_address(struct device *dev,
3648                                          struct device_attribute *attr,
3649                                          char *buf)
3650 {
3651         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3652
3653         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3654 }
3655 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3656
3657 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3658                 struct device_attribute *attr, char *buf)
3659 {
3660         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3661         struct nvmf_ctrl_options *opts = ctrl->opts;
3662
3663         if (ctrl->opts->max_reconnects == -1)
3664                 return sysfs_emit(buf, "off\n");
3665         return sysfs_emit(buf, "%d\n",
3666                           opts->max_reconnects * opts->reconnect_delay);
3667 }
3668
3669 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3670                 struct device_attribute *attr, const char *buf, size_t count)
3671 {
3672         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3673         struct nvmf_ctrl_options *opts = ctrl->opts;
3674         int ctrl_loss_tmo, err;
3675
3676         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3677         if (err)
3678                 return -EINVAL;
3679
3680         if (ctrl_loss_tmo < 0)
3681                 opts->max_reconnects = -1;
3682         else
3683                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3684                                                 opts->reconnect_delay);
3685         return count;
3686 }
3687 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3688         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3689
3690 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3691                 struct device_attribute *attr, char *buf)
3692 {
3693         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3694
3695         if (ctrl->opts->reconnect_delay == -1)
3696                 return sysfs_emit(buf, "off\n");
3697         return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
3698 }
3699
3700 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3701                 struct device_attribute *attr, const char *buf, size_t count)
3702 {
3703         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3704         unsigned int v;
3705         int err;
3706
3707         err = kstrtou32(buf, 10, &v);
3708         if (err)
3709                 return err;
3710
3711         ctrl->opts->reconnect_delay = v;
3712         return count;
3713 }
3714 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3715         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3716
3717 static ssize_t nvme_ctrl_fast_io_fail_tmo_show(struct device *dev,
3718                 struct device_attribute *attr, char *buf)
3719 {
3720         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3721
3722         if (ctrl->opts->fast_io_fail_tmo == -1)
3723                 return sysfs_emit(buf, "off\n");
3724         return sysfs_emit(buf, "%d\n", ctrl->opts->fast_io_fail_tmo);
3725 }
3726
3727 static ssize_t nvme_ctrl_fast_io_fail_tmo_store(struct device *dev,
3728                 struct device_attribute *attr, const char *buf, size_t count)
3729 {
3730         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3731         struct nvmf_ctrl_options *opts = ctrl->opts;
3732         int fast_io_fail_tmo, err;
3733
3734         err = kstrtoint(buf, 10, &fast_io_fail_tmo);
3735         if (err)
3736                 return -EINVAL;
3737
3738         if (fast_io_fail_tmo < 0)
3739                 opts->fast_io_fail_tmo = -1;
3740         else
3741                 opts->fast_io_fail_tmo = fast_io_fail_tmo;
3742         return count;
3743 }
3744 static DEVICE_ATTR(fast_io_fail_tmo, S_IRUGO | S_IWUSR,
3745         nvme_ctrl_fast_io_fail_tmo_show, nvme_ctrl_fast_io_fail_tmo_store);
3746
3747 static ssize_t cntrltype_show(struct device *dev,
3748                               struct device_attribute *attr, char *buf)
3749 {
3750         static const char * const type[] = {
3751                 [NVME_CTRL_IO] = "io\n",
3752                 [NVME_CTRL_DISC] = "discovery\n",
3753                 [NVME_CTRL_ADMIN] = "admin\n",
3754         };
3755         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3756
3757         if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
3758                 return sysfs_emit(buf, "reserved\n");
3759
3760         return sysfs_emit(buf, type[ctrl->cntrltype]);
3761 }
3762 static DEVICE_ATTR_RO(cntrltype);
3763
3764 static ssize_t dctype_show(struct device *dev,
3765                            struct device_attribute *attr, char *buf)
3766 {
3767         static const char * const type[] = {
3768                 [NVME_DCTYPE_NOT_REPORTED] = "none\n",
3769                 [NVME_DCTYPE_DDC] = "ddc\n",
3770                 [NVME_DCTYPE_CDC] = "cdc\n",
3771         };
3772         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3773
3774         if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
3775                 return sysfs_emit(buf, "reserved\n");
3776
3777         return sysfs_emit(buf, type[ctrl->dctype]);
3778 }
3779 static DEVICE_ATTR_RO(dctype);
3780
3781 #ifdef CONFIG_NVME_AUTH
3782 static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev,
3783                 struct device_attribute *attr, char *buf)
3784 {
3785         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3786         struct nvmf_ctrl_options *opts = ctrl->opts;
3787
3788         if (!opts->dhchap_secret)
3789                 return sysfs_emit(buf, "none\n");
3790         return sysfs_emit(buf, "%s\n", opts->dhchap_secret);
3791 }
3792
3793 static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev,
3794                 struct device_attribute *attr, const char *buf, size_t count)
3795 {
3796         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3797         struct nvmf_ctrl_options *opts = ctrl->opts;
3798         char *dhchap_secret;
3799
3800         if (!ctrl->opts->dhchap_secret)
3801                 return -EINVAL;
3802         if (count < 7)
3803                 return -EINVAL;
3804         if (memcmp(buf, "DHHC-1:", 7))
3805                 return -EINVAL;
3806
3807         dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3808         if (!dhchap_secret)
3809                 return -ENOMEM;
3810         memcpy(dhchap_secret, buf, count);
3811         nvme_auth_stop(ctrl);
3812         if (strcmp(dhchap_secret, opts->dhchap_secret)) {
3813                 struct nvme_dhchap_key *key, *host_key;
3814                 int ret;
3815
3816                 ret = nvme_auth_generate_key(dhchap_secret, &key);
3817                 if (ret)
3818                         return ret;
3819                 kfree(opts->dhchap_secret);
3820                 opts->dhchap_secret = dhchap_secret;
3821                 host_key = ctrl->host_key;
3822                 mutex_lock(&ctrl->dhchap_auth_mutex);
3823                 ctrl->host_key = key;
3824                 mutex_unlock(&ctrl->dhchap_auth_mutex);
3825                 nvme_auth_free_key(host_key);
3826         }
3827         /* Start re-authentication */
3828         dev_info(ctrl->device, "re-authenticating controller\n");
3829         queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3830
3831         return count;
3832 }
3833 static DEVICE_ATTR(dhchap_secret, S_IRUGO | S_IWUSR,
3834         nvme_ctrl_dhchap_secret_show, nvme_ctrl_dhchap_secret_store);
3835
3836 static ssize_t nvme_ctrl_dhchap_ctrl_secret_show(struct device *dev,
3837                 struct device_attribute *attr, char *buf)
3838 {
3839         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3840         struct nvmf_ctrl_options *opts = ctrl->opts;
3841
3842         if (!opts->dhchap_ctrl_secret)
3843                 return sysfs_emit(buf, "none\n");
3844         return sysfs_emit(buf, "%s\n", opts->dhchap_ctrl_secret);
3845 }
3846
3847 static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev,
3848                 struct device_attribute *attr, const char *buf, size_t count)
3849 {
3850         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3851         struct nvmf_ctrl_options *opts = ctrl->opts;
3852         char *dhchap_secret;
3853
3854         if (!ctrl->opts->dhchap_ctrl_secret)
3855                 return -EINVAL;
3856         if (count < 7)
3857                 return -EINVAL;
3858         if (memcmp(buf, "DHHC-1:", 7))
3859                 return -EINVAL;
3860
3861         dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3862         if (!dhchap_secret)
3863                 return -ENOMEM;
3864         memcpy(dhchap_secret, buf, count);
3865         nvme_auth_stop(ctrl);
3866         if (strcmp(dhchap_secret, opts->dhchap_ctrl_secret)) {
3867                 struct nvme_dhchap_key *key, *ctrl_key;
3868                 int ret;
3869
3870                 ret = nvme_auth_generate_key(dhchap_secret, &key);
3871                 if (ret)
3872                         return ret;
3873                 kfree(opts->dhchap_ctrl_secret);
3874                 opts->dhchap_ctrl_secret = dhchap_secret;
3875                 ctrl_key = ctrl->ctrl_key;
3876                 mutex_lock(&ctrl->dhchap_auth_mutex);
3877                 ctrl->ctrl_key = key;
3878                 mutex_unlock(&ctrl->dhchap_auth_mutex);
3879                 nvme_auth_free_key(ctrl_key);
3880         }
3881         /* Start re-authentication */
3882         dev_info(ctrl->device, "re-authenticating controller\n");
3883         queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3884
3885         return count;
3886 }
3887 static DEVICE_ATTR(dhchap_ctrl_secret, S_IRUGO | S_IWUSR,
3888         nvme_ctrl_dhchap_ctrl_secret_show, nvme_ctrl_dhchap_ctrl_secret_store);
3889 #endif
3890
3891 static struct attribute *nvme_dev_attrs[] = {
3892         &dev_attr_reset_controller.attr,
3893         &dev_attr_rescan_controller.attr,
3894         &dev_attr_model.attr,
3895         &dev_attr_serial.attr,
3896         &dev_attr_firmware_rev.attr,
3897         &dev_attr_cntlid.attr,
3898         &dev_attr_delete_controller.attr,
3899         &dev_attr_transport.attr,
3900         &dev_attr_subsysnqn.attr,
3901         &dev_attr_address.attr,
3902         &dev_attr_state.attr,
3903         &dev_attr_numa_node.attr,
3904         &dev_attr_queue_count.attr,
3905         &dev_attr_sqsize.attr,
3906         &dev_attr_hostnqn.attr,
3907         &dev_attr_hostid.attr,
3908         &dev_attr_ctrl_loss_tmo.attr,
3909         &dev_attr_reconnect_delay.attr,
3910         &dev_attr_fast_io_fail_tmo.attr,
3911         &dev_attr_kato.attr,
3912         &dev_attr_cntrltype.attr,
3913         &dev_attr_dctype.attr,
3914 #ifdef CONFIG_NVME_AUTH
3915         &dev_attr_dhchap_secret.attr,
3916         &dev_attr_dhchap_ctrl_secret.attr,
3917 #endif
3918         NULL
3919 };
3920
3921 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3922                 struct attribute *a, int n)
3923 {
3924         struct device *dev = container_of(kobj, struct device, kobj);
3925         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3926
3927         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3928                 return 0;
3929         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3930                 return 0;
3931         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3932                 return 0;
3933         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3934                 return 0;
3935         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3936                 return 0;
3937         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3938                 return 0;
3939         if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts)
3940                 return 0;
3941 #ifdef CONFIG_NVME_AUTH
3942         if (a == &dev_attr_dhchap_secret.attr && !ctrl->opts)
3943                 return 0;
3944         if (a == &dev_attr_dhchap_ctrl_secret.attr && !ctrl->opts)
3945                 return 0;
3946 #endif
3947
3948         return a->mode;
3949 }
3950
3951 const struct attribute_group nvme_dev_attrs_group = {
3952         .attrs          = nvme_dev_attrs,
3953         .is_visible     = nvme_dev_attrs_are_visible,
3954 };
3955 EXPORT_SYMBOL_GPL(nvme_dev_attrs_group);
3956
3957 static const struct attribute_group *nvme_dev_attr_groups[] = {
3958         &nvme_dev_attrs_group,
3959         NULL,
3960 };
3961
3962 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3963                 unsigned nsid)
3964 {
3965         struct nvme_ns_head *h;
3966
3967         lockdep_assert_held(&ctrl->subsys->lock);
3968
3969         list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3970                 /*
3971                  * Private namespaces can share NSIDs under some conditions.
3972                  * In that case we can't use the same ns_head for namespaces
3973                  * with the same NSID.
3974                  */
3975                 if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3976                         continue;
3977                 if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3978                         return h;
3979         }
3980
3981         return NULL;
3982 }
3983
3984 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3985                 struct nvme_ns_ids *ids)
3986 {
3987         bool has_uuid = !uuid_is_null(&ids->uuid);
3988         bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3989         bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
3990         struct nvme_ns_head *h;
3991
3992         lockdep_assert_held(&subsys->lock);
3993
3994         list_for_each_entry(h, &subsys->nsheads, entry) {
3995                 if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3996                         return -EINVAL;
3997                 if (has_nguid &&
3998                     memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3999                         return -EINVAL;
4000                 if (has_eui64 &&
4001                     memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
4002                         return -EINVAL;
4003         }
4004
4005         return 0;
4006 }
4007
4008 static void nvme_cdev_rel(struct device *dev)
4009 {
4010         ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
4011 }
4012
4013 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
4014 {
4015         cdev_device_del(cdev, cdev_device);
4016         put_device(cdev_device);
4017 }
4018
4019 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
4020                 const struct file_operations *fops, struct module *owner)
4021 {
4022         int minor, ret;
4023
4024         minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
4025         if (minor < 0)
4026                 return minor;
4027         cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
4028         cdev_device->class = nvme_ns_chr_class;
4029         cdev_device->release = nvme_cdev_rel;
4030         device_initialize(cdev_device);
4031         cdev_init(cdev, fops);
4032         cdev->owner = owner;
4033         ret = cdev_device_add(cdev, cdev_device);
4034         if (ret)
4035                 put_device(cdev_device);
4036
4037         return ret;
4038 }
4039
4040 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
4041 {
4042         return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
4043 }
4044
4045 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
4046 {
4047         nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
4048         return 0;
4049 }
4050
4051 static const struct file_operations nvme_ns_chr_fops = {
4052         .owner          = THIS_MODULE,
4053         .open           = nvme_ns_chr_open,
4054         .release        = nvme_ns_chr_release,
4055         .unlocked_ioctl = nvme_ns_chr_ioctl,
4056         .compat_ioctl   = compat_ptr_ioctl,
4057         .uring_cmd      = nvme_ns_chr_uring_cmd,
4058         .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
4059 };
4060
4061 static int nvme_add_ns_cdev(struct nvme_ns *ns)
4062 {
4063         int ret;
4064
4065         ns->cdev_device.parent = ns->ctrl->device;
4066         ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
4067                            ns->ctrl->instance, ns->head->instance);
4068         if (ret)
4069                 return ret;
4070
4071         return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
4072                              ns->ctrl->ops->module);
4073 }
4074
4075 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
4076                 struct nvme_ns_info *info)
4077 {
4078         struct nvme_ns_head *head;
4079         size_t size = sizeof(*head);
4080         int ret = -ENOMEM;
4081
4082 #ifdef CONFIG_NVME_MULTIPATH
4083         size += num_possible_nodes() * sizeof(struct nvme_ns *);
4084 #endif
4085
4086         head = kzalloc(size, GFP_KERNEL);
4087         if (!head)
4088                 goto out;
4089         ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
4090         if (ret < 0)
4091                 goto out_free_head;
4092         head->instance = ret;
4093         INIT_LIST_HEAD(&head->list);
4094         ret = init_srcu_struct(&head->srcu);
4095         if (ret)
4096                 goto out_ida_remove;
4097         head->subsys = ctrl->subsys;
4098         head->ns_id = info->nsid;
4099         head->ids = info->ids;
4100         head->shared = info->is_shared;
4101         kref_init(&head->ref);
4102
4103         if (head->ids.csi) {
4104                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
4105                 if (ret)
4106                         goto out_cleanup_srcu;
4107         } else
4108                 head->effects = ctrl->effects;
4109
4110         ret = nvme_mpath_alloc_disk(ctrl, head);
4111         if (ret)
4112                 goto out_cleanup_srcu;
4113
4114         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
4115
4116         kref_get(&ctrl->subsys->ref);
4117
4118         return head;
4119 out_cleanup_srcu:
4120         cleanup_srcu_struct(&head->srcu);
4121 out_ida_remove:
4122         ida_free(&ctrl->subsys->ns_ida, head->instance);
4123 out_free_head:
4124         kfree(head);
4125 out:
4126         if (ret > 0)
4127                 ret = blk_status_to_errno(nvme_error_status(ret));
4128         return ERR_PTR(ret);
4129 }
4130
4131 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
4132                 struct nvme_ns_ids *ids)
4133 {
4134         struct nvme_subsystem *s;
4135         int ret = 0;
4136
4137         /*
4138          * Note that this check is racy as we try to avoid holding the global
4139          * lock over the whole ns_head creation.  But it is only intended as
4140          * a sanity check anyway.
4141          */
4142         mutex_lock(&nvme_subsystems_lock);
4143         list_for_each_entry(s, &nvme_subsystems, entry) {
4144                 if (s == this)
4145                         continue;
4146                 mutex_lock(&s->lock);
4147                 ret = nvme_subsys_check_duplicate_ids(s, ids);
4148                 mutex_unlock(&s->lock);
4149                 if (ret)
4150                         break;
4151         }
4152         mutex_unlock(&nvme_subsystems_lock);
4153
4154         return ret;
4155 }
4156
4157 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
4158 {
4159         struct nvme_ctrl *ctrl = ns->ctrl;
4160         struct nvme_ns_head *head = NULL;
4161         int ret;
4162
4163         ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
4164         if (ret) {
4165                 dev_err(ctrl->device,
4166                         "globally duplicate IDs for nsid %d\n", info->nsid);
4167                 nvme_print_device_info(ctrl);
4168                 return ret;
4169         }
4170
4171         mutex_lock(&ctrl->subsys->lock);
4172         head = nvme_find_ns_head(ctrl, info->nsid);
4173         if (!head) {
4174                 ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids);
4175                 if (ret) {
4176                         dev_err(ctrl->device,
4177                                 "duplicate IDs in subsystem for nsid %d\n",
4178                                 info->nsid);
4179                         goto out_unlock;
4180                 }
4181                 head = nvme_alloc_ns_head(ctrl, info);
4182                 if (IS_ERR(head)) {
4183                         ret = PTR_ERR(head);
4184                         goto out_unlock;
4185                 }
4186         } else {
4187                 ret = -EINVAL;
4188                 if (!info->is_shared || !head->shared) {
4189                         dev_err(ctrl->device,
4190                                 "Duplicate unshared namespace %d\n",
4191                                 info->nsid);
4192                         goto out_put_ns_head;
4193                 }
4194                 if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
4195                         dev_err(ctrl->device,
4196                                 "IDs don't match for shared namespace %d\n",
4197                                         info->nsid);
4198                         goto out_put_ns_head;
4199                 }
4200
4201                 if (!multipath && !list_empty(&head->list)) {
4202                         dev_warn(ctrl->device,
4203                                 "Found shared namespace %d, but multipathing not supported.\n",
4204                                 info->nsid);
4205                         dev_warn_once(ctrl->device,
4206                                 "Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0\n.");
4207                 }
4208         }
4209
4210         list_add_tail_rcu(&ns->siblings, &head->list);
4211         ns->head = head;
4212         mutex_unlock(&ctrl->subsys->lock);
4213         return 0;
4214
4215 out_put_ns_head:
4216         nvme_put_ns_head(head);
4217 out_unlock:
4218         mutex_unlock(&ctrl->subsys->lock);
4219         return ret;
4220 }
4221
4222 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4223 {
4224         struct nvme_ns *ns, *ret = NULL;
4225
4226         down_read(&ctrl->namespaces_rwsem);
4227         list_for_each_entry(ns, &ctrl->namespaces, list) {
4228                 if (ns->head->ns_id == nsid) {
4229                         if (!nvme_get_ns(ns))
4230                                 continue;
4231                         ret = ns;
4232                         break;
4233                 }
4234                 if (ns->head->ns_id > nsid)
4235                         break;
4236         }
4237         up_read(&ctrl->namespaces_rwsem);
4238         return ret;
4239 }
4240 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
4241
4242 /*
4243  * Add the namespace to the controller list while keeping the list ordered.
4244  */
4245 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
4246 {
4247         struct nvme_ns *tmp;
4248
4249         list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
4250                 if (tmp->head->ns_id < ns->head->ns_id) {
4251                         list_add(&ns->list, &tmp->list);
4252                         return;
4253                 }
4254         }
4255         list_add(&ns->list, &ns->ctrl->namespaces);
4256 }
4257
4258 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
4259 {
4260         struct nvme_ns *ns;
4261         struct gendisk *disk;
4262         int node = ctrl->numa_node;
4263
4264         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
4265         if (!ns)
4266                 return;
4267
4268         disk = blk_mq_alloc_disk(ctrl->tagset, ns);
4269         if (IS_ERR(disk))
4270                 goto out_free_ns;
4271         disk->fops = &nvme_bdev_ops;
4272         disk->private_data = ns;
4273
4274         ns->disk = disk;
4275         ns->queue = disk->queue;
4276
4277         if (ctrl->opts && ctrl->opts->data_digest)
4278                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
4279
4280         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
4281         if (ctrl->ops->supports_pci_p2pdma &&
4282             ctrl->ops->supports_pci_p2pdma(ctrl))
4283                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
4284
4285         ns->ctrl = ctrl;
4286         kref_init(&ns->kref);
4287
4288         if (nvme_init_ns_head(ns, info))
4289                 goto out_cleanup_disk;
4290
4291         /*
4292          * If multipathing is enabled, the device name for all disks and not
4293          * just those that represent shared namespaces needs to be based on the
4294          * subsystem instance.  Using the controller instance for private
4295          * namespaces could lead to naming collisions between shared and private
4296          * namespaces if they don't use a common numbering scheme.
4297          *
4298          * If multipathing is not enabled, disk names must use the controller
4299          * instance as shared namespaces will show up as multiple block
4300          * devices.
4301          */
4302         if (ns->head->disk) {
4303                 sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
4304                         ctrl->instance, ns->head->instance);
4305                 disk->flags |= GENHD_FL_HIDDEN;
4306         } else if (multipath) {
4307                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
4308                         ns->head->instance);
4309         } else {
4310                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
4311                         ns->head->instance);
4312         }
4313
4314         if (nvme_update_ns_info(ns, info))
4315                 goto out_unlink_ns;
4316
4317         down_write(&ctrl->namespaces_rwsem);
4318         nvme_ns_add_to_ctrl_list(ns);
4319         up_write(&ctrl->namespaces_rwsem);
4320         nvme_get_ctrl(ctrl);
4321
4322         if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups))
4323                 goto out_cleanup_ns_from_list;
4324
4325         if (!nvme_ns_head_multipath(ns->head))
4326                 nvme_add_ns_cdev(ns);
4327
4328         nvme_mpath_add_disk(ns, info->anagrpid);
4329         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
4330
4331         return;
4332
4333  out_cleanup_ns_from_list:
4334         nvme_put_ctrl(ctrl);
4335         down_write(&ctrl->namespaces_rwsem);
4336         list_del_init(&ns->list);
4337         up_write(&ctrl->namespaces_rwsem);
4338  out_unlink_ns:
4339         mutex_lock(&ctrl->subsys->lock);
4340         list_del_rcu(&ns->siblings);
4341         if (list_empty(&ns->head->list))
4342                 list_del_init(&ns->head->entry);
4343         mutex_unlock(&ctrl->subsys->lock);
4344         nvme_put_ns_head(ns->head);
4345  out_cleanup_disk:
4346         put_disk(disk);
4347  out_free_ns:
4348         kfree(ns);
4349 }
4350
4351 static void nvme_ns_remove(struct nvme_ns *ns)
4352 {
4353         bool last_path = false;
4354
4355         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4356                 return;
4357
4358         clear_bit(NVME_NS_READY, &ns->flags);
4359         set_capacity(ns->disk, 0);
4360         nvme_fault_inject_fini(&ns->fault_inject);
4361
4362         /*
4363          * Ensure that !NVME_NS_READY is seen by other threads to prevent
4364          * this ns going back into current_path.
4365          */
4366         synchronize_srcu(&ns->head->srcu);
4367
4368         /* wait for concurrent submissions */
4369         if (nvme_mpath_clear_current_path(ns))
4370                 synchronize_srcu(&ns->head->srcu);
4371
4372         mutex_lock(&ns->ctrl->subsys->lock);
4373         list_del_rcu(&ns->siblings);
4374         if (list_empty(&ns->head->list)) {
4375                 list_del_init(&ns->head->entry);
4376                 last_path = true;
4377         }
4378         mutex_unlock(&ns->ctrl->subsys->lock);
4379
4380         /* guarantee not available in head->list */
4381         synchronize_srcu(&ns->head->srcu);
4382
4383         if (!nvme_ns_head_multipath(ns->head))
4384                 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
4385         del_gendisk(ns->disk);
4386
4387         down_write(&ns->ctrl->namespaces_rwsem);
4388         list_del_init(&ns->list);
4389         up_write(&ns->ctrl->namespaces_rwsem);
4390
4391         if (last_path)
4392                 nvme_mpath_shutdown_disk(ns->head);
4393         nvme_put_ns(ns);
4394 }
4395
4396 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4397 {
4398         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4399
4400         if (ns) {
4401                 nvme_ns_remove(ns);
4402                 nvme_put_ns(ns);
4403         }
4404 }
4405
4406 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info)
4407 {
4408         int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4409
4410         if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) {
4411                 dev_err(ns->ctrl->device,
4412                         "identifiers changed for nsid %d\n", ns->head->ns_id);
4413                 goto out;
4414         }
4415
4416         ret = nvme_update_ns_info(ns, info);
4417 out:
4418         /*
4419          * Only remove the namespace if we got a fatal error back from the
4420          * device, otherwise ignore the error and just move on.
4421          *
4422          * TODO: we should probably schedule a delayed retry here.
4423          */
4424         if (ret > 0 && (ret & NVME_SC_DNR))
4425                 nvme_ns_remove(ns);
4426 }
4427
4428 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4429 {
4430         struct nvme_ns_info info = { .nsid = nsid };
4431         struct nvme_ns *ns;
4432         int ret;
4433
4434         if (nvme_identify_ns_descs(ctrl, &info))
4435                 return;
4436
4437         if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) {
4438                 dev_warn(ctrl->device,
4439                         "command set not reported for nsid: %d\n", nsid);
4440                 return;
4441         }
4442
4443         /*
4444          * If available try to use the Command Set Idependent Identify Namespace
4445          * data structure to find all the generic information that is needed to
4446          * set up a namespace.  If not fall back to the legacy version.
4447          */
4448         if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) ||
4449             (info.ids.csi != NVME_CSI_NVM && info.ids.csi != NVME_CSI_ZNS))
4450                 ret = nvme_ns_info_from_id_cs_indep(ctrl, &info);
4451         else
4452                 ret = nvme_ns_info_from_identify(ctrl, &info);
4453
4454         if (info.is_removed)
4455                 nvme_ns_remove_by_nsid(ctrl, nsid);
4456
4457         /*
4458          * Ignore the namespace if it is not ready. We will get an AEN once it
4459          * becomes ready and restart the scan.
4460          */
4461         if (ret || !info.is_ready)
4462                 return;
4463
4464         ns = nvme_find_get_ns(ctrl, nsid);
4465         if (ns) {
4466                 nvme_validate_ns(ns, &info);
4467                 nvme_put_ns(ns);
4468         } else {
4469                 nvme_alloc_ns(ctrl, &info);
4470         }
4471 }
4472
4473 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4474                                         unsigned nsid)
4475 {
4476         struct nvme_ns *ns, *next;
4477         LIST_HEAD(rm_list);
4478
4479         down_write(&ctrl->namespaces_rwsem);
4480         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4481                 if (ns->head->ns_id > nsid)
4482                         list_move_tail(&ns->list, &rm_list);
4483         }
4484         up_write(&ctrl->namespaces_rwsem);
4485
4486         list_for_each_entry_safe(ns, next, &rm_list, list)
4487                 nvme_ns_remove(ns);
4488
4489 }
4490
4491 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4492 {
4493         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4494         __le32 *ns_list;
4495         u32 prev = 0;
4496         int ret = 0, i;
4497
4498         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4499         if (!ns_list)
4500                 return -ENOMEM;
4501
4502         for (;;) {
4503                 struct nvme_command cmd = {
4504                         .identify.opcode        = nvme_admin_identify,
4505                         .identify.cns           = NVME_ID_CNS_NS_ACTIVE_LIST,
4506                         .identify.nsid          = cpu_to_le32(prev),
4507                 };
4508
4509                 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4510                                             NVME_IDENTIFY_DATA_SIZE);
4511                 if (ret) {
4512                         dev_warn(ctrl->device,
4513                                 "Identify NS List failed (status=0x%x)\n", ret);
4514                         goto free;
4515                 }
4516
4517                 for (i = 0; i < nr_entries; i++) {
4518                         u32 nsid = le32_to_cpu(ns_list[i]);
4519
4520                         if (!nsid)      /* end of the list? */
4521                                 goto out;
4522                         nvme_scan_ns(ctrl, nsid);
4523                         while (++prev < nsid)
4524                                 nvme_ns_remove_by_nsid(ctrl, prev);
4525                 }
4526         }
4527  out:
4528         nvme_remove_invalid_namespaces(ctrl, prev);
4529  free:
4530         kfree(ns_list);
4531         return ret;
4532 }
4533
4534 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4535 {
4536         struct nvme_id_ctrl *id;
4537         u32 nn, i;
4538
4539         if (nvme_identify_ctrl(ctrl, &id))
4540                 return;
4541         nn = le32_to_cpu(id->nn);
4542         kfree(id);
4543
4544         for (i = 1; i <= nn; i++)
4545                 nvme_scan_ns(ctrl, i);
4546
4547         nvme_remove_invalid_namespaces(ctrl, nn);
4548 }
4549
4550 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4551 {
4552         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4553         __le32 *log;
4554         int error;
4555
4556         log = kzalloc(log_size, GFP_KERNEL);
4557         if (!log)
4558                 return;
4559
4560         /*
4561          * We need to read the log to clear the AEN, but we don't want to rely
4562          * on it for the changed namespace information as userspace could have
4563          * raced with us in reading the log page, which could cause us to miss
4564          * updates.
4565          */
4566         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4567                         NVME_CSI_NVM, log, log_size, 0);
4568         if (error)
4569                 dev_warn(ctrl->device,
4570                         "reading changed ns log failed: %d\n", error);
4571
4572         kfree(log);
4573 }
4574
4575 static void nvme_scan_work(struct work_struct *work)
4576 {
4577         struct nvme_ctrl *ctrl =
4578                 container_of(work, struct nvme_ctrl, scan_work);
4579         int ret;
4580
4581         /* No tagset on a live ctrl means IO queues could not created */
4582         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4583                 return;
4584
4585         /*
4586          * Identify controller limits can change at controller reset due to
4587          * new firmware download, even though it is not common we cannot ignore
4588          * such scenario. Controller's non-mdts limits are reported in the unit
4589          * of logical blocks that is dependent on the format of attached
4590          * namespace. Hence re-read the limits at the time of ns allocation.
4591          */
4592         ret = nvme_init_non_mdts_limits(ctrl);
4593         if (ret < 0) {
4594                 dev_warn(ctrl->device,
4595                         "reading non-mdts-limits failed: %d\n", ret);
4596                 return;
4597         }
4598
4599         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4600                 dev_info(ctrl->device, "rescanning namespaces.\n");
4601                 nvme_clear_changed_ns_log(ctrl);
4602         }
4603
4604         mutex_lock(&ctrl->scan_lock);
4605         if (nvme_ctrl_limited_cns(ctrl)) {
4606                 nvme_scan_ns_sequential(ctrl);
4607         } else {
4608                 /*
4609                  * Fall back to sequential scan if DNR is set to handle broken
4610                  * devices which should support Identify NS List (as per the VS
4611                  * they report) but don't actually support it.
4612                  */
4613                 ret = nvme_scan_ns_list(ctrl);
4614                 if (ret > 0 && ret & NVME_SC_DNR)
4615                         nvme_scan_ns_sequential(ctrl);
4616         }
4617         mutex_unlock(&ctrl->scan_lock);
4618 }
4619
4620 /*
4621  * This function iterates the namespace list unlocked to allow recovery from
4622  * controller failure. It is up to the caller to ensure the namespace list is
4623  * not modified by scan work while this function is executing.
4624  */
4625 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4626 {
4627         struct nvme_ns *ns, *next;
4628         LIST_HEAD(ns_list);
4629
4630         /*
4631          * make sure to requeue I/O to all namespaces as these
4632          * might result from the scan itself and must complete
4633          * for the scan_work to make progress
4634          */
4635         nvme_mpath_clear_ctrl_paths(ctrl);
4636
4637         /* prevent racing with ns scanning */
4638         flush_work(&ctrl->scan_work);
4639
4640         /*
4641          * The dead states indicates the controller was not gracefully
4642          * disconnected. In that case, we won't be able to flush any data while
4643          * removing the namespaces' disks; fail all the queues now to avoid
4644          * potentially having to clean up the failed sync later.
4645          */
4646         if (ctrl->state == NVME_CTRL_DEAD) {
4647                 nvme_mark_namespaces_dead(ctrl);
4648                 nvme_unquiesce_io_queues(ctrl);
4649         }
4650
4651         /* this is a no-op when called from the controller reset handler */
4652         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4653
4654         down_write(&ctrl->namespaces_rwsem);
4655         list_splice_init(&ctrl->namespaces, &ns_list);
4656         up_write(&ctrl->namespaces_rwsem);
4657
4658         list_for_each_entry_safe(ns, next, &ns_list, list)
4659                 nvme_ns_remove(ns);
4660 }
4661 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4662
4663 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)
4664 {
4665         const struct nvme_ctrl *ctrl =
4666                 container_of(dev, struct nvme_ctrl, ctrl_device);
4667         struct nvmf_ctrl_options *opts = ctrl->opts;
4668         int ret;
4669
4670         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4671         if (ret)
4672                 return ret;
4673
4674         if (opts) {
4675                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4676                 if (ret)
4677                         return ret;
4678
4679                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4680                                 opts->trsvcid ?: "none");
4681                 if (ret)
4682                         return ret;
4683
4684                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4685                                 opts->host_traddr ?: "none");
4686                 if (ret)
4687                         return ret;
4688
4689                 ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4690                                 opts->host_iface ?: "none");
4691         }
4692         return ret;
4693 }
4694
4695 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4696 {
4697         char *envp[2] = { envdata, NULL };
4698
4699         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4700 }
4701
4702 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4703 {
4704         char *envp[2] = { NULL, NULL };
4705         u32 aen_result = ctrl->aen_result;
4706
4707         ctrl->aen_result = 0;
4708         if (!aen_result)
4709                 return;
4710
4711         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4712         if (!envp[0])
4713                 return;
4714         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4715         kfree(envp[0]);
4716 }
4717
4718 static void nvme_async_event_work(struct work_struct *work)
4719 {
4720         struct nvme_ctrl *ctrl =
4721                 container_of(work, struct nvme_ctrl, async_event_work);
4722
4723         nvme_aen_uevent(ctrl);
4724
4725         /*
4726          * The transport drivers must guarantee AER submission here is safe by
4727          * flushing ctrl async_event_work after changing the controller state
4728          * from LIVE and before freeing the admin queue.
4729         */
4730         if (ctrl->state == NVME_CTRL_LIVE)
4731                 ctrl->ops->submit_async_event(ctrl);
4732 }
4733
4734 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4735 {
4736
4737         u32 csts;
4738
4739         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4740                 return false;
4741
4742         if (csts == ~0)
4743                 return false;
4744
4745         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4746 }
4747
4748 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4749 {
4750         struct nvme_fw_slot_info_log *log;
4751
4752         log = kmalloc(sizeof(*log), GFP_KERNEL);
4753         if (!log)
4754                 return;
4755
4756         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4757                         log, sizeof(*log), 0))
4758                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4759         kfree(log);
4760 }
4761
4762 static void nvme_fw_act_work(struct work_struct *work)
4763 {
4764         struct nvme_ctrl *ctrl = container_of(work,
4765                                 struct nvme_ctrl, fw_act_work);
4766         unsigned long fw_act_timeout;
4767
4768         if (ctrl->mtfa)
4769                 fw_act_timeout = jiffies +
4770                                 msecs_to_jiffies(ctrl->mtfa * 100);
4771         else
4772                 fw_act_timeout = jiffies +
4773                                 msecs_to_jiffies(admin_timeout * 1000);
4774
4775         nvme_quiesce_io_queues(ctrl);
4776         while (nvme_ctrl_pp_status(ctrl)) {
4777                 if (time_after(jiffies, fw_act_timeout)) {
4778                         dev_warn(ctrl->device,
4779                                 "Fw activation timeout, reset controller\n");
4780                         nvme_try_sched_reset(ctrl);
4781                         return;
4782                 }
4783                 msleep(100);
4784         }
4785
4786         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4787                 return;
4788
4789         nvme_unquiesce_io_queues(ctrl);
4790         /* read FW slot information to clear the AER */
4791         nvme_get_fw_slot_info(ctrl);
4792
4793         queue_work(nvme_wq, &ctrl->async_event_work);
4794 }
4795
4796 static u32 nvme_aer_type(u32 result)
4797 {
4798         return result & 0x7;
4799 }
4800
4801 static u32 nvme_aer_subtype(u32 result)
4802 {
4803         return (result & 0xff00) >> 8;
4804 }
4805
4806 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4807 {
4808         u32 aer_notice_type = nvme_aer_subtype(result);
4809         bool requeue = true;
4810
4811         switch (aer_notice_type) {
4812         case NVME_AER_NOTICE_NS_CHANGED:
4813                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4814                 nvme_queue_scan(ctrl);
4815                 break;
4816         case NVME_AER_NOTICE_FW_ACT_STARTING:
4817                 /*
4818                  * We are (ab)using the RESETTING state to prevent subsequent
4819                  * recovery actions from interfering with the controller's
4820                  * firmware activation.
4821                  */
4822                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
4823                         nvme_auth_stop(ctrl);
4824                         requeue = false;
4825                         queue_work(nvme_wq, &ctrl->fw_act_work);
4826                 }
4827                 break;
4828 #ifdef CONFIG_NVME_MULTIPATH
4829         case NVME_AER_NOTICE_ANA:
4830                 if (!ctrl->ana_log_buf)
4831                         break;
4832                 queue_work(nvme_wq, &ctrl->ana_work);
4833                 break;
4834 #endif
4835         case NVME_AER_NOTICE_DISC_CHANGED:
4836                 ctrl->aen_result = result;
4837                 break;
4838         default:
4839                 dev_warn(ctrl->device, "async event result %08x\n", result);
4840         }
4841         return requeue;
4842 }
4843
4844 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4845 {
4846         dev_warn(ctrl->device, "resetting controller due to AER\n");
4847         nvme_reset_ctrl(ctrl);
4848 }
4849
4850 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4851                 volatile union nvme_result *res)
4852 {
4853         u32 result = le32_to_cpu(res->u32);
4854         u32 aer_type = nvme_aer_type(result);
4855         u32 aer_subtype = nvme_aer_subtype(result);
4856         bool requeue = true;
4857
4858         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4859                 return;
4860
4861         trace_nvme_async_event(ctrl, result);
4862         switch (aer_type) {
4863         case NVME_AER_NOTICE:
4864                 requeue = nvme_handle_aen_notice(ctrl, result);
4865                 break;
4866         case NVME_AER_ERROR:
4867                 /*
4868                  * For a persistent internal error, don't run async_event_work
4869                  * to submit a new AER. The controller reset will do it.
4870                  */
4871                 if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4872                         nvme_handle_aer_persistent_error(ctrl);
4873                         return;
4874                 }
4875                 fallthrough;
4876         case NVME_AER_SMART:
4877         case NVME_AER_CSS:
4878         case NVME_AER_VS:
4879                 ctrl->aen_result = result;
4880                 break;
4881         default:
4882                 break;
4883         }
4884
4885         if (requeue)
4886                 queue_work(nvme_wq, &ctrl->async_event_work);
4887 }
4888 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4889
4890 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4891                 const struct blk_mq_ops *ops, unsigned int cmd_size)
4892 {
4893         int ret;
4894
4895         memset(set, 0, sizeof(*set));
4896         set->ops = ops;
4897         set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
4898         if (ctrl->ops->flags & NVME_F_FABRICS)
4899                 set->reserved_tags = NVMF_RESERVED_TAGS;
4900         set->numa_node = ctrl->numa_node;
4901         set->flags = BLK_MQ_F_NO_SCHED;
4902         if (ctrl->ops->flags & NVME_F_BLOCKING)
4903                 set->flags |= BLK_MQ_F_BLOCKING;
4904         set->cmd_size = cmd_size;
4905         set->driver_data = ctrl;
4906         set->nr_hw_queues = 1;
4907         set->timeout = NVME_ADMIN_TIMEOUT;
4908         ret = blk_mq_alloc_tag_set(set);
4909         if (ret)
4910                 return ret;
4911
4912         ctrl->admin_q = blk_mq_init_queue(set);
4913         if (IS_ERR(ctrl->admin_q)) {
4914                 ret = PTR_ERR(ctrl->admin_q);
4915                 goto out_free_tagset;
4916         }
4917
4918         if (ctrl->ops->flags & NVME_F_FABRICS) {
4919                 ctrl->fabrics_q = blk_mq_init_queue(set);
4920                 if (IS_ERR(ctrl->fabrics_q)) {
4921                         ret = PTR_ERR(ctrl->fabrics_q);
4922                         goto out_cleanup_admin_q;
4923                 }
4924         }
4925
4926         ctrl->admin_tagset = set;
4927         return 0;
4928
4929 out_cleanup_admin_q:
4930         blk_mq_destroy_queue(ctrl->admin_q);
4931         blk_put_queue(ctrl->admin_q);
4932 out_free_tagset:
4933         blk_mq_free_tag_set(set);
4934         ctrl->admin_q = NULL;
4935         ctrl->fabrics_q = NULL;
4936         return ret;
4937 }
4938 EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set);
4939
4940 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
4941 {
4942         blk_mq_destroy_queue(ctrl->admin_q);
4943         blk_put_queue(ctrl->admin_q);
4944         if (ctrl->ops->flags & NVME_F_FABRICS) {
4945                 blk_mq_destroy_queue(ctrl->fabrics_q);
4946                 blk_put_queue(ctrl->fabrics_q);
4947         }
4948         blk_mq_free_tag_set(ctrl->admin_tagset);
4949 }
4950 EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
4951
4952 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4953                 const struct blk_mq_ops *ops, unsigned int nr_maps,
4954                 unsigned int cmd_size)
4955 {
4956         int ret;
4957
4958         memset(set, 0, sizeof(*set));
4959         set->ops = ops;
4960         set->queue_depth = min_t(unsigned, ctrl->sqsize, BLK_MQ_MAX_DEPTH - 1);
4961         /*
4962          * Some Apple controllers requires tags to be unique across admin and
4963          * the (only) I/O queue, so reserve the first 32 tags of the I/O queue.
4964          */
4965         if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
4966                 set->reserved_tags = NVME_AQ_DEPTH;
4967         else if (ctrl->ops->flags & NVME_F_FABRICS)
4968                 set->reserved_tags = NVMF_RESERVED_TAGS;
4969         set->numa_node = ctrl->numa_node;
4970         set->flags = BLK_MQ_F_SHOULD_MERGE;
4971         if (ctrl->ops->flags & NVME_F_BLOCKING)
4972                 set->flags |= BLK_MQ_F_BLOCKING;
4973         set->cmd_size = cmd_size,
4974         set->driver_data = ctrl;
4975         set->nr_hw_queues = ctrl->queue_count - 1;
4976         set->timeout = NVME_IO_TIMEOUT;
4977         set->nr_maps = nr_maps;
4978         ret = blk_mq_alloc_tag_set(set);
4979         if (ret)
4980                 return ret;
4981
4982         if (ctrl->ops->flags & NVME_F_FABRICS) {
4983                 ctrl->connect_q = blk_mq_init_queue(set);
4984                 if (IS_ERR(ctrl->connect_q)) {
4985                         ret = PTR_ERR(ctrl->connect_q);
4986                         goto out_free_tag_set;
4987                 }
4988                 blk_queue_flag_set(QUEUE_FLAG_SKIP_TAGSET_QUIESCE,
4989                                    ctrl->connect_q);
4990         }
4991
4992         ctrl->tagset = set;
4993         return 0;
4994
4995 out_free_tag_set:
4996         blk_mq_free_tag_set(set);
4997         ctrl->connect_q = NULL;
4998         return ret;
4999 }
5000 EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set);
5001
5002 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)
5003 {
5004         if (ctrl->ops->flags & NVME_F_FABRICS) {
5005                 blk_mq_destroy_queue(ctrl->connect_q);
5006                 blk_put_queue(ctrl->connect_q);
5007         }
5008         blk_mq_free_tag_set(ctrl->tagset);
5009 }
5010 EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
5011
5012 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
5013 {
5014         nvme_mpath_stop(ctrl);
5015         nvme_auth_stop(ctrl);
5016         nvme_stop_keep_alive(ctrl);
5017         nvme_stop_failfast_work(ctrl);
5018         flush_work(&ctrl->async_event_work);
5019         cancel_work_sync(&ctrl->fw_act_work);
5020         if (ctrl->ops->stop_ctrl)
5021                 ctrl->ops->stop_ctrl(ctrl);
5022 }
5023 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
5024
5025 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
5026 {
5027         nvme_start_keep_alive(ctrl);
5028
5029         nvme_enable_aen(ctrl);
5030
5031         /*
5032          * persistent discovery controllers need to send indication to userspace
5033          * to re-read the discovery log page to learn about possible changes
5034          * that were missed. We identify persistent discovery controllers by
5035          * checking that they started once before, hence are reconnecting back.
5036          */
5037         if (test_and_set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
5038             nvme_discovery_ctrl(ctrl))
5039                 nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
5040
5041         if (ctrl->queue_count > 1) {
5042                 nvme_queue_scan(ctrl);
5043                 nvme_unquiesce_io_queues(ctrl);
5044                 nvme_mpath_update(ctrl);
5045         }
5046
5047         nvme_change_uevent(ctrl, "NVME_EVENT=connected");
5048 }
5049 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
5050
5051 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
5052 {
5053         nvme_hwmon_exit(ctrl);
5054         nvme_fault_inject_fini(&ctrl->fault_inject);
5055         dev_pm_qos_hide_latency_tolerance(ctrl->device);
5056         cdev_device_del(&ctrl->cdev, ctrl->device);
5057         nvme_put_ctrl(ctrl);
5058 }
5059 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
5060
5061 static void nvme_free_cels(struct nvme_ctrl *ctrl)
5062 {
5063         struct nvme_effects_log *cel;
5064         unsigned long i;
5065
5066         xa_for_each(&ctrl->cels, i, cel) {
5067                 xa_erase(&ctrl->cels, i);
5068                 kfree(cel);
5069         }
5070
5071         xa_destroy(&ctrl->cels);
5072 }
5073
5074 static void nvme_free_ctrl(struct device *dev)
5075 {
5076         struct nvme_ctrl *ctrl =
5077                 container_of(dev, struct nvme_ctrl, ctrl_device);
5078         struct nvme_subsystem *subsys = ctrl->subsys;
5079
5080         if (!subsys || ctrl->instance != subsys->instance)
5081                 ida_free(&nvme_instance_ida, ctrl->instance);
5082
5083         nvme_free_cels(ctrl);
5084         nvme_mpath_uninit(ctrl);
5085         nvme_auth_stop(ctrl);
5086         nvme_auth_free(ctrl);
5087         __free_page(ctrl->discard_page);
5088         free_opal_dev(ctrl->opal_dev);
5089
5090         if (subsys) {
5091                 mutex_lock(&nvme_subsystems_lock);
5092                 list_del(&ctrl->subsys_entry);
5093                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
5094                 mutex_unlock(&nvme_subsystems_lock);
5095         }
5096
5097         ctrl->ops->free_ctrl(ctrl);
5098
5099         if (subsys)
5100                 nvme_put_subsystem(subsys);
5101 }
5102
5103 /*
5104  * Initialize a NVMe controller structures.  This needs to be called during
5105  * earliest initialization so that we have the initialized structured around
5106  * during probing.
5107  */
5108 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
5109                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
5110 {
5111         int ret;
5112
5113         ctrl->state = NVME_CTRL_NEW;
5114         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
5115         spin_lock_init(&ctrl->lock);
5116         mutex_init(&ctrl->scan_lock);
5117         INIT_LIST_HEAD(&ctrl->namespaces);
5118         xa_init(&ctrl->cels);
5119         init_rwsem(&ctrl->namespaces_rwsem);
5120         ctrl->dev = dev;
5121         ctrl->ops = ops;
5122         ctrl->quirks = quirks;
5123         ctrl->numa_node = NUMA_NO_NODE;
5124         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
5125         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
5126         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
5127         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
5128         init_waitqueue_head(&ctrl->state_wq);
5129
5130         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
5131         INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
5132         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
5133         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
5134
5135         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
5136                         PAGE_SIZE);
5137         ctrl->discard_page = alloc_page(GFP_KERNEL);
5138         if (!ctrl->discard_page) {
5139                 ret = -ENOMEM;
5140                 goto out;
5141         }
5142
5143         ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
5144         if (ret < 0)
5145                 goto out;
5146         ctrl->instance = ret;
5147
5148         device_initialize(&ctrl->ctrl_device);
5149         ctrl->device = &ctrl->ctrl_device;
5150         ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
5151                         ctrl->instance);
5152         ctrl->device->class = nvme_class;
5153         ctrl->device->parent = ctrl->dev;
5154         if (ops->dev_attr_groups)
5155                 ctrl->device->groups = ops->dev_attr_groups;
5156         else
5157                 ctrl->device->groups = nvme_dev_attr_groups;
5158         ctrl->device->release = nvme_free_ctrl;
5159         dev_set_drvdata(ctrl->device, ctrl);
5160         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
5161         if (ret)
5162                 goto out_release_instance;
5163
5164         nvme_get_ctrl(ctrl);
5165         cdev_init(&ctrl->cdev, &nvme_dev_fops);
5166         ctrl->cdev.owner = ops->module;
5167         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
5168         if (ret)
5169                 goto out_free_name;
5170
5171         /*
5172          * Initialize latency tolerance controls.  The sysfs files won't
5173          * be visible to userspace unless the device actually supports APST.
5174          */
5175         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
5176         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
5177                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
5178
5179         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
5180         nvme_mpath_init_ctrl(ctrl);
5181         ret = nvme_auth_init_ctrl(ctrl);
5182         if (ret)
5183                 goto out_free_cdev;
5184
5185         return 0;
5186 out_free_cdev:
5187         cdev_device_del(&ctrl->cdev, ctrl->device);
5188 out_free_name:
5189         nvme_put_ctrl(ctrl);
5190         kfree_const(ctrl->device->kobj.name);
5191 out_release_instance:
5192         ida_free(&nvme_instance_ida, ctrl->instance);
5193 out:
5194         if (ctrl->discard_page)
5195                 __free_page(ctrl->discard_page);
5196         return ret;
5197 }
5198 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
5199
5200 /* let I/O to all namespaces fail in preparation for surprise removal */
5201 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl)
5202 {
5203         struct nvme_ns *ns;
5204
5205         down_read(&ctrl->namespaces_rwsem);
5206         list_for_each_entry(ns, &ctrl->namespaces, list)
5207                 blk_mark_disk_dead(ns->disk);
5208         up_read(&ctrl->namespaces_rwsem);
5209 }
5210 EXPORT_SYMBOL_GPL(nvme_mark_namespaces_dead);
5211
5212 void nvme_unfreeze(struct nvme_ctrl *ctrl)
5213 {
5214         struct nvme_ns *ns;
5215
5216         down_read(&ctrl->namespaces_rwsem);
5217         list_for_each_entry(ns, &ctrl->namespaces, list)
5218                 blk_mq_unfreeze_queue(ns->queue);
5219         up_read(&ctrl->namespaces_rwsem);
5220 }
5221 EXPORT_SYMBOL_GPL(nvme_unfreeze);
5222
5223 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
5224 {
5225         struct nvme_ns *ns;
5226
5227         down_read(&ctrl->namespaces_rwsem);
5228         list_for_each_entry(ns, &ctrl->namespaces, list) {
5229                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
5230                 if (timeout <= 0)
5231                         break;
5232         }
5233         up_read(&ctrl->namespaces_rwsem);
5234         return timeout;
5235 }
5236 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
5237
5238 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
5239 {
5240         struct nvme_ns *ns;
5241
5242         down_read(&ctrl->namespaces_rwsem);
5243         list_for_each_entry(ns, &ctrl->namespaces, list)
5244                 blk_mq_freeze_queue_wait(ns->queue);
5245         up_read(&ctrl->namespaces_rwsem);
5246 }
5247 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
5248
5249 void nvme_start_freeze(struct nvme_ctrl *ctrl)
5250 {
5251         struct nvme_ns *ns;
5252
5253         down_read(&ctrl->namespaces_rwsem);
5254         list_for_each_entry(ns, &ctrl->namespaces, list)
5255                 blk_freeze_queue_start(ns->queue);
5256         up_read(&ctrl->namespaces_rwsem);
5257 }
5258 EXPORT_SYMBOL_GPL(nvme_start_freeze);
5259
5260 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)
5261 {
5262         if (!ctrl->tagset)
5263                 return;
5264         if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))
5265                 blk_mq_quiesce_tagset(ctrl->tagset);
5266         else
5267                 blk_mq_wait_quiesce_done(ctrl->tagset);
5268 }
5269 EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);
5270
5271 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)
5272 {
5273         if (!ctrl->tagset)
5274                 return;
5275         if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))
5276                 blk_mq_unquiesce_tagset(ctrl->tagset);
5277 }
5278 EXPORT_SYMBOL_GPL(nvme_unquiesce_io_queues);
5279
5280 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl)
5281 {
5282         if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5283                 blk_mq_quiesce_queue(ctrl->admin_q);
5284         else
5285                 blk_mq_wait_quiesce_done(ctrl->admin_q->tag_set);
5286 }
5287 EXPORT_SYMBOL_GPL(nvme_quiesce_admin_queue);
5288
5289 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl)
5290 {
5291         if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5292                 blk_mq_unquiesce_queue(ctrl->admin_q);
5293 }
5294 EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);
5295
5296 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
5297 {
5298         struct nvme_ns *ns;
5299
5300         down_read(&ctrl->namespaces_rwsem);
5301         list_for_each_entry(ns, &ctrl->namespaces, list)
5302                 blk_sync_queue(ns->queue);
5303         up_read(&ctrl->namespaces_rwsem);
5304 }
5305 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
5306
5307 void nvme_sync_queues(struct nvme_ctrl *ctrl)
5308 {
5309         nvme_sync_io_queues(ctrl);
5310         if (ctrl->admin_q)
5311                 blk_sync_queue(ctrl->admin_q);
5312 }
5313 EXPORT_SYMBOL_GPL(nvme_sync_queues);
5314
5315 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
5316 {
5317         if (file->f_op != &nvme_dev_fops)
5318                 return NULL;
5319         return file->private_data;
5320 }
5321 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
5322
5323 /*
5324  * Check we didn't inadvertently grow the command structure sizes:
5325  */
5326 static inline void _nvme_check_size(void)
5327 {
5328         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
5329         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
5330         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
5331         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
5332         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
5333         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
5334         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
5335         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
5336         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
5337         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
5338         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
5339         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
5340         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
5341         BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
5342                         NVME_IDENTIFY_DATA_SIZE);
5343         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
5344         BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
5345         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
5346         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
5347         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
5348         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
5349         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
5350         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
5351         BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
5352 }
5353
5354
5355 static int __init nvme_core_init(void)
5356 {
5357         int result = -ENOMEM;
5358
5359         _nvme_check_size();
5360
5361         nvme_wq = alloc_workqueue("nvme-wq",
5362                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5363         if (!nvme_wq)
5364                 goto out;
5365
5366         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
5367                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5368         if (!nvme_reset_wq)
5369                 goto destroy_wq;
5370
5371         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
5372                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5373         if (!nvme_delete_wq)
5374                 goto destroy_reset_wq;
5375
5376         result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
5377                         NVME_MINORS, "nvme");
5378         if (result < 0)
5379                 goto destroy_delete_wq;
5380
5381         nvme_class = class_create(THIS_MODULE, "nvme");
5382         if (IS_ERR(nvme_class)) {
5383                 result = PTR_ERR(nvme_class);
5384                 goto unregister_chrdev;
5385         }
5386         nvme_class->dev_uevent = nvme_class_uevent;
5387
5388         nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
5389         if (IS_ERR(nvme_subsys_class)) {
5390                 result = PTR_ERR(nvme_subsys_class);
5391                 goto destroy_class;
5392         }
5393
5394         result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
5395                                      "nvme-generic");
5396         if (result < 0)
5397                 goto destroy_subsys_class;
5398
5399         nvme_ns_chr_class = class_create(THIS_MODULE, "nvme-generic");
5400         if (IS_ERR(nvme_ns_chr_class)) {
5401                 result = PTR_ERR(nvme_ns_chr_class);
5402                 goto unregister_generic_ns;
5403         }
5404
5405         result = nvme_init_auth();
5406         if (result)
5407                 goto destroy_ns_chr;
5408         return 0;
5409
5410 destroy_ns_chr:
5411         class_destroy(nvme_ns_chr_class);
5412 unregister_generic_ns:
5413         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5414 destroy_subsys_class:
5415         class_destroy(nvme_subsys_class);
5416 destroy_class:
5417         class_destroy(nvme_class);
5418 unregister_chrdev:
5419         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5420 destroy_delete_wq:
5421         destroy_workqueue(nvme_delete_wq);
5422 destroy_reset_wq:
5423         destroy_workqueue(nvme_reset_wq);
5424 destroy_wq:
5425         destroy_workqueue(nvme_wq);
5426 out:
5427         return result;
5428 }
5429
5430 static void __exit nvme_core_exit(void)
5431 {
5432         nvme_exit_auth();
5433         class_destroy(nvme_ns_chr_class);
5434         class_destroy(nvme_subsys_class);
5435         class_destroy(nvme_class);
5436         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5437         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5438         destroy_workqueue(nvme_delete_wq);
5439         destroy_workqueue(nvme_reset_wq);
5440         destroy_workqueue(nvme_wq);
5441         ida_destroy(&nvme_ns_chr_minor_ida);
5442         ida_destroy(&nvme_instance_ida);
5443 }
5444
5445 MODULE_LICENSE("GPL");
5446 MODULE_VERSION("1.0");
5447 module_init(nvme_core_init);
5448 module_exit(nvme_core_exit);