drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/blk-integrity.h>
  10 #include <linux/compat.h>
  11 #include <linux/delay.h>
  12 #include <linux/errno.h>
  13 #include <linux/hdreg.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <asm/unaligned.h>
  24
  25 #include "nvme.h"
  26 #include "fabrics.h"
  27 #include <linux/nvme-auth.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include "trace.h"
  31
  32 #define NVME_MINORS             (1U << MINORBITS)
  33
  34 struct nvme_ns_info {
  35         struct nvme_ns_ids ids;
  36         u32 nsid;
  37         __le32 anagrpid;
  38         bool is_shared;
  39         bool is_readonly;
  40         bool is_ready;
  41         bool is_removed;
  42 };
  43
  44 unsigned int admin_timeout = 60;
  45 module_param(admin_timeout, uint, 0644);
  46 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  47 EXPORT_SYMBOL_GPL(admin_timeout);
  48
  49 unsigned int nvme_io_timeout = 30;
  50 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  51 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  52 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  53
  54 static unsigned char shutdown_timeout = 5;
  55 module_param(shutdown_timeout, byte, 0644);
  56 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  57
  58 static u8 nvme_max_retries = 5;
  59 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  60 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  61
  62 static unsigned long default_ps_max_latency_us = 100000;
  63 module_param(default_ps_max_latency_us, ulong, 0644);
  64 MODULE_PARM_DESC(default_ps_max_latency_us,
  65                  "max power saving latency for new devices; use PM QOS to change per device");
  66
  67 static bool force_apst;
  68 module_param(force_apst, bool, 0644);
  69 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  70
  71 static unsigned long apst_primary_timeout_ms = 100;
  72 module_param(apst_primary_timeout_ms, ulong, 0644);
  73 MODULE_PARM_DESC(apst_primary_timeout_ms,
  74         "primary APST timeout in ms");
  75
  76 static unsigned long apst_secondary_timeout_ms = 2000;
  77 module_param(apst_secondary_timeout_ms, ulong, 0644);
  78 MODULE_PARM_DESC(apst_secondary_timeout_ms,
  79         "secondary APST timeout in ms");
  80
  81 static unsigned long apst_primary_latency_tol_us = 15000;
  82 module_param(apst_primary_latency_tol_us, ulong, 0644);
  83 MODULE_PARM_DESC(apst_primary_latency_tol_us,
  84         "primary APST latency tolerance in us");
  85
  86 static unsigned long apst_secondary_latency_tol_us = 100000;
  87 module_param(apst_secondary_latency_tol_us, ulong, 0644);
  88 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
  89         "secondary APST latency tolerance in us");
  90
  91 /*
  92  * nvme_wq - hosts nvme related works that are not reset or delete
  93  * nvme_reset_wq - hosts nvme reset works
  94  * nvme_delete_wq - hosts nvme delete works
  95  *
  96  * nvme_wq will host works such as scan, aen handling, fw activation,
  97  * keep-alive, periodic reconnects etc. nvme_reset_wq
  98  * runs reset works which also flush works hosted on nvme_wq for
  99  * serialization purposes. nvme_delete_wq host controller deletion
 100  * works which flush reset works for serialization.
 101  */
 102 struct workqueue_struct *nvme_wq;
 103 EXPORT_SYMBOL_GPL(nvme_wq);
 104
 105 struct workqueue_struct *nvme_reset_wq;
 106 EXPORT_SYMBOL_GPL(nvme_reset_wq);
 107
 108 struct workqueue_struct *nvme_delete_wq;
 109 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 110
 111 static LIST_HEAD(nvme_subsystems);
 112 static DEFINE_MUTEX(nvme_subsystems_lock);
 113
 114 static DEFINE_IDA(nvme_instance_ida);
 115 static dev_t nvme_ctrl_base_chr_devt;
 116 static struct class *nvme_class;
 117 static struct class *nvme_subsys_class;
 118
 119 static DEFINE_IDA(nvme_ns_chr_minor_ida);
 120 static dev_t nvme_ns_chr_devt;
 121 static struct class *nvme_ns_chr_class;
 122
 123 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
 124 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
 125                                            unsigned nsid);
 126 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
 127                                    struct nvme_command *cmd);
 128
 129 void nvme_queue_scan(struct nvme_ctrl *ctrl)
 130 {
 131         /*
 132          * Only new queue scan work when admin and IO queues are both alive
 133          */
 134         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 135                 queue_work(nvme_wq, &ctrl->scan_work);
 136 }
 137
 138 /*
 139  * Use this function to proceed with scheduling reset_work for a controller
 140  * that had previously been set to the resetting state. This is intended for
 141  * code paths that can't be interrupted by other reset attempts. A hot removal
 142  * may prevent this from succeeding.
 143  */
 144 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 145 {
 146         if (ctrl->state != NVME_CTRL_RESETTING)
 147                 return -EBUSY;
 148         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 149                 return -EBUSY;
 150         return 0;
 151 }
 152 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 153
 154 static void nvme_failfast_work(struct work_struct *work)
 155 {
 156         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
 157                         struct nvme_ctrl, failfast_work);
 158
 159         if (ctrl->state != NVME_CTRL_CONNECTING)
 160                 return;
 161
 162         set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 163         dev_info(ctrl->device, "failfast expired\n");
 164         nvme_kick_requeue_lists(ctrl);
 165 }
 166
 167 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
 168 {
 169         if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
 170                 return;
 171
 172         schedule_delayed_work(&ctrl->failfast_work,
 173                               ctrl->opts->fast_io_fail_tmo * HZ);
 174 }
 175
 176 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
 177 {
 178         if (!ctrl->opts)
 179                 return;
 180
 181         cancel_delayed_work_sync(&ctrl->failfast_work);
 182         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 183 }
 184
 185
 186 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 187 {
 188         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 189                 return -EBUSY;
 190         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 191                 return -EBUSY;
 192         return 0;
 193 }
 194 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 195
 196 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 197 {
 198         int ret;
 199
 200         ret = nvme_reset_ctrl(ctrl);
 201         if (!ret) {
 202                 flush_work(&ctrl->reset_work);
 203                 if (ctrl->state != NVME_CTRL_LIVE)
 204                         ret = -ENETRESET;
 205         }
 206
 207         return ret;
 208 }
 209
 210 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 211 {
 212         dev_info(ctrl->device,
 213                  "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
 214
 215         flush_work(&ctrl->reset_work);
 216         nvme_stop_ctrl(ctrl);
 217         nvme_remove_namespaces(ctrl);
 218         ctrl->ops->delete_ctrl(ctrl);
 219         nvme_uninit_ctrl(ctrl);
 220 }
 221
 222 static void nvme_delete_ctrl_work(struct work_struct *work)
 223 {
 224         struct nvme_ctrl *ctrl =
 225                 container_of(work, struct nvme_ctrl, delete_work);
 226
 227         nvme_do_delete_ctrl(ctrl);
 228 }
 229
 230 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 231 {
 232         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 233                 return -EBUSY;
 234         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 235                 return -EBUSY;
 236         return 0;
 237 }
 238 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 239
 240 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 241 {
 242         /*
 243          * Keep a reference until nvme_do_delete_ctrl() complete,
 244          * since ->delete_ctrl can free the controller.
 245          */
 246         nvme_get_ctrl(ctrl);
 247         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 248                 nvme_do_delete_ctrl(ctrl);
 249         nvme_put_ctrl(ctrl);
 250 }
 251
 252 static blk_status_t nvme_error_status(u16 status)
 253 {
 254         switch (status & 0x7ff) {
 255         case NVME_SC_SUCCESS:
 256                 return BLK_STS_OK;
 257         case NVME_SC_CAP_EXCEEDED:
 258                 return BLK_STS_NOSPC;
 259         case NVME_SC_LBA_RANGE:
 260         case NVME_SC_CMD_INTERRUPTED:
 261         case NVME_SC_NS_NOT_READY:
 262                 return BLK_STS_TARGET;
 263         case NVME_SC_BAD_ATTRIBUTES:
 264         case NVME_SC_ONCS_NOT_SUPPORTED:
 265         case NVME_SC_INVALID_OPCODE:
 266         case NVME_SC_INVALID_FIELD:
 267         case NVME_SC_INVALID_NS:
 268                 return BLK_STS_NOTSUPP;
 269         case NVME_SC_WRITE_FAULT:
 270         case NVME_SC_READ_ERROR:
 271         case NVME_SC_UNWRITTEN_BLOCK:
 272         case NVME_SC_ACCESS_DENIED:
 273         case NVME_SC_READ_ONLY:
 274         case NVME_SC_COMPARE_FAILED:
 275                 return BLK_STS_MEDIUM;
 276         case NVME_SC_GUARD_CHECK:
 277         case NVME_SC_APPTAG_CHECK:
 278         case NVME_SC_REFTAG_CHECK:
 279         case NVME_SC_INVALID_PI:
 280                 return BLK_STS_PROTECTION;
 281         case NVME_SC_RESERVATION_CONFLICT:
 282                 return BLK_STS_NEXUS;
 283         case NVME_SC_HOST_PATH_ERROR:
 284                 return BLK_STS_TRANSPORT;
 285         case NVME_SC_ZONE_TOO_MANY_ACTIVE:
 286                 return BLK_STS_ZONE_ACTIVE_RESOURCE;
 287         case NVME_SC_ZONE_TOO_MANY_OPEN:
 288                 return BLK_STS_ZONE_OPEN_RESOURCE;
 289         default:
 290                 return BLK_STS_IOERR;
 291         }
 292 }
 293
 294 static void nvme_retry_req(struct request *req)
 295 {
 296         unsigned long delay = 0;
 297         u16 crd;
 298
 299         /* The mask and shift result must be <= 3 */
 300         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 301         if (crd)
 302                 delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
 303
 304         nvme_req(req)->retries++;
 305         blk_mq_requeue_request(req, false);
 306         blk_mq_delay_kick_requeue_list(req->q, delay);
 307 }
 308
 309 static void nvme_log_error(struct request *req)
 310 {
 311         struct nvme_ns *ns = req->q->queuedata;
 312         struct nvme_request *nr = nvme_req(req);
 313
 314         if (ns) {
 315                 pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
 316                        ns->disk ? ns->disk->disk_name : "?",
 317                        nvme_get_opcode_str(nr->cmd->common.opcode),
 318                        nr->cmd->common.opcode,
 319                        (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)),
 320                        (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
 321                        nvme_get_error_status_str(nr->status),
 322                        nr->status >> 8 & 7,     /* Status Code Type */
 323                        nr->status & 0xff,       /* Status Code */
 324                        nr->status & NVME_SC_MORE ? "MORE " : "",
 325                        nr->status & NVME_SC_DNR  ? "DNR "  : "");
 326                 return;
 327         }
 328
 329         pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
 330                            dev_name(nr->ctrl->device),
 331                            nvme_get_admin_opcode_str(nr->cmd->common.opcode),
 332                            nr->cmd->common.opcode,
 333                            nvme_get_error_status_str(nr->status),
 334                            nr->status >> 8 & 7, /* Status Code Type */
 335                            nr->status & 0xff,   /* Status Code */
 336                            nr->status & NVME_SC_MORE ? "MORE " : "",
 337                            nr->status & NVME_SC_DNR  ? "DNR "  : "");
 338 }
 339
 340 enum nvme_disposition {
 341         COMPLETE,
 342         RETRY,
 343         FAILOVER,
 344         AUTHENTICATE,
 345 };
 346
 347 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 348 {
 349         if (likely(nvme_req(req)->status == 0))
 350                 return COMPLETE;
 351
 352         if ((nvme_req(req)->status & 0x7ff) == NVME_SC_AUTH_REQUIRED)
 353                 return AUTHENTICATE;
 354
 355         if (blk_noretry_request(req) ||
 356             (nvme_req(req)->status & NVME_SC_DNR) ||
 357             nvme_req(req)->retries >= nvme_max_retries)
 358                 return COMPLETE;
 359
 360         if (req->cmd_flags & REQ_NVME_MPATH) {
 361                 if (nvme_is_path_error(nvme_req(req)->status) ||
 362                     blk_queue_dying(req->q))
 363                         return FAILOVER;
 364         } else {
 365                 if (blk_queue_dying(req->q))
 366                         return COMPLETE;
 367         }
 368
 369         return RETRY;
 370 }
 371
 372 static inline void nvme_end_req_zoned(struct request *req)
 373 {
 374         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 375             req_op(req) == REQ_OP_ZONE_APPEND)
 376                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 377                         le64_to_cpu(nvme_req(req)->result.u64));
 378 }
 379
 380 static inline void nvme_end_req(struct request *req)
 381 {
 382         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 383
 384         if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET)))
 385                 nvme_log_error(req);
 386         nvme_end_req_zoned(req);
 387         nvme_trace_bio_complete(req);
 388         if (req->cmd_flags & REQ_NVME_MPATH)
 389                 nvme_mpath_end_request(req);
 390         blk_mq_end_request(req, status);
 391 }
 392
 393 void nvme_complete_rq(struct request *req)
 394 {
 395         struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 396
 397         trace_nvme_complete_rq(req);
 398         nvme_cleanup_cmd(req);
 399
 400         if (ctrl->kas)
 401                 ctrl->comp_seen = true;
 402
 403         switch (nvme_decide_disposition(req)) {
 404         case COMPLETE:
 405                 nvme_end_req(req);
 406                 return;
 407         case RETRY:
 408                 nvme_retry_req(req);
 409                 return;
 410         case FAILOVER:
 411                 nvme_failover_req(req);
 412                 return;
 413         case AUTHENTICATE:
 414 #ifdef CONFIG_NVME_AUTH
 415                 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
 416                 nvme_retry_req(req);
 417 #else
 418                 nvme_end_req(req);
 419 #endif
 420                 return;
 421         }
 422 }
 423 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 424
 425 void nvme_complete_batch_req(struct request *req)
 426 {
 427         trace_nvme_complete_rq(req);
 428         nvme_cleanup_cmd(req);
 429         nvme_end_req_zoned(req);
 430 }
 431 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
 432
 433 /*
 434  * Called to unwind from ->queue_rq on a failed command submission so that the
 435  * multipathing code gets called to potentially failover to another path.
 436  * The caller needs to unwind all transport specific resource allocations and
 437  * must return propagate the return value.
 438  */
 439 blk_status_t nvme_host_path_error(struct request *req)
 440 {
 441         nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
 442         blk_mq_set_request_complete(req);
 443         nvme_complete_rq(req);
 444         return BLK_STS_OK;
 445 }
 446 EXPORT_SYMBOL_GPL(nvme_host_path_error);
 447
 448 bool nvme_cancel_request(struct request *req, void *data)
 449 {
 450         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 451                                 "Cancelling I/O %d", req->tag);
 452
 453         /* don't abort one completed or idle request */
 454         if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
 455                 return true;
 456
 457         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 458         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 459         blk_mq_complete_request(req);
 460         return true;
 461 }
 462 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 463
 464 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
 465 {
 466         if (ctrl->tagset) {
 467                 blk_mq_tagset_busy_iter(ctrl->tagset,
 468                                 nvme_cancel_request, ctrl);
 469                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
 470         }
 471 }
 472 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
 473
 474 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 475 {
 476         if (ctrl->admin_tagset) {
 477                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
 478                                 nvme_cancel_request, ctrl);
 479                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
 480         }
 481 }
 482 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 483
 484 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 485                 enum nvme_ctrl_state new_state)
 486 {
 487         enum nvme_ctrl_state old_state;
 488         unsigned long flags;
 489         bool changed = false;
 490
 491         spin_lock_irqsave(&ctrl->lock, flags);
 492
 493         old_state = ctrl->state;
 494         switch (new_state) {
 495         case NVME_CTRL_LIVE:
 496                 switch (old_state) {
 497                 case NVME_CTRL_NEW:
 498                 case NVME_CTRL_RESETTING:
 499                 case NVME_CTRL_CONNECTING:
 500                         changed = true;
 501                         fallthrough;
 502                 default:
 503                         break;
 504                 }
 505                 break;
 506         case NVME_CTRL_RESETTING:
 507                 switch (old_state) {
 508                 case NVME_CTRL_NEW:
 509                 case NVME_CTRL_LIVE:
 510                         changed = true;
 511                         fallthrough;
 512                 default:
 513                         break;
 514                 }
 515                 break;
 516         case NVME_CTRL_CONNECTING:
 517                 switch (old_state) {
 518                 case NVME_CTRL_NEW:
 519                 case NVME_CTRL_RESETTING:
 520                         changed = true;
 521                         fallthrough;
 522                 default:
 523                         break;
 524                 }
 525                 break;
 526         case NVME_CTRL_DELETING:
 527                 switch (old_state) {
 528                 case NVME_CTRL_LIVE:
 529                 case NVME_CTRL_RESETTING:
 530                 case NVME_CTRL_CONNECTING:
 531                         changed = true;
 532                         fallthrough;
 533                 default:
 534                         break;
 535                 }
 536                 break;
 537         case NVME_CTRL_DELETING_NOIO:
 538                 switch (old_state) {
 539                 case NVME_CTRL_DELETING:
 540                 case NVME_CTRL_DEAD:
 541                         changed = true;
 542                         fallthrough;
 543                 default:
 544                         break;
 545                 }
 546                 break;
 547         case NVME_CTRL_DEAD:
 548                 switch (old_state) {
 549                 case NVME_CTRL_DELETING:
 550                         changed = true;
 551                         fallthrough;
 552                 default:
 553                         break;
 554                 }
 555                 break;
 556         default:
 557                 break;
 558         }
 559
 560         if (changed) {
 561                 ctrl->state = new_state;
 562                 wake_up_all(&ctrl->state_wq);
 563         }
 564
 565         spin_unlock_irqrestore(&ctrl->lock, flags);
 566         if (!changed)
 567                 return false;
 568
 569         if (ctrl->state == NVME_CTRL_LIVE) {
 570                 if (old_state == NVME_CTRL_CONNECTING)
 571                         nvme_stop_failfast_work(ctrl);
 572                 nvme_kick_requeue_lists(ctrl);
 573         } else if (ctrl->state == NVME_CTRL_CONNECTING &&
 574                 old_state == NVME_CTRL_RESETTING) {
 575                 nvme_start_failfast_work(ctrl);
 576         }
 577         return changed;
 578 }
 579 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 580
 581 /*
 582  * Returns true for sink states that can't ever transition back to live.
 583  */
 584 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 585 {
 586         switch (ctrl->state) {
 587         case NVME_CTRL_NEW:
 588         case NVME_CTRL_LIVE:
 589         case NVME_CTRL_RESETTING:
 590         case NVME_CTRL_CONNECTING:
 591                 return false;
 592         case NVME_CTRL_DELETING:
 593         case NVME_CTRL_DELETING_NOIO:
 594         case NVME_CTRL_DEAD:
 595                 return true;
 596         default:
 597                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 598                 return true;
 599         }
 600 }
 601
 602 /*
 603  * Waits for the controller state to be resetting, or returns false if it is
 604  * not possible to ever transition to that state.
 605  */
 606 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 607 {
 608         wait_event(ctrl->state_wq,
 609                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 610                    nvme_state_terminal(ctrl));
 611         return ctrl->state == NVME_CTRL_RESETTING;
 612 }
 613 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 614
 615 static void nvme_free_ns_head(struct kref *ref)
 616 {
 617         struct nvme_ns_head *head =
 618                 container_of(ref, struct nvme_ns_head, ref);
 619
 620         nvme_mpath_remove_disk(head);
 621         ida_free(&head->subsys->ns_ida, head->instance);
 622         cleanup_srcu_struct(&head->srcu);
 623         nvme_put_subsystem(head->subsys);
 624         kfree(head);
 625 }
 626
 627 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
 628 {
 629         return kref_get_unless_zero(&head->ref);
 630 }
 631
 632 void nvme_put_ns_head(struct nvme_ns_head *head)
 633 {
 634         kref_put(&head->ref, nvme_free_ns_head);
 635 }
 636
 637 static void nvme_free_ns(struct kref *kref)
 638 {
 639         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 640
 641         put_disk(ns->disk);
 642         nvme_put_ns_head(ns->head);
 643         nvme_put_ctrl(ns->ctrl);
 644         kfree(ns);
 645 }
 646
 647 static inline bool nvme_get_ns(struct nvme_ns *ns)
 648 {
 649         return kref_get_unless_zero(&ns->kref);
 650 }
 651
 652 void nvme_put_ns(struct nvme_ns *ns)
 653 {
 654         kref_put(&ns->kref, nvme_free_ns);
 655 }
 656 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 657
 658 static inline void nvme_clear_nvme_request(struct request *req)
 659 {
 660         nvme_req(req)->status = 0;
 661         nvme_req(req)->retries = 0;
 662         nvme_req(req)->flags = 0;
 663         req->rq_flags |= RQF_DONTPREP;
 664 }
 665
 666 /* initialize a passthrough request */
 667 void nvme_init_request(struct request *req, struct nvme_command *cmd)
 668 {
 669         if (req->q->queuedata)
 670                 req->timeout = NVME_IO_TIMEOUT;
 671         else /* no queuedata implies admin queue */
 672                 req->timeout = NVME_ADMIN_TIMEOUT;
 673
 674         /* passthru commands should let the driver set the SGL flags */
 675         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 676
 677         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 678         if (req->mq_hctx->type == HCTX_TYPE_POLL)
 679                 req->cmd_flags |= REQ_POLLED;
 680         nvme_clear_nvme_request(req);
 681         req->rq_flags |= RQF_QUIET;
 682         memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
 683 }
 684 EXPORT_SYMBOL_GPL(nvme_init_request);
 685
 686 /*
 687  * For something we're not in a state to send to the device the default action
 688  * is to busy it and retry it after the controller state is recovered.  However,
 689  * if the controller is deleting or if anything is marked for failfast or
 690  * nvme multipath it is immediately failed.
 691  *
 692  * Note: commands used to initialize the controller will be marked for failfast.
 693  * Note: nvme cli/ioctl commands are marked for failfast.
 694  */
 695 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
 696                 struct request *rq)
 697 {
 698         if (ctrl->state != NVME_CTRL_DELETING_NOIO &&
 699             ctrl->state != NVME_CTRL_DELETING &&
 700             ctrl->state != NVME_CTRL_DEAD &&
 701             !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
 702             !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
 703                 return BLK_STS_RESOURCE;
 704         return nvme_host_path_error(rq);
 705 }
 706 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
 707
 708 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
 709                 bool queue_live)
 710 {
 711         struct nvme_request *req = nvme_req(rq);
 712
 713         /*
 714          * currently we have a problem sending passthru commands
 715          * on the admin_q if the controller is not LIVE because we can't
 716          * make sure that they are going out after the admin connect,
 717          * controller enable and/or other commands in the initialization
 718          * sequence. until the controller will be LIVE, fail with
 719          * BLK_STS_RESOURCE so that they will be rescheduled.
 720          */
 721         if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
 722                 return false;
 723
 724         if (ctrl->ops->flags & NVME_F_FABRICS) {
 725                 /*
 726                  * Only allow commands on a live queue, except for the connect
 727                  * command, which is require to set the queue live in the
 728                  * appropinquate states.
 729                  */
 730                 switch (ctrl->state) {
 731                 case NVME_CTRL_CONNECTING:
 732                         if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
 733                             (req->cmd->fabrics.fctype == nvme_fabrics_type_connect ||
 734                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_send ||
 735                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive))
 736                                 return true;
 737                         break;
 738                 default:
 739                         break;
 740                 case NVME_CTRL_DEAD:
 741                         return false;
 742                 }
 743         }
 744
 745         return queue_live;
 746 }
 747 EXPORT_SYMBOL_GPL(__nvme_check_ready);
 748
 749 static inline void nvme_setup_flush(struct nvme_ns *ns,
 750                 struct nvme_command *cmnd)
 751 {
 752         memset(cmnd, 0, sizeof(*cmnd));
 753         cmnd->common.opcode = nvme_cmd_flush;
 754         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 755 }
 756
 757 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 758                 struct nvme_command *cmnd)
 759 {
 760         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 761         struct nvme_dsm_range *range;
 762         struct bio *bio;
 763
 764         /*
 765          * Some devices do not consider the DSM 'Number of Ranges' field when
 766          * determining how much data to DMA. Always allocate memory for maximum
 767          * number of segments to prevent device reading beyond end of buffer.
 768          */
 769         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 770
 771         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 772         if (!range) {
 773                 /*
 774                  * If we fail allocation our range, fallback to the controller
 775                  * discard page. If that's also busy, it's safe to return
 776                  * busy, as we know we can make progress once that's freed.
 777                  */
 778                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 779                         return BLK_STS_RESOURCE;
 780
 781                 range = page_address(ns->ctrl->discard_page);
 782         }
 783
 784         if (queue_max_discard_segments(req->q) == 1) {
 785                 u64 slba = nvme_sect_to_lba(ns, blk_rq_pos(req));
 786                 u32 nlb = blk_rq_sectors(req) >> (ns->lba_shift - 9);
 787
 788                 range[0].cattr = cpu_to_le32(0);
 789                 range[0].nlb = cpu_to_le32(nlb);
 790                 range[0].slba = cpu_to_le64(slba);
 791                 n = 1;
 792         } else {
 793                 __rq_for_each_bio(bio, req) {
 794                         u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 795                         u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 796
 797                         if (n < segments) {
 798                                 range[n].cattr = cpu_to_le32(0);
 799                                 range[n].nlb = cpu_to_le32(nlb);
 800                                 range[n].slba = cpu_to_le64(slba);
 801                         }
 802                         n++;
 803                 }
 804         }
 805
 806         if (WARN_ON_ONCE(n != segments)) {
 807                 if (virt_to_page(range) == ns->ctrl->discard_page)
 808                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 809                 else
 810                         kfree(range);
 811                 return BLK_STS_IOERR;
 812         }
 813
 814         memset(cmnd, 0, sizeof(*cmnd));
 815         cmnd->dsm.opcode = nvme_cmd_dsm;
 816         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 817         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 818         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 819
 820         bvec_set_virt(&req->special_vec, range, alloc_size);
 821         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 822
 823         return BLK_STS_OK;
 824 }
 825
 826 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
 827                               struct request *req)
 828 {
 829         u32 upper, lower;
 830         u64 ref48;
 831
 832         /* both rw and write zeroes share the same reftag format */
 833         switch (ns->guard_type) {
 834         case NVME_NVM_NS_16B_GUARD:
 835                 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 836                 break;
 837         case NVME_NVM_NS_64B_GUARD:
 838                 ref48 = ext_pi_ref_tag(req);
 839                 lower = lower_32_bits(ref48);
 840                 upper = upper_32_bits(ref48);
 841
 842                 cmnd->rw.reftag = cpu_to_le32(lower);
 843                 cmnd->rw.cdw3 = cpu_to_le32(upper);
 844                 break;
 845         default:
 846                 break;
 847         }
 848 }
 849
 850 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 851                 struct request *req, struct nvme_command *cmnd)
 852 {
 853         memset(cmnd, 0, sizeof(*cmnd));
 854
 855         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 856                 return nvme_setup_discard(ns, req, cmnd);
 857
 858         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 859         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 860         cmnd->write_zeroes.slba =
 861                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 862         cmnd->write_zeroes.length =
 863                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 864
 865         if (!(req->cmd_flags & REQ_NOUNMAP) && (ns->features & NVME_NS_DEAC))
 866                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC);
 867
 868         if (nvme_ns_has_pi(ns)) {
 869                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT);
 870
 871                 switch (ns->pi_type) {
 872                 case NVME_NS_DPS_PI_TYPE1:
 873                 case NVME_NS_DPS_PI_TYPE2:
 874                         nvme_set_ref_tag(ns, cmnd, req);
 875                         break;
 876                 }
 877         }
 878
 879         return BLK_STS_OK;
 880 }
 881
 882 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 883                 struct request *req, struct nvme_command *cmnd,
 884                 enum nvme_opcode op)
 885 {
 886         u16 control = 0;
 887         u32 dsmgmt = 0;
 888
 889         if (req->cmd_flags & REQ_FUA)
 890                 control |= NVME_RW_FUA;
 891         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 892                 control |= NVME_RW_LR;
 893
 894         if (req->cmd_flags & REQ_RAHEAD)
 895                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 896
 897         cmnd->rw.opcode = op;
 898         cmnd->rw.flags = 0;
 899         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 900         cmnd->rw.cdw2 = 0;
 901         cmnd->rw.cdw3 = 0;
 902         cmnd->rw.metadata = 0;
 903         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 904         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 905         cmnd->rw.reftag = 0;
 906         cmnd->rw.apptag = 0;
 907         cmnd->rw.appmask = 0;
 908
 909         if (ns->ms) {
 910                 /*
 911                  * If formated with metadata, the block layer always provides a
 912                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 913                  * we enable the PRACT bit for protection information or set the
 914                  * namespace capacity to zero to prevent any I/O.
 915                  */
 916                 if (!blk_integrity_rq(req)) {
 917                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 918                                 return BLK_STS_NOTSUPP;
 919                         control |= NVME_RW_PRINFO_PRACT;
 920                 }
 921
 922                 switch (ns->pi_type) {
 923                 case NVME_NS_DPS_PI_TYPE3:
 924                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 925                         break;
 926                 case NVME_NS_DPS_PI_TYPE1:
 927                 case NVME_NS_DPS_PI_TYPE2:
 928                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 929                                         NVME_RW_PRINFO_PRCHK_REF;
 930                         if (op == nvme_cmd_zone_append)
 931                                 control |= NVME_RW_APPEND_PIREMAP;
 932                         nvme_set_ref_tag(ns, cmnd, req);
 933                         break;
 934                 }
 935         }
 936
 937         cmnd->rw.control = cpu_to_le16(control);
 938         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 939         return 0;
 940 }
 941
 942 void nvme_cleanup_cmd(struct request *req)
 943 {
 944         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 945                 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 946
 947                 if (req->special_vec.bv_page == ctrl->discard_page)
 948                         clear_bit_unlock(0, &ctrl->discard_page_busy);
 949                 else
 950                         kfree(bvec_virt(&req->special_vec));
 951         }
 952 }
 953 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 954
 955 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
 956 {
 957         struct nvme_command *cmd = nvme_req(req)->cmd;
 958         blk_status_t ret = BLK_STS_OK;
 959
 960         if (!(req->rq_flags & RQF_DONTPREP))
 961                 nvme_clear_nvme_request(req);
 962
 963         switch (req_op(req)) {
 964         case REQ_OP_DRV_IN:
 965         case REQ_OP_DRV_OUT:
 966                 /* these are setup prior to execution in nvme_init_request() */
 967                 break;
 968         case REQ_OP_FLUSH:
 969                 nvme_setup_flush(ns, cmd);
 970                 break;
 971         case REQ_OP_ZONE_RESET_ALL:
 972         case REQ_OP_ZONE_RESET:
 973                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 974                 break;
 975         case REQ_OP_ZONE_OPEN:
 976                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 977                 break;
 978         case REQ_OP_ZONE_CLOSE:
 979                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 980                 break;
 981         case REQ_OP_ZONE_FINISH:
 982                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 983                 break;
 984         case REQ_OP_WRITE_ZEROES:
 985                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 986                 break;
 987         case REQ_OP_DISCARD:
 988                 ret = nvme_setup_discard(ns, req, cmd);
 989                 break;
 990         case REQ_OP_READ:
 991                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 992                 break;
 993         case REQ_OP_WRITE:
 994                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 995                 break;
 996         case REQ_OP_ZONE_APPEND:
 997                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 998                 break;
 999         default:
1000                 WARN_ON_ONCE(1);
1001                 return BLK_STS_IOERR;
1002         }
1003
1004         cmd->common.command_id = nvme_cid(req);
1005         trace_nvme_setup_cmd(req, cmd);
1006         return ret;
1007 }
1008 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
1009
1010 /*
1011  * Return values:
1012  * 0:  success
1013  * >0: nvme controller's cqe status response
1014  * <0: kernel error in lieu of controller response
1015  */
1016 int nvme_execute_rq(struct request *rq, bool at_head)
1017 {
1018         blk_status_t status;
1019
1020         status = blk_execute_rq(rq, at_head);
1021         if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
1022                 return -EINTR;
1023         if (nvme_req(rq)->status)
1024                 return nvme_req(rq)->status;
1025         return blk_status_to_errno(status);
1026 }
1027 EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);
1028
1029 /*
1030  * Returns 0 on success.  If the result is negative, it's a Linux error code;
1031  * if the result is positive, it's an NVM Express status code
1032  */
1033 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1034                 union nvme_result *result, void *buffer, unsigned bufflen,
1035                 int qid, int at_head, blk_mq_req_flags_t flags)
1036 {
1037         struct request *req;
1038         int ret;
1039
1040         if (qid == NVME_QID_ANY)
1041                 req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
1042         else
1043                 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
1044                                                 qid - 1);
1045
1046         if (IS_ERR(req))
1047                 return PTR_ERR(req);
1048         nvme_init_request(req, cmd);
1049
1050         if (buffer && bufflen) {
1051                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1052                 if (ret)
1053                         goto out;
1054         }
1055
1056         ret = nvme_execute_rq(req, at_head);
1057         if (result && ret >= 0)
1058                 *result = nvme_req(req)->result;
1059  out:
1060         blk_mq_free_request(req);
1061         return ret;
1062 }
1063 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1064
1065 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1066                 void *buffer, unsigned bufflen)
1067 {
1068         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
1069                         NVME_QID_ANY, 0, 0);
1070 }
1071 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1072
1073 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1074 {
1075         u32 effects = 0;
1076
1077         if (ns) {
1078                 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1079                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1080                         dev_warn_once(ctrl->device,
1081                                 "IO command:%02x has unusual effects:%08x\n",
1082                                 opcode, effects);
1083
1084                 /*
1085                  * NVME_CMD_EFFECTS_CSE_MASK causes a freeze all I/O queues,
1086                  * which would deadlock when done on an I/O command.  Note that
1087                  * We already warn about an unusual effect above.
1088                  */
1089                 effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1090         } else {
1091                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1092         }
1093
1094         return effects;
1095 }
1096 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1097
1098 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1099 {
1100         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1101
1102         /*
1103          * For simplicity, IO to all namespaces is quiesced even if the command
1104          * effects say only one namespace is affected.
1105          */
1106         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1107                 mutex_lock(&ctrl->scan_lock);
1108                 mutex_lock(&ctrl->subsys->lock);
1109                 nvme_mpath_start_freeze(ctrl->subsys);
1110                 nvme_mpath_wait_freeze(ctrl->subsys);
1111                 nvme_start_freeze(ctrl);
1112                 nvme_wait_freeze(ctrl);
1113         }
1114         return effects;
1115 }
1116 EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, NVME_TARGET_PASSTHRU);
1117
1118 void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,
1119                        struct nvme_command *cmd, int status)
1120 {
1121         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1122                 nvme_unfreeze(ctrl);
1123                 nvme_mpath_unfreeze(ctrl->subsys);
1124                 mutex_unlock(&ctrl->subsys->lock);
1125                 mutex_unlock(&ctrl->scan_lock);
1126         }
1127         if (effects & NVME_CMD_EFFECTS_CCC) {
1128                 dev_info(ctrl->device,
1129 "controller capabilities changed, reset may be required to take effect.\n");
1130         }
1131         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1132                 nvme_queue_scan(ctrl);
1133                 flush_work(&ctrl->scan_work);
1134         }
1135
1136         switch (cmd->common.opcode) {
1137         case nvme_admin_set_features:
1138                 switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1139                 case NVME_FEAT_KATO:
1140                         /*
1141                          * Keep alive commands interval on the host should be
1142                          * updated when KATO is modified by Set Features
1143                          * commands.
1144                          */
1145                         if (!status)
1146                                 nvme_update_keep_alive(ctrl, cmd);
1147                         break;
1148                 default:
1149                         break;
1150                 }
1151                 break;
1152         default:
1153                 break;
1154         }
1155 }
1156 EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU);
1157
1158 /*
1159  * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1160  *
1161  *   The host should send Keep Alive commands at half of the Keep Alive Timeout
1162  *   accounting for transport roundtrip times [..].
1163  */
1164 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1165 {
1166         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2);
1167 }
1168
1169 static enum rq_end_io_ret nvme_keep_alive_end_io(struct request *rq,
1170                                                  blk_status_t status)
1171 {
1172         struct nvme_ctrl *ctrl = rq->end_io_data;
1173         unsigned long flags;
1174         bool startka = false;
1175
1176         blk_mq_free_request(rq);
1177
1178         if (status) {
1179                 dev_err(ctrl->device,
1180                         "failed nvme_keep_alive_end_io error=%d\n",
1181                                 status);
1182                 return RQ_END_IO_NONE;
1183         }
1184
1185         ctrl->comp_seen = false;
1186         spin_lock_irqsave(&ctrl->lock, flags);
1187         if (ctrl->state == NVME_CTRL_LIVE ||
1188             ctrl->state == NVME_CTRL_CONNECTING)
1189                 startka = true;
1190         spin_unlock_irqrestore(&ctrl->lock, flags);
1191         if (startka)
1192                 nvme_queue_keep_alive_work(ctrl);
1193         return RQ_END_IO_NONE;
1194 }
1195
1196 static void nvme_keep_alive_work(struct work_struct *work)
1197 {
1198         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1199                         struct nvme_ctrl, ka_work);
1200         bool comp_seen = ctrl->comp_seen;
1201         struct request *rq;
1202
1203         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1204                 dev_dbg(ctrl->device,
1205                         "reschedule traffic based keep-alive timer\n");
1206                 ctrl->comp_seen = false;
1207                 nvme_queue_keep_alive_work(ctrl);
1208                 return;
1209         }
1210
1211         rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1212                                   BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1213         if (IS_ERR(rq)) {
1214                 /* allocation failure, reset the controller */
1215                 dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1216                 nvme_reset_ctrl(ctrl);
1217                 return;
1218         }
1219         nvme_init_request(rq, &ctrl->ka_cmd);
1220
1221         rq->timeout = ctrl->kato * HZ;
1222         rq->end_io = nvme_keep_alive_end_io;
1223         rq->end_io_data = ctrl;
1224         blk_execute_rq_nowait(rq, false);
1225 }
1226
1227 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1228 {
1229         if (unlikely(ctrl->kato == 0))
1230                 return;
1231
1232         nvme_queue_keep_alive_work(ctrl);
1233 }
1234
1235 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1236 {
1237         if (unlikely(ctrl->kato == 0))
1238                 return;
1239
1240         cancel_delayed_work_sync(&ctrl->ka_work);
1241 }
1242 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1243
1244 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1245                                    struct nvme_command *cmd)
1246 {
1247         unsigned int new_kato =
1248                 DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1249
1250         dev_info(ctrl->device,
1251                  "keep alive interval updated from %u ms to %u ms\n",
1252                  ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1253
1254         nvme_stop_keep_alive(ctrl);
1255         ctrl->kato = new_kato;
1256         nvme_start_keep_alive(ctrl);
1257 }
1258
1259 /*
1260  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1261  * flag, thus sending any new CNS opcodes has a big chance of not working.
1262  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1263  * (but not for any later version).
1264  */
1265 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1266 {
1267         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1268                 return ctrl->vs < NVME_VS(1, 2, 0);
1269         return ctrl->vs < NVME_VS(1, 1, 0);
1270 }
1271
1272 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1273 {
1274         struct nvme_command c = { };
1275         int error;
1276
1277         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1278         c.identify.opcode = nvme_admin_identify;
1279         c.identify.cns = NVME_ID_CNS_CTRL;
1280
1281         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1282         if (!*id)
1283                 return -ENOMEM;
1284
1285         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1286                         sizeof(struct nvme_id_ctrl));
1287         if (error)
1288                 kfree(*id);
1289         return error;
1290 }
1291
1292 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1293                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1294 {
1295         const char *warn_str = "ctrl returned bogus length:";
1296         void *data = cur;
1297
1298         switch (cur->nidt) {
1299         case NVME_NIDT_EUI64:
1300                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1301                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1302                                  warn_str, cur->nidl);
1303                         return -1;
1304                 }
1305                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1306                         return NVME_NIDT_EUI64_LEN;
1307                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1308                 return NVME_NIDT_EUI64_LEN;
1309         case NVME_NIDT_NGUID:
1310                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1311                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1312                                  warn_str, cur->nidl);
1313                         return -1;
1314                 }
1315                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1316                         return NVME_NIDT_NGUID_LEN;
1317                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1318                 return NVME_NIDT_NGUID_LEN;
1319         case NVME_NIDT_UUID:
1320                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1321                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1322                                  warn_str, cur->nidl);
1323                         return -1;
1324                 }
1325                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1326                         return NVME_NIDT_UUID_LEN;
1327                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1328                 return NVME_NIDT_UUID_LEN;
1329         case NVME_NIDT_CSI:
1330                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1331                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1332                                  warn_str, cur->nidl);
1333                         return -1;
1334                 }
1335                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1336                 *csi_seen = true;
1337                 return NVME_NIDT_CSI_LEN;
1338         default:
1339                 /* Skip unknown types */
1340                 return cur->nidl;
1341         }
1342 }
1343
1344 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,
1345                 struct nvme_ns_info *info)
1346 {
1347         struct nvme_command c = { };
1348         bool csi_seen = false;
1349         int status, pos, len;
1350         void *data;
1351
1352         if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1353                 return 0;
1354         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1355                 return 0;
1356
1357         c.identify.opcode = nvme_admin_identify;
1358         c.identify.nsid = cpu_to_le32(info->nsid);
1359         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1360
1361         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1362         if (!data)
1363                 return -ENOMEM;
1364
1365         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1366                                       NVME_IDENTIFY_DATA_SIZE);
1367         if (status) {
1368                 dev_warn(ctrl->device,
1369                         "Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1370                         info->nsid, status);
1371                 goto free_data;
1372         }
1373
1374         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1375                 struct nvme_ns_id_desc *cur = data + pos;
1376
1377                 if (cur->nidl == 0)
1378                         break;
1379
1380                 len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen);
1381                 if (len < 0)
1382                         break;
1383
1384                 len += sizeof(*cur);
1385         }
1386
1387         if (nvme_multi_css(ctrl) && !csi_seen) {
1388                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1389                          info->nsid);
1390                 status = -EINVAL;
1391         }
1392
1393 free_data:
1394         kfree(data);
1395         return status;
1396 }
1397
1398 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1399                         struct nvme_id_ns **id)
1400 {
1401         struct nvme_command c = { };
1402         int error;
1403
1404         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1405         c.identify.opcode = nvme_admin_identify;
1406         c.identify.nsid = cpu_to_le32(nsid);
1407         c.identify.cns = NVME_ID_CNS_NS;
1408
1409         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1410         if (!*id)
1411                 return -ENOMEM;
1412
1413         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1414         if (error) {
1415                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1416                 kfree(*id);
1417         }
1418         return error;
1419 }
1420
1421 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,
1422                 struct nvme_ns_info *info)
1423 {
1424         struct nvme_ns_ids *ids = &info->ids;
1425         struct nvme_id_ns *id;
1426         int ret;
1427
1428         ret = nvme_identify_ns(ctrl, info->nsid, &id);
1429         if (ret)
1430                 return ret;
1431
1432         if (id->ncap == 0) {
1433                 /* namespace not allocated or attached */
1434                 info->is_removed = true;
1435                 return -ENODEV;
1436         }
1437
1438         info->anagrpid = id->anagrpid;
1439         info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1440         info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1441         info->is_ready = true;
1442         if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1443                 dev_info(ctrl->device,
1444                          "Ignoring bogus Namespace Identifiers\n");
1445         } else {
1446                 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1447                     !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1448                         memcpy(ids->eui64, id->eui64, sizeof(ids->eui64));
1449                 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1450                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1451                         memcpy(ids->nguid, id->nguid, sizeof(ids->nguid));
1452         }
1453         kfree(id);
1454         return 0;
1455 }
1456
1457 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,
1458                 struct nvme_ns_info *info)
1459 {
1460         struct nvme_id_ns_cs_indep *id;
1461         struct nvme_command c = {
1462                 .identify.opcode        = nvme_admin_identify,
1463                 .identify.nsid          = cpu_to_le32(info->nsid),
1464                 .identify.cns           = NVME_ID_CNS_NS_CS_INDEP,
1465         };
1466         int ret;
1467
1468         id = kmalloc(sizeof(*id), GFP_KERNEL);
1469         if (!id)
1470                 return -ENOMEM;
1471
1472         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1473         if (!ret) {
1474                 info->anagrpid = id->anagrpid;
1475                 info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1476                 info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1477                 info->is_ready = id->nstat & NVME_NSTAT_NRDY;
1478         }
1479         kfree(id);
1480         return ret;
1481 }
1482
1483 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1484                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1485 {
1486         union nvme_result res = { 0 };
1487         struct nvme_command c = { };
1488         int ret;
1489
1490         c.features.opcode = op;
1491         c.features.fid = cpu_to_le32(fid);
1492         c.features.dword11 = cpu_to_le32(dword11);
1493
1494         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1495                         buffer, buflen, NVME_QID_ANY, 0, 0);
1496         if (ret >= 0 && result)
1497                 *result = le32_to_cpu(res.u32);
1498         return ret;
1499 }
1500
1501 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1502                       unsigned int dword11, void *buffer, size_t buflen,
1503                       u32 *result)
1504 {
1505         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1506                              buflen, result);
1507 }
1508 EXPORT_SYMBOL_GPL(nvme_set_features);
1509
1510 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1511                       unsigned int dword11, void *buffer, size_t buflen,
1512                       u32 *result)
1513 {
1514         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1515                              buflen, result);
1516 }
1517 EXPORT_SYMBOL_GPL(nvme_get_features);
1518
1519 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1520 {
1521         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1522         u32 result;
1523         int status, nr_io_queues;
1524
1525         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1526                         &result);
1527         if (status < 0)
1528                 return status;
1529
1530         /*
1531          * Degraded controllers might return an error when setting the queue
1532          * count.  We still want to be able to bring them online and offer
1533          * access to the admin queue, as that might be only way to fix them up.
1534          */
1535         if (status > 0) {
1536                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1537                 *count = 0;
1538         } else {
1539                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1540                 *count = min(*count, nr_io_queues);
1541         }
1542
1543         return 0;
1544 }
1545 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1546
1547 #define NVME_AEN_SUPPORTED \
1548         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1549          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1550
1551 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1552 {
1553         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1554         int status;
1555
1556         if (!supported_aens)
1557                 return;
1558
1559         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1560                         NULL, 0, &result);
1561         if (status)
1562                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1563                          supported_aens);
1564
1565         queue_work(nvme_wq, &ctrl->async_event_work);
1566 }
1567
1568 static int nvme_ns_open(struct nvme_ns *ns)
1569 {
1570
1571         /* should never be called due to GENHD_FL_HIDDEN */
1572         if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1573                 goto fail;
1574         if (!nvme_get_ns(ns))
1575                 goto fail;
1576         if (!try_module_get(ns->ctrl->ops->module))
1577                 goto fail_put_ns;
1578
1579         return 0;
1580
1581 fail_put_ns:
1582         nvme_put_ns(ns);
1583 fail:
1584         return -ENXIO;
1585 }
1586
1587 static void nvme_ns_release(struct nvme_ns *ns)
1588 {
1589
1590         module_put(ns->ctrl->ops->module);
1591         nvme_put_ns(ns);
1592 }
1593
1594 static int nvme_open(struct block_device *bdev, fmode_t mode)
1595 {
1596         return nvme_ns_open(bdev->bd_disk->private_data);
1597 }
1598
1599 static void nvme_release(struct gendisk *disk, fmode_t mode)
1600 {
1601         nvme_ns_release(disk->private_data);
1602 }
1603
1604 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1605 {
1606         /* some standard values */
1607         geo->heads = 1 << 6;
1608         geo->sectors = 1 << 5;
1609         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1610         return 0;
1611 }
1612
1613 #ifdef CONFIG_BLK_DEV_INTEGRITY
1614 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1615                                 u32 max_integrity_segments)
1616 {
1617         struct blk_integrity integrity = { };
1618
1619         switch (ns->pi_type) {
1620         case NVME_NS_DPS_PI_TYPE3:
1621                 switch (ns->guard_type) {
1622                 case NVME_NVM_NS_16B_GUARD:
1623                         integrity.profile = &t10_pi_type3_crc;
1624                         integrity.tag_size = sizeof(u16) + sizeof(u32);
1625                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1626                         break;
1627                 case NVME_NVM_NS_64B_GUARD:
1628                         integrity.profile = &ext_pi_type3_crc64;
1629                         integrity.tag_size = sizeof(u16) + 6;
1630                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1631                         break;
1632                 default:
1633                         integrity.profile = NULL;
1634                         break;
1635                 }
1636                 break;
1637         case NVME_NS_DPS_PI_TYPE1:
1638         case NVME_NS_DPS_PI_TYPE2:
1639                 switch (ns->guard_type) {
1640                 case NVME_NVM_NS_16B_GUARD:
1641                         integrity.profile = &t10_pi_type1_crc;
1642                         integrity.tag_size = sizeof(u16);
1643                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1644                         break;
1645                 case NVME_NVM_NS_64B_GUARD:
1646                         integrity.profile = &ext_pi_type1_crc64;
1647                         integrity.tag_size = sizeof(u16);
1648                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1649                         break;
1650                 default:
1651                         integrity.profile = NULL;
1652                         break;
1653                 }
1654                 break;
1655         default:
1656                 integrity.profile = NULL;
1657                 break;
1658         }
1659
1660         integrity.tuple_size = ns->ms;
1661         blk_integrity_register(disk, &integrity);
1662         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1663 }
1664 #else
1665 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1666                                 u32 max_integrity_segments)
1667 {
1668 }
1669 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1670
1671 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1672 {
1673         struct nvme_ctrl *ctrl = ns->ctrl;
1674         struct request_queue *queue = disk->queue;
1675         u32 size = queue_logical_block_size(queue);
1676
1677         if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns, UINT_MAX))
1678                 ctrl->max_discard_sectors = nvme_lba_to_sect(ns, ctrl->dmrsl);
1679
1680         if (ctrl->max_discard_sectors == 0) {
1681                 blk_queue_max_discard_sectors(queue, 0);
1682                 return;
1683         }
1684
1685         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1686                         NVME_DSM_MAX_RANGES);
1687
1688         queue->limits.discard_granularity = size;
1689
1690         /* If discard is already enabled, don't reset queue limits */
1691         if (queue->limits.max_discard_sectors)
1692                 return;
1693
1694         blk_queue_max_discard_sectors(queue, ctrl->max_discard_sectors);
1695         blk_queue_max_discard_segments(queue, ctrl->max_discard_segments);
1696
1697         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1698                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1699 }
1700
1701 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1702 {
1703         return uuid_equal(&a->uuid, &b->uuid) &&
1704                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1705                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1706                 a->csi == b->csi;
1707 }
1708
1709 static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id)
1710 {
1711         bool first = id->dps & NVME_NS_DPS_PI_FIRST;
1712         unsigned lbaf = nvme_lbaf_index(id->flbas);
1713         struct nvme_ctrl *ctrl = ns->ctrl;
1714         struct nvme_command c = { };
1715         struct nvme_id_ns_nvm *nvm;
1716         int ret = 0;
1717         u32 elbaf;
1718
1719         ns->pi_size = 0;
1720         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1721         if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1722                 ns->pi_size = sizeof(struct t10_pi_tuple);
1723                 ns->guard_type = NVME_NVM_NS_16B_GUARD;
1724                 goto set_pi;
1725         }
1726
1727         nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1728         if (!nvm)
1729                 return -ENOMEM;
1730
1731         c.identify.opcode = nvme_admin_identify;
1732         c.identify.nsid = cpu_to_le32(ns->head->ns_id);
1733         c.identify.cns = NVME_ID_CNS_CS_NS;
1734         c.identify.csi = NVME_CSI_NVM;
1735
1736         ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm));
1737         if (ret)
1738                 goto free_data;
1739
1740         elbaf = le32_to_cpu(nvm->elbaf[lbaf]);
1741
1742         /* no support for storage tag formats right now */
1743         if (nvme_elbaf_sts(elbaf))
1744                 goto free_data;
1745
1746         ns->guard_type = nvme_elbaf_guard_type(elbaf);
1747         switch (ns->guard_type) {
1748         case NVME_NVM_NS_64B_GUARD:
1749                 ns->pi_size = sizeof(struct crc64_pi_tuple);
1750                 break;
1751         case NVME_NVM_NS_16B_GUARD:
1752                 ns->pi_size = sizeof(struct t10_pi_tuple);
1753                 break;
1754         default:
1755                 break;
1756         }
1757
1758 free_data:
1759         kfree(nvm);
1760 set_pi:
1761         if (ns->pi_size && (first || ns->ms == ns->pi_size))
1762                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1763         else
1764                 ns->pi_type = 0;
1765
1766         return ret;
1767 }
1768
1769 static void nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1770 {
1771         struct nvme_ctrl *ctrl = ns->ctrl;
1772
1773         if (nvme_init_ms(ns, id))
1774                 return;
1775
1776         ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1777         if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1778                 return;
1779
1780         if (ctrl->ops->flags & NVME_F_FABRICS) {
1781                 /*
1782                  * The NVMe over Fabrics specification only supports metadata as
1783                  * part of the extended data LBA.  We rely on HCA/HBA support to
1784                  * remap the separate metadata buffer from the block layer.
1785                  */
1786                 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1787                         return;
1788
1789                 ns->features |= NVME_NS_EXT_LBAS;
1790
1791                 /*
1792                  * The current fabrics transport drivers support namespace
1793                  * metadata formats only if nvme_ns_has_pi() returns true.
1794                  * Suppress support for all other formats so the namespace will
1795                  * have a 0 capacity and not be usable through the block stack.
1796                  *
1797                  * Note, this check will need to be modified if any drivers
1798                  * gain the ability to use other metadata formats.
1799                  */
1800                 if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns))
1801                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1802         } else {
1803                 /*
1804                  * For PCIe controllers, we can't easily remap the separate
1805                  * metadata buffer from the block layer and thus require a
1806                  * separate metadata buffer for block layer metadata/PI support.
1807                  * We allow extended LBAs for the passthrough interface, though.
1808                  */
1809                 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1810                         ns->features |= NVME_NS_EXT_LBAS;
1811                 else
1812                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1813         }
1814 }
1815
1816 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1817                 struct request_queue *q)
1818 {
1819         bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1820
1821         if (ctrl->max_hw_sectors) {
1822                 u32 max_segments =
1823                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1824
1825                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1826                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1827                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1828         }
1829         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1830         blk_queue_dma_alignment(q, 3);
1831         blk_queue_write_cache(q, vwc, vwc);
1832 }
1833
1834 static void nvme_update_disk_info(struct gendisk *disk,
1835                 struct nvme_ns *ns, struct nvme_id_ns *id)
1836 {
1837         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1838         unsigned short bs = 1 << ns->lba_shift;
1839         u32 atomic_bs, phys_bs, io_opt = 0;
1840
1841         /*
1842          * The block layer can't support LBA sizes larger than the page size
1843          * yet, so catch this early and don't allow block I/O.
1844          */
1845         if (ns->lba_shift > PAGE_SHIFT) {
1846                 capacity = 0;
1847                 bs = (1 << 9);
1848         }
1849
1850         blk_integrity_unregister(disk);
1851
1852         atomic_bs = phys_bs = bs;
1853         if (id->nabo == 0) {
1854                 /*
1855                  * Bit 1 indicates whether NAWUPF is defined for this namespace
1856                  * and whether it should be used instead of AWUPF. If NAWUPF ==
1857                  * 0 then AWUPF must be used instead.
1858                  */
1859                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1860                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1861                 else
1862                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1863         }
1864
1865         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1866                 /* NPWG = Namespace Preferred Write Granularity */
1867                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
1868                 /* NOWS = Namespace Optimal Write Size */
1869                 io_opt = bs * (1 + le16_to_cpu(id->nows));
1870         }
1871
1872         blk_queue_logical_block_size(disk->queue, bs);
1873         /*
1874          * Linux filesystems assume writing a single physical block is
1875          * an atomic operation. Hence limit the physical block size to the
1876          * value of the Atomic Write Unit Power Fail parameter.
1877          */
1878         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1879         blk_queue_io_min(disk->queue, phys_bs);
1880         blk_queue_io_opt(disk->queue, io_opt);
1881
1882         /*
1883          * Register a metadata profile for PI, or the plain non-integrity NVMe
1884          * metadata masquerading as Type 0 if supported, otherwise reject block
1885          * I/O to namespaces with metadata except when the namespace supports
1886          * PI, as it can strip/insert in that case.
1887          */
1888         if (ns->ms) {
1889                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1890                     (ns->features & NVME_NS_METADATA_SUPPORTED))
1891                         nvme_init_integrity(disk, ns,
1892                                             ns->ctrl->max_integrity_segments);
1893                 else if (!nvme_ns_has_pi(ns))
1894                         capacity = 0;
1895         }
1896
1897         set_capacity_and_notify(disk, capacity);
1898
1899         nvme_config_discard(disk, ns);
1900         blk_queue_max_write_zeroes_sectors(disk->queue,
1901                                            ns->ctrl->max_zeroes_sectors);
1902 }
1903
1904 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info)
1905 {
1906         return info->is_readonly || test_bit(NVME_NS_FORCE_RO, &ns->flags);
1907 }
1908
1909 static inline bool nvme_first_scan(struct gendisk *disk)
1910 {
1911         /* nvme_alloc_ns() scans the disk prior to adding it */
1912         return !disk_live(disk);
1913 }
1914
1915 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
1916 {
1917         struct nvme_ctrl *ctrl = ns->ctrl;
1918         u32 iob;
1919
1920         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1921             is_power_of_2(ctrl->max_hw_sectors))
1922                 iob = ctrl->max_hw_sectors;
1923         else
1924                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
1925
1926         if (!iob)
1927                 return;
1928
1929         if (!is_power_of_2(iob)) {
1930                 if (nvme_first_scan(ns->disk))
1931                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
1932                                 ns->disk->disk_name, iob);
1933                 return;
1934         }
1935
1936         if (blk_queue_is_zoned(ns->disk->queue)) {
1937                 if (nvme_first_scan(ns->disk))
1938                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
1939                                 ns->disk->disk_name);
1940                 return;
1941         }
1942
1943         blk_queue_chunk_sectors(ns->queue, iob);
1944 }
1945
1946 static int nvme_update_ns_info_generic(struct nvme_ns *ns,
1947                 struct nvme_ns_info *info)
1948 {
1949         blk_mq_freeze_queue(ns->disk->queue);
1950         nvme_set_queue_limits(ns->ctrl, ns->queue);
1951         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
1952         blk_mq_unfreeze_queue(ns->disk->queue);
1953
1954         if (nvme_ns_head_multipath(ns->head)) {
1955                 blk_mq_freeze_queue(ns->head->disk->queue);
1956                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
1957                 nvme_mpath_revalidate_paths(ns);
1958                 blk_stack_limits(&ns->head->disk->queue->limits,
1959                                  &ns->queue->limits, 0);
1960                 ns->head->disk->flags |= GENHD_FL_HIDDEN;
1961                 blk_mq_unfreeze_queue(ns->head->disk->queue);
1962         }
1963
1964         /* Hide the block-interface for these devices */
1965         ns->disk->flags |= GENHD_FL_HIDDEN;
1966         set_bit(NVME_NS_READY, &ns->flags);
1967
1968         return 0;
1969 }
1970
1971 static int nvme_update_ns_info_block(struct nvme_ns *ns,
1972                 struct nvme_ns_info *info)
1973 {
1974         struct nvme_id_ns *id;
1975         unsigned lbaf;
1976         int ret;
1977
1978         ret = nvme_identify_ns(ns->ctrl, info->nsid, &id);
1979         if (ret)
1980                 return ret;
1981
1982         blk_mq_freeze_queue(ns->disk->queue);
1983         lbaf = nvme_lbaf_index(id->flbas);
1984         ns->lba_shift = id->lbaf[lbaf].ds;
1985         nvme_set_queue_limits(ns->ctrl, ns->queue);
1986
1987         nvme_configure_metadata(ns, id);
1988         nvme_set_chunk_sectors(ns, id);
1989         nvme_update_disk_info(ns->disk, ns, id);
1990
1991         if (ns->head->ids.csi == NVME_CSI_ZNS) {
1992                 ret = nvme_update_zone_info(ns, lbaf);
1993                 if (ret) {
1994                         blk_mq_unfreeze_queue(ns->disk->queue);
1995                         goto out;
1996                 }
1997         }
1998
1999         /*
2000          * Only set the DEAC bit if the device guarantees that reads from
2001          * deallocated data return zeroes.  While the DEAC bit does not
2002          * require that, it must be a no-op if reads from deallocated data
2003          * do not return zeroes.
2004          */
2005         if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3)))
2006                 ns->features |= NVME_NS_DEAC;
2007         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2008         set_bit(NVME_NS_READY, &ns->flags);
2009         blk_mq_unfreeze_queue(ns->disk->queue);
2010
2011         if (blk_queue_is_zoned(ns->queue)) {
2012                 ret = nvme_revalidate_zones(ns);
2013                 if (ret && !nvme_first_scan(ns->disk))
2014                         goto out;
2015         }
2016
2017         if (nvme_ns_head_multipath(ns->head)) {
2018                 blk_mq_freeze_queue(ns->head->disk->queue);
2019                 nvme_update_disk_info(ns->head->disk, ns, id);
2020                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
2021                 nvme_mpath_revalidate_paths(ns);
2022                 blk_stack_limits(&ns->head->disk->queue->limits,
2023                                  &ns->queue->limits, 0);
2024                 disk_update_readahead(ns->head->disk);
2025                 blk_mq_unfreeze_queue(ns->head->disk->queue);
2026         }
2027
2028         ret = 0;
2029 out:
2030         /*
2031          * If probing fails due an unsupported feature, hide the block device,
2032          * but still allow other access.
2033          */
2034         if (ret == -ENODEV) {
2035                 ns->disk->flags |= GENHD_FL_HIDDEN;
2036                 set_bit(NVME_NS_READY, &ns->flags);
2037                 ret = 0;
2038         }
2039         kfree(id);
2040         return ret;
2041 }
2042
2043 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
2044 {
2045         switch (info->ids.csi) {
2046         case NVME_CSI_ZNS:
2047                 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2048                         dev_info(ns->ctrl->device,
2049         "block device for nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
2050                                 info->nsid);
2051                         return nvme_update_ns_info_generic(ns, info);
2052                 }
2053                 return nvme_update_ns_info_block(ns, info);
2054         case NVME_CSI_NVM:
2055                 return nvme_update_ns_info_block(ns, info);
2056         default:
2057                 dev_info(ns->ctrl->device,
2058                         "block device for nsid %u not supported (csi %u)\n",
2059                         info->nsid, info->ids.csi);
2060                 return nvme_update_ns_info_generic(ns, info);
2061         }
2062 }
2063
2064 static char nvme_pr_type(enum pr_type type)
2065 {
2066         switch (type) {
2067         case PR_WRITE_EXCLUSIVE:
2068                 return 1;
2069         case PR_EXCLUSIVE_ACCESS:
2070                 return 2;
2071         case PR_WRITE_EXCLUSIVE_REG_ONLY:
2072                 return 3;
2073         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2074                 return 4;
2075         case PR_WRITE_EXCLUSIVE_ALL_REGS:
2076                 return 5;
2077         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2078                 return 6;
2079         default:
2080                 return 0;
2081         }
2082 }
2083
2084 static int nvme_send_ns_head_pr_command(struct block_device *bdev,
2085                 struct nvme_command *c, u8 data[16])
2086 {
2087         struct nvme_ns_head *head = bdev->bd_disk->private_data;
2088         int srcu_idx = srcu_read_lock(&head->srcu);
2089         struct nvme_ns *ns = nvme_find_path(head);
2090         int ret = -EWOULDBLOCK;
2091
2092         if (ns) {
2093                 c->common.nsid = cpu_to_le32(ns->head->ns_id);
2094                 ret = nvme_submit_sync_cmd(ns->queue, c, data, 16);
2095         }
2096         srcu_read_unlock(&head->srcu, srcu_idx);
2097         return ret;
2098 }
2099
2100 static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c,
2101                 u8 data[16])
2102 {
2103         c->common.nsid = cpu_to_le32(ns->head->ns_id);
2104         return nvme_submit_sync_cmd(ns->queue, c, data, 16);
2105 }
2106
2107 static int nvme_sc_to_pr_err(int nvme_sc)
2108 {
2109         if (nvme_is_path_error(nvme_sc))
2110                 return PR_STS_PATH_FAILED;
2111
2112         switch (nvme_sc) {
2113         case NVME_SC_SUCCESS:
2114                 return PR_STS_SUCCESS;
2115         case NVME_SC_RESERVATION_CONFLICT:
2116                 return PR_STS_RESERVATION_CONFLICT;
2117         case NVME_SC_ONCS_NOT_SUPPORTED:
2118                 return -EOPNOTSUPP;
2119         case NVME_SC_BAD_ATTRIBUTES:
2120         case NVME_SC_INVALID_OPCODE:
2121         case NVME_SC_INVALID_FIELD:
2122         case NVME_SC_INVALID_NS:
2123                 return -EINVAL;
2124         default:
2125                 return PR_STS_IOERR;
2126         }
2127 }
2128
2129 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2130                                 u64 key, u64 sa_key, u8 op)
2131 {
2132         struct nvme_command c = { };
2133         u8 data[16] = { 0, };
2134         int ret;
2135
2136         put_unaligned_le64(key, &data[0]);
2137         put_unaligned_le64(sa_key, &data[8]);
2138
2139         c.common.opcode = op;
2140         c.common.cdw10 = cpu_to_le32(cdw10);
2141
2142         if (IS_ENABLED(CONFIG_NVME_MULTIPATH) &&
2143             bdev->bd_disk->fops == &nvme_ns_head_ops)
2144                 ret = nvme_send_ns_head_pr_command(bdev, &c, data);
2145         else
2146                 ret = nvme_send_ns_pr_command(bdev->bd_disk->private_data, &c,
2147                                               data);
2148         if (ret < 0)
2149                 return ret;
2150
2151         return nvme_sc_to_pr_err(ret);
2152 }
2153
2154 static int nvme_pr_register(struct block_device *bdev, u64 old,
2155                 u64 new, unsigned flags)
2156 {
2157         u32 cdw10;
2158
2159         if (flags & ~PR_FL_IGNORE_KEY)
2160                 return -EOPNOTSUPP;
2161
2162         cdw10 = old ? 2 : 0;
2163         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2164         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2165         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2166 }
2167
2168 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2169                 enum pr_type type, unsigned flags)
2170 {
2171         u32 cdw10;
2172
2173         if (flags & ~PR_FL_IGNORE_KEY)
2174                 return -EOPNOTSUPP;
2175
2176         cdw10 = nvme_pr_type(type) << 8;
2177         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2178         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2179 }
2180
2181 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2182                 enum pr_type type, bool abort)
2183 {
2184         u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2185
2186         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2187 }
2188
2189 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2190 {
2191         u32 cdw10 = 1 | (key ? 0 : 1 << 3);
2192
2193         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2194 }
2195
2196 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2197 {
2198         u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 0 : 1 << 3);
2199
2200         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2201 }
2202
2203 const struct pr_ops nvme_pr_ops = {
2204         .pr_register    = nvme_pr_register,
2205         .pr_reserve     = nvme_pr_reserve,
2206         .pr_release     = nvme_pr_release,
2207         .pr_preempt     = nvme_pr_preempt,
2208         .pr_clear       = nvme_pr_clear,
2209 };
2210
2211 #ifdef CONFIG_BLK_SED_OPAL
2212 static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2213                 bool send)
2214 {
2215         struct nvme_ctrl *ctrl = data;
2216         struct nvme_command cmd = { };
2217
2218         if (send)
2219                 cmd.common.opcode = nvme_admin_security_send;
2220         else
2221                 cmd.common.opcode = nvme_admin_security_recv;
2222         cmd.common.nsid = 0;
2223         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2224         cmd.common.cdw11 = cpu_to_le32(len);
2225
2226         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2227                         NVME_QID_ANY, 1, 0);
2228 }
2229
2230 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2231 {
2232         if (ctrl->oacs & NVME_CTRL_OACS_SEC_SUPP) {
2233                 if (!ctrl->opal_dev)
2234                         ctrl->opal_dev = init_opal_dev(ctrl, &nvme_sec_submit);
2235                 else if (was_suspended)
2236                         opal_unlock_from_suspend(ctrl->opal_dev);
2237         } else {
2238                 free_opal_dev(ctrl->opal_dev);
2239                 ctrl->opal_dev = NULL;
2240         }
2241 }
2242 #else
2243 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2244 {
2245 }
2246 #endif /* CONFIG_BLK_SED_OPAL */
2247
2248 #ifdef CONFIG_BLK_DEV_ZONED
2249 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2250                 unsigned int nr_zones, report_zones_cb cb, void *data)
2251 {
2252         return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2253                         data);
2254 }
2255 #else
2256 #define nvme_report_zones       NULL
2257 #endif /* CONFIG_BLK_DEV_ZONED */
2258
2259 static const struct block_device_operations nvme_bdev_ops = {
2260         .owner          = THIS_MODULE,
2261         .ioctl          = nvme_ioctl,
2262         .compat_ioctl   = blkdev_compat_ptr_ioctl,
2263         .open           = nvme_open,
2264         .release        = nvme_release,
2265         .getgeo         = nvme_getgeo,
2266         .report_zones   = nvme_report_zones,
2267         .pr_ops         = &nvme_pr_ops,
2268 };
2269
2270 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
2271                 u32 timeout, const char *op)
2272 {
2273         unsigned long timeout_jiffies = jiffies + timeout * HZ;
2274         u32 csts;
2275         int ret;
2276
2277         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2278                 if (csts == ~0)
2279                         return -ENODEV;
2280                 if ((csts & mask) == val)
2281                         break;
2282
2283                 usleep_range(1000, 2000);
2284                 if (fatal_signal_pending(current))
2285                         return -EINTR;
2286                 if (time_after(jiffies, timeout_jiffies)) {
2287                         dev_err(ctrl->device,
2288                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2289                                 op, csts);
2290                         return -ENODEV;
2291                 }
2292         }
2293
2294         return ret;
2295 }
2296
2297 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2298 {
2299         int ret;
2300
2301         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2302         if (shutdown)
2303                 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2304         else
2305                 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2306
2307         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2308         if (ret)
2309                 return ret;
2310
2311         if (shutdown) {
2312                 return nvme_wait_ready(ctrl, NVME_CSTS_SHST_MASK,
2313                                        NVME_CSTS_SHST_CMPLT,
2314                                        ctrl->shutdown_timeout, "shutdown");
2315         }
2316         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2317                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2318         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, 0,
2319                                (NVME_CAP_TIMEOUT(ctrl->cap) + 1) / 2, "reset");
2320 }
2321 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2322
2323 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2324 {
2325         unsigned dev_page_min;
2326         u32 timeout;
2327         int ret;
2328
2329         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2330         if (ret) {
2331                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2332                 return ret;
2333         }
2334         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2335
2336         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2337                 dev_err(ctrl->device,
2338                         "Minimum device page size %u too large for host (%u)\n",
2339                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2340                 return -ENODEV;
2341         }
2342
2343         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2344                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2345         else
2346                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2347
2348         if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2349                 u32 crto;
2350
2351                 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2352                 if (ret) {
2353                         dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2354                                 ret);
2355                         return ret;
2356                 }
2357
2358                 if (ctrl->cap & NVME_CAP_CRMS_CRIMS) {
2359                         ctrl->ctrl_config |= NVME_CC_CRIME;
2360                         timeout = NVME_CRTO_CRIMT(crto);
2361                 } else {
2362                         timeout = NVME_CRTO_CRWMT(crto);
2363                 }
2364         } else {
2365                 timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2366         }
2367
2368         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2369         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2370         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2371         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2372         if (ret)
2373                 return ret;
2374
2375         /* Flush write to device (required if transport is PCI) */
2376         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CC, &ctrl->ctrl_config);
2377         if (ret)
2378                 return ret;
2379
2380         ctrl->ctrl_config |= NVME_CC_ENABLE;
2381         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2382         if (ret)
2383                 return ret;
2384         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, NVME_CSTS_RDY,
2385                                (timeout + 1) / 2, "initialisation");
2386 }
2387 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2388
2389 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2390 {
2391         __le64 ts;
2392         int ret;
2393
2394         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2395                 return 0;
2396
2397         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2398         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2399                         NULL);
2400         if (ret)
2401                 dev_warn_once(ctrl->device,
2402                         "could not set timestamp (%d)\n", ret);
2403         return ret;
2404 }
2405
2406 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2407 {
2408         struct nvme_feat_host_behavior *host;
2409         u8 acre = 0, lbafee = 0;
2410         int ret;
2411
2412         /* Don't bother enabling the feature if retry delay is not reported */
2413         if (ctrl->crdt[0])
2414                 acre = NVME_ENABLE_ACRE;
2415         if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2416                 lbafee = NVME_ENABLE_LBAFEE;
2417
2418         if (!acre && !lbafee)
2419                 return 0;
2420
2421         host = kzalloc(sizeof(*host), GFP_KERNEL);
2422         if (!host)
2423                 return 0;
2424
2425         host->acre = acre;
2426         host->lbafee = lbafee;
2427         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2428                                 host, sizeof(*host), NULL);
2429         kfree(host);
2430         return ret;
2431 }
2432
2433 /*
2434  * The function checks whether the given total (exlat + enlat) latency of
2435  * a power state allows the latter to be used as an APST transition target.
2436  * It does so by comparing the latency to the primary and secondary latency
2437  * tolerances defined by module params. If there's a match, the corresponding
2438  * timeout value is returned and the matching tolerance index (1 or 2) is
2439  * reported.
2440  */
2441 static bool nvme_apst_get_transition_time(u64 total_latency,
2442                 u64 *transition_time, unsigned *last_index)
2443 {
2444         if (total_latency <= apst_primary_latency_tol_us) {
2445                 if (*last_index == 1)
2446                         return false;
2447                 *last_index = 1;
2448                 *transition_time = apst_primary_timeout_ms;
2449                 return true;
2450         }
2451         if (apst_secondary_timeout_ms &&
2452                 total_latency <= apst_secondary_latency_tol_us) {
2453                 if (*last_index <= 2)
2454                         return false;
2455                 *last_index = 2;
2456                 *transition_time = apst_secondary_timeout_ms;
2457                 return true;
2458         }
2459         return false;
2460 }
2461
2462 /*
2463  * APST (Autonomous Power State Transition) lets us program a table of power
2464  * state transitions that the controller will perform automatically.
2465  *
2466  * Depending on module params, one of the two supported techniques will be used:
2467  *
2468  * - If the parameters provide explicit timeouts and tolerances, they will be
2469  *   used to build a table with up to 2 non-operational states to transition to.
2470  *   The default parameter values were selected based on the values used by
2471  *   Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2472  *   regeneration of the APST table in the event of switching between external
2473  *   and battery power, the timeouts and tolerances reflect a compromise
2474  *   between values used by Microsoft for AC and battery scenarios.
2475  * - If not, we'll configure the table with a simple heuristic: we are willing
2476  *   to spend at most 2% of the time transitioning between power states.
2477  *   Therefore, when running in any given state, we will enter the next
2478  *   lower-power non-operational state after waiting 50 * (enlat + exlat)
2479  *   microseconds, as long as that state's exit latency is under the requested
2480  *   maximum latency.
2481  *
2482  * We will not autonomously enter any non-operational state for which the total
2483  * latency exceeds ps_max_latency_us.
2484  *
2485  * Users can set ps_max_latency_us to zero to turn off APST.
2486  */
2487 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2488 {
2489         struct nvme_feat_auto_pst *table;
2490         unsigned apste = 0;
2491         u64 max_lat_us = 0;
2492         __le64 target = 0;
2493         int max_ps = -1;
2494         int state;
2495         int ret;
2496         unsigned last_lt_index = UINT_MAX;
2497
2498         /*
2499          * If APST isn't supported or if we haven't been initialized yet,
2500          * then don't do anything.
2501          */
2502         if (!ctrl->apsta)
2503                 return 0;
2504
2505         if (ctrl->npss > 31) {
2506                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2507                 return 0;
2508         }
2509
2510         table = kzalloc(sizeof(*table), GFP_KERNEL);
2511         if (!table)
2512                 return 0;
2513
2514         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2515                 /* Turn off APST. */
2516                 dev_dbg(ctrl->device, "APST disabled\n");
2517                 goto done;
2518         }
2519
2520         /*
2521          * Walk through all states from lowest- to highest-power.
2522          * According to the spec, lower-numbered states use more power.  NPSS,
2523          * despite the name, is the index of the lowest-power state, not the
2524          * number of states.
2525          */
2526         for (state = (int)ctrl->npss; state >= 0; state--) {
2527                 u64 total_latency_us, exit_latency_us, transition_ms;
2528
2529                 if (target)
2530                         table->entries[state] = target;
2531
2532                 /*
2533                  * Don't allow transitions to the deepest state if it's quirked
2534                  * off.
2535                  */
2536                 if (state == ctrl->npss &&
2537                     (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2538                         continue;
2539
2540                 /*
2541                  * Is this state a useful non-operational state for higher-power
2542                  * states to autonomously transition to?
2543                  */
2544                 if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2545                         continue;
2546
2547                 exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2548                 if (exit_latency_us > ctrl->ps_max_latency_us)
2549                         continue;
2550
2551                 total_latency_us = exit_latency_us +
2552                         le32_to_cpu(ctrl->psd[state].entry_lat);
2553
2554                 /*
2555                  * This state is good. It can be used as the APST idle target
2556                  * for higher power states.
2557                  */
2558                 if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2559                         if (!nvme_apst_get_transition_time(total_latency_us,
2560                                         &transition_ms, &last_lt_index))
2561                                 continue;
2562                 } else {
2563                         transition_ms = total_latency_us + 19;
2564                         do_div(transition_ms, 20);
2565                         if (transition_ms > (1 << 24) - 1)
2566                                 transition_ms = (1 << 24) - 1;
2567                 }
2568
2569                 target = cpu_to_le64((state << 3) | (transition_ms << 8));
2570                 if (max_ps == -1)
2571                         max_ps = state;
2572                 if (total_latency_us > max_lat_us)
2573                         max_lat_us = total_latency_us;
2574         }
2575
2576         if (max_ps == -1)
2577                 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2578         else
2579                 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2580                         max_ps, max_lat_us, (int)sizeof(*table), table);
2581         apste = 1;
2582
2583 done:
2584         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2585                                 table, sizeof(*table), NULL);
2586         if (ret)
2587                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2588         kfree(table);
2589         return ret;
2590 }
2591
2592 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2593 {
2594         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2595         u64 latency;
2596
2597         switch (val) {
2598         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2599         case PM_QOS_LATENCY_ANY:
2600                 latency = U64_MAX;
2601                 break;
2602
2603         default:
2604                 latency = val;
2605         }
2606
2607         if (ctrl->ps_max_latency_us != latency) {
2608                 ctrl->ps_max_latency_us = latency;
2609                 if (ctrl->state == NVME_CTRL_LIVE)
2610                         nvme_configure_apst(ctrl);
2611         }
2612 }
2613
2614 struct nvme_core_quirk_entry {
2615         /*
2616          * NVMe model and firmware strings are padded with spaces.  For
2617          * simplicity, strings in the quirk table are padded with NULLs
2618          * instead.
2619          */
2620         u16 vid;
2621         const char *mn;
2622         const char *fr;
2623         unsigned long quirks;
2624 };
2625
2626 static const struct nvme_core_quirk_entry core_quirks[] = {
2627         {
2628                 /*
2629                  * This Toshiba device seems to die using any APST states.  See:
2630                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2631                  */
2632                 .vid = 0x1179,
2633                 .mn = "THNSF5256GPUK TOSHIBA",
2634                 .quirks = NVME_QUIRK_NO_APST,
2635         },
2636         {
2637                 /*
2638                  * This LiteON CL1-3D*-Q11 firmware version has a race
2639                  * condition associated with actions related to suspend to idle
2640                  * LiteON has resolved the problem in future firmware
2641                  */
2642                 .vid = 0x14a4,
2643                 .fr = "22301111",
2644                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2645         },
2646         {
2647                 /*
2648                  * This Kioxia CD6-V Series / HPE PE8030 device times out and
2649                  * aborts I/O during any load, but more easily reproducible
2650                  * with discards (fstrim).
2651                  *
2652                  * The device is left in a state where it is also not possible
2653                  * to use "nvme set-feature" to disable APST, but booting with
2654                  * nvme_core.default_ps_max_latency=0 works.
2655                  */
2656                 .vid = 0x1e0f,
2657                 .mn = "KCD6XVUL6T40",
2658                 .quirks = NVME_QUIRK_NO_APST,
2659         },
2660         {
2661                 /*
2662                  * The external Samsung X5 SSD fails initialization without a
2663                  * delay before checking if it is ready and has a whole set of
2664                  * other problems.  To make this even more interesting, it
2665                  * shares the PCI ID with internal Samsung 970 Evo Plus that
2666                  * does not need or want these quirks.
2667                  */
2668                 .vid = 0x144d,
2669                 .mn = "Samsung Portable SSD X5",
2670                 .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2671                           NVME_QUIRK_NO_DEEPEST_PS |
2672                           NVME_QUIRK_IGNORE_DEV_SUBNQN,
2673         }
2674 };
2675
2676 /* match is null-terminated but idstr is space-padded. */
2677 static bool string_matches(const char *idstr, const char *match, size_t len)
2678 {
2679         size_t matchlen;
2680
2681         if (!match)
2682                 return true;
2683
2684         matchlen = strlen(match);
2685         WARN_ON_ONCE(matchlen > len);
2686
2687         if (memcmp(idstr, match, matchlen))
2688                 return false;
2689
2690         for (; matchlen < len; matchlen++)
2691                 if (idstr[matchlen] != ' ')
2692                         return false;
2693
2694         return true;
2695 }
2696
2697 static bool quirk_matches(const struct nvme_id_ctrl *id,
2698                           const struct nvme_core_quirk_entry *q)
2699 {
2700         return q->vid == le16_to_cpu(id->vid) &&
2701                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2702                 string_matches(id->fr, q->fr, sizeof(id->fr));
2703 }
2704
2705 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2706                 struct nvme_id_ctrl *id)
2707 {
2708         size_t nqnlen;
2709         int off;
2710
2711         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2712                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2713                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2714                         strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2715                         return;
2716                 }
2717
2718                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2719                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2720         }
2721
2722         /*
2723          * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe
2724          * Base Specification 2.0.  It is slightly different from the format
2725          * specified there due to historic reasons, and we can't change it now.
2726          */
2727         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2728                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2729                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2730         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2731         off += sizeof(id->sn);
2732         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2733         off += sizeof(id->mn);
2734         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2735 }
2736
2737 static void nvme_release_subsystem(struct device *dev)
2738 {
2739         struct nvme_subsystem *subsys =
2740                 container_of(dev, struct nvme_subsystem, dev);
2741
2742         if (subsys->instance >= 0)
2743                 ida_free(&nvme_instance_ida, subsys->instance);
2744         kfree(subsys);
2745 }
2746
2747 static void nvme_destroy_subsystem(struct kref *ref)
2748 {
2749         struct nvme_subsystem *subsys =
2750                         container_of(ref, struct nvme_subsystem, ref);
2751
2752         mutex_lock(&nvme_subsystems_lock);
2753         list_del(&subsys->entry);
2754         mutex_unlock(&nvme_subsystems_lock);
2755
2756         ida_destroy(&subsys->ns_ida);
2757         device_del(&subsys->dev);
2758         put_device(&subsys->dev);
2759 }
2760
2761 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2762 {
2763         kref_put(&subsys->ref, nvme_destroy_subsystem);
2764 }
2765
2766 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2767 {
2768         struct nvme_subsystem *subsys;
2769
2770         lockdep_assert_held(&nvme_subsystems_lock);
2771
2772         /*
2773          * Fail matches for discovery subsystems. This results
2774          * in each discovery controller bound to a unique subsystem.
2775          * This avoids issues with validating controller values
2776          * that can only be true when there is a single unique subsystem.
2777          * There may be multiple and completely independent entities
2778          * that provide discovery controllers.
2779          */
2780         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2781                 return NULL;
2782
2783         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2784                 if (strcmp(subsys->subnqn, subsysnqn))
2785                         continue;
2786                 if (!kref_get_unless_zero(&subsys->ref))
2787                         continue;
2788                 return subsys;
2789         }
2790
2791         return NULL;
2792 }
2793
2794 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2795         struct device_attribute subsys_attr_##_name = \
2796                 __ATTR(_name, _mode, _show, NULL)
2797
2798 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2799                                     struct device_attribute *attr,
2800                                     char *buf)
2801 {
2802         struct nvme_subsystem *subsys =
2803                 container_of(dev, struct nvme_subsystem, dev);
2804
2805         return sysfs_emit(buf, "%s\n", subsys->subnqn);
2806 }
2807 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2808
2809 static ssize_t nvme_subsys_show_type(struct device *dev,
2810                                     struct device_attribute *attr,
2811                                     char *buf)
2812 {
2813         struct nvme_subsystem *subsys =
2814                 container_of(dev, struct nvme_subsystem, dev);
2815
2816         switch (subsys->subtype) {
2817         case NVME_NQN_DISC:
2818                 return sysfs_emit(buf, "discovery\n");
2819         case NVME_NQN_NVME:
2820                 return sysfs_emit(buf, "nvm\n");
2821         default:
2822                 return sysfs_emit(buf, "reserved\n");
2823         }
2824 }
2825 static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type);
2826
2827 #define nvme_subsys_show_str_function(field)                            \
2828 static ssize_t subsys_##field##_show(struct device *dev,                \
2829                             struct device_attribute *attr, char *buf)   \
2830 {                                                                       \
2831         struct nvme_subsystem *subsys =                                 \
2832                 container_of(dev, struct nvme_subsystem, dev);          \
2833         return sysfs_emit(buf, "%.*s\n",                                \
2834                            (int)sizeof(subsys->field), subsys->field);  \
2835 }                                                                       \
2836 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2837
2838 nvme_subsys_show_str_function(model);
2839 nvme_subsys_show_str_function(serial);
2840 nvme_subsys_show_str_function(firmware_rev);
2841
2842 static struct attribute *nvme_subsys_attrs[] = {
2843         &subsys_attr_model.attr,
2844         &subsys_attr_serial.attr,
2845         &subsys_attr_firmware_rev.attr,
2846         &subsys_attr_subsysnqn.attr,
2847         &subsys_attr_subsystype.attr,
2848 #ifdef CONFIG_NVME_MULTIPATH
2849         &subsys_attr_iopolicy.attr,
2850 #endif
2851         NULL,
2852 };
2853
2854 static const struct attribute_group nvme_subsys_attrs_group = {
2855         .attrs = nvme_subsys_attrs,
2856 };
2857
2858 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2859         &nvme_subsys_attrs_group,
2860         NULL,
2861 };
2862
2863 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2864 {
2865         return ctrl->opts && ctrl->opts->discovery_nqn;
2866 }
2867
2868 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2869                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2870 {
2871         struct nvme_ctrl *tmp;
2872
2873         lockdep_assert_held(&nvme_subsystems_lock);
2874
2875         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2876                 if (nvme_state_terminal(tmp))
2877                         continue;
2878
2879                 if (tmp->cntlid == ctrl->cntlid) {
2880                         dev_err(ctrl->device,
2881                                 "Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2882                                 ctrl->cntlid, dev_name(tmp->device),
2883                                 subsys->subnqn);
2884                         return false;
2885                 }
2886
2887                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2888                     nvme_discovery_ctrl(ctrl))
2889                         continue;
2890
2891                 dev_err(ctrl->device,
2892                         "Subsystem does not support multiple controllers\n");
2893                 return false;
2894         }
2895
2896         return true;
2897 }
2898
2899 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2900 {
2901         struct nvme_subsystem *subsys, *found;
2902         int ret;
2903
2904         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2905         if (!subsys)
2906                 return -ENOMEM;
2907
2908         subsys->instance = -1;
2909         mutex_init(&subsys->lock);
2910         kref_init(&subsys->ref);
2911         INIT_LIST_HEAD(&subsys->ctrls);
2912         INIT_LIST_HEAD(&subsys->nsheads);
2913         nvme_init_subnqn(subsys, ctrl, id);
2914         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2915         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2916         subsys->vendor_id = le16_to_cpu(id->vid);
2917         subsys->cmic = id->cmic;
2918
2919         /* Versions prior to 1.4 don't necessarily report a valid type */
2920         if (id->cntrltype == NVME_CTRL_DISC ||
2921             !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2922                 subsys->subtype = NVME_NQN_DISC;
2923         else
2924                 subsys->subtype = NVME_NQN_NVME;
2925
2926         if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2927                 dev_err(ctrl->device,
2928                         "Subsystem %s is not a discovery controller",
2929                         subsys->subnqn);
2930                 kfree(subsys);
2931                 return -EINVAL;
2932         }
2933         subsys->awupf = le16_to_cpu(id->awupf);
2934         nvme_mpath_default_iopolicy(subsys);
2935
2936         subsys->dev.class = nvme_subsys_class;
2937         subsys->dev.release = nvme_release_subsystem;
2938         subsys->dev.groups = nvme_subsys_attrs_groups;
2939         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2940         device_initialize(&subsys->dev);
2941
2942         mutex_lock(&nvme_subsystems_lock);
2943         found = __nvme_find_get_subsystem(subsys->subnqn);
2944         if (found) {
2945                 put_device(&subsys->dev);
2946                 subsys = found;
2947
2948                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2949                         ret = -EINVAL;
2950                         goto out_put_subsystem;
2951                 }
2952         } else {
2953                 ret = device_add(&subsys->dev);
2954                 if (ret) {
2955                         dev_err(ctrl->device,
2956                                 "failed to register subsystem device.\n");
2957                         put_device(&subsys->dev);
2958                         goto out_unlock;
2959                 }
2960                 ida_init(&subsys->ns_ida);
2961                 list_add_tail(&subsys->entry, &nvme_subsystems);
2962         }
2963
2964         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2965                                 dev_name(ctrl->device));
2966         if (ret) {
2967                 dev_err(ctrl->device,
2968                         "failed to create sysfs link from subsystem.\n");
2969                 goto out_put_subsystem;
2970         }
2971
2972         if (!found)
2973                 subsys->instance = ctrl->instance;
2974         ctrl->subsys = subsys;
2975         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2976         mutex_unlock(&nvme_subsystems_lock);
2977         return 0;
2978
2979 out_put_subsystem:
2980         nvme_put_subsystem(subsys);
2981 out_unlock:
2982         mutex_unlock(&nvme_subsystems_lock);
2983         return ret;
2984 }
2985
2986 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2987                 void *log, size_t size, u64 offset)
2988 {
2989         struct nvme_command c = { };
2990         u32 dwlen = nvme_bytes_to_numd(size);
2991
2992         c.get_log_page.opcode = nvme_admin_get_log_page;
2993         c.get_log_page.nsid = cpu_to_le32(nsid);
2994         c.get_log_page.lid = log_page;
2995         c.get_log_page.lsp = lsp;
2996         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2997         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2998         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2999         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3000         c.get_log_page.csi = csi;
3001
3002         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3003 }
3004
3005 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3006                                 struct nvme_effects_log **log)
3007 {
3008         struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
3009         int ret;
3010
3011         if (cel)
3012                 goto out;
3013
3014         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3015         if (!cel)
3016                 return -ENOMEM;
3017
3018         ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
3019                         cel, sizeof(*cel), 0);
3020         if (ret) {
3021                 kfree(cel);
3022                 return ret;
3023         }
3024
3025         xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
3026 out:
3027         *log = cel;
3028         return 0;
3029 }
3030
3031 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
3032 {
3033         u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
3034
3035         if (check_shl_overflow(1U, units + page_shift - 9, &val))
3036                 return UINT_MAX;
3037         return val;
3038 }
3039
3040 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
3041 {
3042         struct nvme_command c = { };
3043         struct nvme_id_ctrl_nvm *id;
3044         int ret;
3045
3046         if (ctrl->oncs & NVME_CTRL_ONCS_DSM) {
3047                 ctrl->max_discard_sectors = UINT_MAX;
3048                 ctrl->max_discard_segments = NVME_DSM_MAX_RANGES;
3049         } else {
3050                 ctrl->max_discard_sectors = 0;
3051                 ctrl->max_discard_segments = 0;
3052         }
3053
3054         /*
3055          * Even though NVMe spec explicitly states that MDTS is not applicable
3056          * to the write-zeroes, we are cautious and limit the size to the
3057          * controllers max_hw_sectors value, which is based on the MDTS field
3058          * and possibly other limiting factors.
3059          */
3060         if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
3061             !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
3062                 ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
3063         else
3064                 ctrl->max_zeroes_sectors = 0;
3065
3066         if (ctrl->subsys->subtype != NVME_NQN_NVME ||
3067             nvme_ctrl_limited_cns(ctrl))
3068                 return 0;
3069
3070         id = kzalloc(sizeof(*id), GFP_KERNEL);
3071         if (!id)
3072                 return -ENOMEM;
3073
3074         c.identify.opcode = nvme_admin_identify;
3075         c.identify.cns = NVME_ID_CNS_CS_CTRL;
3076         c.identify.csi = NVME_CSI_NVM;
3077
3078         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
3079         if (ret)
3080                 goto free_data;
3081
3082         if (id->dmrl)
3083                 ctrl->max_discard_segments = id->dmrl;
3084         ctrl->dmrsl = le32_to_cpu(id->dmrsl);
3085         if (id->wzsl)
3086                 ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
3087
3088 free_data:
3089         kfree(id);
3090         return ret;
3091 }
3092
3093 static void nvme_init_known_nvm_effects(struct nvme_ctrl *ctrl)
3094 {
3095         struct nvme_effects_log *log = ctrl->effects;
3096
3097         log->acs[nvme_admin_format_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3098                                                 NVME_CMD_EFFECTS_NCC |
3099                                                 NVME_CMD_EFFECTS_CSE_MASK);
3100         log->acs[nvme_admin_sanitize_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3101                                                 NVME_CMD_EFFECTS_CSE_MASK);
3102
3103         /*
3104          * The spec says the result of a security receive command depends on
3105          * the previous security send command. As such, many vendors log this
3106          * command as one to submitted only when no other commands to the same
3107          * namespace are outstanding. The intention is to tell the host to
3108          * prevent mixing security send and receive.
3109          *
3110          * This driver can only enforce such exclusive access against IO
3111          * queues, though. We are not readily able to enforce such a rule for
3112          * two commands to the admin queue, which is the only queue that
3113          * matters for this command.
3114          *
3115          * Rather than blindly freezing the IO queues for this effect that
3116          * doesn't even apply to IO, mask it off.
3117          */
3118         log->acs[nvme_admin_security_recv] &= cpu_to_le32(~NVME_CMD_EFFECTS_CSE_MASK);
3119
3120         log->iocs[nvme_cmd_write] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3121         log->iocs[nvme_cmd_write_zeroes] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3122         log->iocs[nvme_cmd_write_uncor] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3123 }
3124
3125 static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
3126 {
3127         int ret = 0;
3128
3129         if (ctrl->effects)
3130                 return 0;
3131
3132         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3133                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3134                 if (ret < 0)
3135                         return ret;
3136         }
3137
3138         if (!ctrl->effects) {
3139                 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
3140                 if (!ctrl->effects)
3141                         return -ENOMEM;
3142                 xa_store(&ctrl->cels, NVME_CSI_NVM, ctrl->effects, GFP_KERNEL);
3143         }
3144
3145         nvme_init_known_nvm_effects(ctrl);
3146         return 0;
3147 }
3148
3149 static int nvme_init_identify(struct nvme_ctrl *ctrl)
3150 {
3151         struct nvme_id_ctrl *id;
3152         u32 max_hw_sectors;
3153         bool prev_apst_enabled;
3154         int ret;
3155
3156         ret = nvme_identify_ctrl(ctrl, &id);
3157         if (ret) {
3158                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3159                 return -EIO;
3160         }
3161
3162         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3163                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3164
3165         if (!ctrl->identified) {
3166                 unsigned int i;
3167
3168                 /*
3169                  * Check for quirks.  Quirk can depend on firmware version,
3170                  * so, in principle, the set of quirks present can change
3171                  * across a reset.  As a possible future enhancement, we
3172                  * could re-scan for quirks every time we reinitialize
3173                  * the device, but we'd have to make sure that the driver
3174                  * behaves intelligently if the quirks change.
3175                  */
3176                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3177                         if (quirk_matches(id, &core_quirks[i]))
3178                                 ctrl->quirks |= core_quirks[i].quirks;
3179                 }
3180
3181                 ret = nvme_init_subsystem(ctrl, id);
3182                 if (ret)
3183                         goto out_free;
3184
3185                 ret = nvme_init_effects(ctrl, id);
3186                 if (ret)
3187                         goto out_free;
3188         }
3189         memcpy(ctrl->subsys->firmware_rev, id->fr,
3190                sizeof(ctrl->subsys->firmware_rev));
3191
3192         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3193                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3194                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3195         }
3196
3197         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3198         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3199         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3200
3201         ctrl->oacs = le16_to_cpu(id->oacs);
3202         ctrl->oncs = le16_to_cpu(id->oncs);
3203         ctrl->mtfa = le16_to_cpu(id->mtfa);
3204         ctrl->oaes = le32_to_cpu(id->oaes);
3205         ctrl->wctemp = le16_to_cpu(id->wctemp);
3206         ctrl->cctemp = le16_to_cpu(id->cctemp);
3207
3208         atomic_set(&ctrl->abort_limit, id->acl + 1);
3209         ctrl->vwc = id->vwc;
3210         if (id->mdts)
3211                 max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3212         else
3213                 max_hw_sectors = UINT_MAX;
3214         ctrl->max_hw_sectors =
3215                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3216
3217         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3218         ctrl->sgls = le32_to_cpu(id->sgls);
3219         ctrl->kas = le16_to_cpu(id->kas);
3220         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3221         ctrl->ctratt = le32_to_cpu(id->ctratt);
3222
3223         ctrl->cntrltype = id->cntrltype;
3224         ctrl->dctype = id->dctype;
3225
3226         if (id->rtd3e) {
3227                 /* us -> s */
3228                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3229
3230                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3231                                                  shutdown_timeout, 60);
3232
3233                 if (ctrl->shutdown_timeout != shutdown_timeout)
3234                         dev_info(ctrl->device,
3235                                  "Shutdown timeout set to %u seconds\n",
3236                                  ctrl->shutdown_timeout);
3237         } else
3238                 ctrl->shutdown_timeout = shutdown_timeout;
3239
3240         ctrl->npss = id->npss;
3241         ctrl->apsta = id->apsta;
3242         prev_apst_enabled = ctrl->apst_enabled;
3243         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3244                 if (force_apst && id->apsta) {
3245                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3246                         ctrl->apst_enabled = true;
3247                 } else {
3248                         ctrl->apst_enabled = false;
3249                 }
3250         } else {
3251                 ctrl->apst_enabled = id->apsta;
3252         }
3253         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3254
3255         if (ctrl->ops->flags & NVME_F_FABRICS) {
3256                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3257                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3258                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3259                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3260
3261                 /*
3262                  * In fabrics we need to verify the cntlid matches the
3263                  * admin connect
3264                  */
3265                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3266                         dev_err(ctrl->device,
3267                                 "Mismatching cntlid: Connect %u vs Identify "
3268                                 "%u, rejecting\n",
3269                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3270                         ret = -EINVAL;
3271                         goto out_free;
3272                 }
3273
3274                 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3275                         dev_err(ctrl->device,
3276                                 "keep-alive support is mandatory for fabrics\n");
3277                         ret = -EINVAL;
3278                         goto out_free;
3279                 }
3280         } else {
3281                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3282                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3283                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3284                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3285         }
3286
3287         ret = nvme_mpath_init_identify(ctrl, id);
3288         if (ret < 0)
3289                 goto out_free;
3290
3291         if (ctrl->apst_enabled && !prev_apst_enabled)
3292                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3293         else if (!ctrl->apst_enabled && prev_apst_enabled)
3294                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3295
3296 out_free:
3297         kfree(id);
3298         return ret;
3299 }
3300
3301 /*
3302  * Initialize the cached copies of the Identify data and various controller
3303  * register in our nvme_ctrl structure.  This should be called as soon as
3304  * the admin queue is fully up and running.
3305  */
3306 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
3307 {
3308         int ret;
3309
3310         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3311         if (ret) {
3312                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3313                 return ret;
3314         }
3315
3316         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3317
3318         if (ctrl->vs >= NVME_VS(1, 1, 0))
3319                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3320
3321         ret = nvme_init_identify(ctrl);
3322         if (ret)
3323                 return ret;
3324
3325         ret = nvme_configure_apst(ctrl);
3326         if (ret < 0)
3327                 return ret;
3328
3329         ret = nvme_configure_timestamp(ctrl);
3330         if (ret < 0)
3331                 return ret;
3332
3333         ret = nvme_configure_host_options(ctrl);
3334         if (ret < 0)
3335                 return ret;
3336
3337         nvme_configure_opal(ctrl, was_suspended);
3338
3339         if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3340                 /*
3341                  * Do not return errors unless we are in a controller reset,
3342                  * the controller works perfectly fine without hwmon.
3343                  */
3344                 ret = nvme_hwmon_init(ctrl);
3345                 if (ret == -EINTR)
3346                         return ret;
3347         }
3348
3349         ctrl->identified = true;
3350
3351         return 0;
3352 }
3353 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3354
3355 static int nvme_dev_open(struct inode *inode, struct file *file)
3356 {
3357         struct nvme_ctrl *ctrl =
3358                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3359
3360         switch (ctrl->state) {
3361         case NVME_CTRL_LIVE:
3362                 break;
3363         default:
3364                 return -EWOULDBLOCK;
3365         }
3366
3367         nvme_get_ctrl(ctrl);
3368         if (!try_module_get(ctrl->ops->module)) {
3369                 nvme_put_ctrl(ctrl);
3370                 return -EINVAL;
3371         }
3372
3373         file->private_data = ctrl;
3374         return 0;
3375 }
3376
3377 static int nvme_dev_release(struct inode *inode, struct file *file)
3378 {
3379         struct nvme_ctrl *ctrl =
3380                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3381
3382         module_put(ctrl->ops->module);
3383         nvme_put_ctrl(ctrl);
3384         return 0;
3385 }
3386
3387 static const struct file_operations nvme_dev_fops = {
3388         .owner          = THIS_MODULE,
3389         .open           = nvme_dev_open,
3390         .release        = nvme_dev_release,
3391         .unlocked_ioctl = nvme_dev_ioctl,
3392         .compat_ioctl   = compat_ptr_ioctl,
3393         .uring_cmd      = nvme_dev_uring_cmd,
3394 };
3395
3396 static ssize_t nvme_sysfs_reset(struct device *dev,
3397                                 struct device_attribute *attr, const char *buf,
3398                                 size_t count)
3399 {
3400         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3401         int ret;
3402
3403         ret = nvme_reset_ctrl_sync(ctrl);
3404         if (ret < 0)
3405                 return ret;
3406         return count;
3407 }
3408 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3409
3410 static ssize_t nvme_sysfs_rescan(struct device *dev,
3411                                 struct device_attribute *attr, const char *buf,
3412                                 size_t count)
3413 {
3414         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3415
3416         nvme_queue_scan(ctrl);
3417         return count;
3418 }
3419 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3420
3421 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3422 {
3423         struct gendisk *disk = dev_to_disk(dev);
3424
3425         if (disk->fops == &nvme_bdev_ops)
3426                 return nvme_get_ns_from_dev(dev)->head;
3427         else
3428                 return disk->private_data;
3429 }
3430
3431 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3432                 char *buf)
3433 {
3434         struct nvme_ns_head *head = dev_to_ns_head(dev);
3435         struct nvme_ns_ids *ids = &head->ids;
3436         struct nvme_subsystem *subsys = head->subsys;
3437         int serial_len = sizeof(subsys->serial);
3438         int model_len = sizeof(subsys->model);
3439
3440         if (!uuid_is_null(&ids->uuid))
3441                 return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
3442
3443         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3444                 return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
3445
3446         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3447                 return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
3448
3449         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3450                                   subsys->serial[serial_len - 1] == '\0'))
3451                 serial_len--;
3452         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3453                                  subsys->model[model_len - 1] == '\0'))
3454                 model_len--;
3455
3456         return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3457                 serial_len, subsys->serial, model_len, subsys->model,
3458                 head->ns_id);
3459 }
3460 static DEVICE_ATTR_RO(wwid);
3461
3462 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3463                 char *buf)
3464 {
3465         return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3466 }
3467 static DEVICE_ATTR_RO(nguid);
3468
3469 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3470                 char *buf)
3471 {
3472         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3473
3474         /* For backward compatibility expose the NGUID to userspace if
3475          * we have no UUID set
3476          */
3477         if (uuid_is_null(&ids->uuid)) {
3478                 dev_warn_ratelimited(dev,
3479                         "No UUID available providing old NGUID\n");
3480                 return sysfs_emit(buf, "%pU\n", ids->nguid);
3481         }
3482         return sysfs_emit(buf, "%pU\n", &ids->uuid);
3483 }
3484 static DEVICE_ATTR_RO(uuid);
3485
3486 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3487                 char *buf)
3488 {
3489         return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3490 }
3491 static DEVICE_ATTR_RO(eui);
3492
3493 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3494                 char *buf)
3495 {
3496         return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3497 }
3498 static DEVICE_ATTR_RO(nsid);
3499
3500 static struct attribute *nvme_ns_id_attrs[] = {
3501         &dev_attr_wwid.attr,
3502         &dev_attr_uuid.attr,
3503         &dev_attr_nguid.attr,
3504         &dev_attr_eui.attr,
3505         &dev_attr_nsid.attr,
3506 #ifdef CONFIG_NVME_MULTIPATH
3507         &dev_attr_ana_grpid.attr,
3508         &dev_attr_ana_state.attr,
3509 #endif
3510         NULL,
3511 };
3512
3513 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3514                 struct attribute *a, int n)
3515 {
3516         struct device *dev = container_of(kobj, struct device, kobj);
3517         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3518
3519         if (a == &dev_attr_uuid.attr) {
3520                 if (uuid_is_null(&ids->uuid) &&
3521                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3522                         return 0;
3523         }
3524         if (a == &dev_attr_nguid.attr) {
3525                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3526                         return 0;
3527         }
3528         if (a == &dev_attr_eui.attr) {
3529                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3530                         return 0;
3531         }
3532 #ifdef CONFIG_NVME_MULTIPATH
3533         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3534                 if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */
3535                         return 0;
3536                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3537                         return 0;
3538         }
3539 #endif
3540         return a->mode;
3541 }
3542
3543 static const struct attribute_group nvme_ns_id_attr_group = {
3544         .attrs          = nvme_ns_id_attrs,
3545         .is_visible     = nvme_ns_id_attrs_are_visible,
3546 };
3547
3548 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3549         &nvme_ns_id_attr_group,
3550         NULL,
3551 };
3552
3553 #define nvme_show_str_function(field)                                           \
3554 static ssize_t  field##_show(struct device *dev,                                \
3555                             struct device_attribute *attr, char *buf)           \
3556 {                                                                               \
3557         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3558         return sysfs_emit(buf, "%.*s\n",                                        \
3559                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3560 }                                                                               \
3561 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3562
3563 nvme_show_str_function(model);
3564 nvme_show_str_function(serial);
3565 nvme_show_str_function(firmware_rev);
3566
3567 #define nvme_show_int_function(field)                                           \
3568 static ssize_t  field##_show(struct device *dev,                                \
3569                             struct device_attribute *attr, char *buf)           \
3570 {                                                                               \
3571         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3572         return sysfs_emit(buf, "%d\n", ctrl->field);                            \
3573 }                                                                               \
3574 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3575
3576 nvme_show_int_function(cntlid);
3577 nvme_show_int_function(numa_node);
3578 nvme_show_int_function(queue_count);
3579 nvme_show_int_function(sqsize);
3580 nvme_show_int_function(kato);
3581
3582 static ssize_t nvme_sysfs_delete(struct device *dev,
3583                                 struct device_attribute *attr, const char *buf,
3584                                 size_t count)
3585 {
3586         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3587
3588         if (!test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags))
3589                 return -EBUSY;
3590
3591         if (device_remove_file_self(dev, attr))
3592                 nvme_delete_ctrl_sync(ctrl);
3593         return count;
3594 }
3595 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3596
3597 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3598                                          struct device_attribute *attr,
3599                                          char *buf)
3600 {
3601         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3602
3603         return sysfs_emit(buf, "%s\n", ctrl->ops->name);
3604 }
3605 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3606
3607 static ssize_t nvme_sysfs_show_state(struct device *dev,
3608                                      struct device_attribute *attr,
3609                                      char *buf)
3610 {
3611         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3612         static const char *const state_name[] = {
3613                 [NVME_CTRL_NEW]         = "new",
3614                 [NVME_CTRL_LIVE]        = "live",
3615                 [NVME_CTRL_RESETTING]   = "resetting",
3616                 [NVME_CTRL_CONNECTING]  = "connecting",
3617                 [NVME_CTRL_DELETING]    = "deleting",
3618                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3619                 [NVME_CTRL_DEAD]        = "dead",
3620         };
3621
3622         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3623             state_name[ctrl->state])
3624                 return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
3625
3626         return sysfs_emit(buf, "unknown state\n");
3627 }
3628
3629 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3630
3631 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3632                                          struct device_attribute *attr,
3633                                          char *buf)
3634 {
3635         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3636
3637         return sysfs_emit(buf, "%s\n", ctrl->subsys->subnqn);
3638 }
3639 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3640
3641 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3642                                         struct device_attribute *attr,
3643                                         char *buf)
3644 {
3645         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3646
3647         return sysfs_emit(buf, "%s\n", ctrl->opts->host->nqn);
3648 }
3649 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3650
3651 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3652                                         struct device_attribute *attr,
3653                                         char *buf)
3654 {
3655         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3656
3657         return sysfs_emit(buf, "%pU\n", &ctrl->opts->host->id);
3658 }
3659 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3660
3661 static ssize_t nvme_sysfs_show_address(struct device *dev,
3662                                          struct device_attribute *attr,
3663                                          char *buf)
3664 {
3665         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3666
3667         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3668 }
3669 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3670
3671 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3672                 struct device_attribute *attr, char *buf)
3673 {
3674         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3675         struct nvmf_ctrl_options *opts = ctrl->opts;
3676
3677         if (ctrl->opts->max_reconnects == -1)
3678                 return sysfs_emit(buf, "off\n");
3679         return sysfs_emit(buf, "%d\n",
3680                           opts->max_reconnects * opts->reconnect_delay);
3681 }
3682
3683 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3684                 struct device_attribute *attr, const char *buf, size_t count)
3685 {
3686         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3687         struct nvmf_ctrl_options *opts = ctrl->opts;
3688         int ctrl_loss_tmo, err;
3689
3690         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3691         if (err)
3692                 return -EINVAL;
3693
3694         if (ctrl_loss_tmo < 0)
3695                 opts->max_reconnects = -1;
3696         else
3697                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3698                                                 opts->reconnect_delay);
3699         return count;
3700 }
3701 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3702         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3703
3704 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3705                 struct device_attribute *attr, char *buf)
3706 {
3707         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3708
3709         if (ctrl->opts->reconnect_delay == -1)
3710                 return sysfs_emit(buf, "off\n");
3711         return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
3712 }
3713
3714 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3715                 struct device_attribute *attr, const char *buf, size_t count)
3716 {
3717         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3718         unsigned int v;
3719         int err;
3720
3721         err = kstrtou32(buf, 10, &v);
3722         if (err)
3723                 return err;
3724
3725         ctrl->opts->reconnect_delay = v;
3726         return count;
3727 }
3728 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3729         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3730
3731 static ssize_t nvme_ctrl_fast_io_fail_tmo_show(struct device *dev,
3732                 struct device_attribute *attr, char *buf)
3733 {
3734         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3735
3736         if (ctrl->opts->fast_io_fail_tmo == -1)
3737                 return sysfs_emit(buf, "off\n");
3738         return sysfs_emit(buf, "%d\n", ctrl->opts->fast_io_fail_tmo);
3739 }
3740
3741 static ssize_t nvme_ctrl_fast_io_fail_tmo_store(struct device *dev,
3742                 struct device_attribute *attr, const char *buf, size_t count)
3743 {
3744         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3745         struct nvmf_ctrl_options *opts = ctrl->opts;
3746         int fast_io_fail_tmo, err;
3747
3748         err = kstrtoint(buf, 10, &fast_io_fail_tmo);
3749         if (err)
3750                 return -EINVAL;
3751
3752         if (fast_io_fail_tmo < 0)
3753                 opts->fast_io_fail_tmo = -1;
3754         else
3755                 opts->fast_io_fail_tmo = fast_io_fail_tmo;
3756         return count;
3757 }
3758 static DEVICE_ATTR(fast_io_fail_tmo, S_IRUGO | S_IWUSR,
3759         nvme_ctrl_fast_io_fail_tmo_show, nvme_ctrl_fast_io_fail_tmo_store);
3760
3761 static ssize_t cntrltype_show(struct device *dev,
3762                               struct device_attribute *attr, char *buf)
3763 {
3764         static const char * const type[] = {
3765                 [NVME_CTRL_IO] = "io\n",
3766                 [NVME_CTRL_DISC] = "discovery\n",
3767                 [NVME_CTRL_ADMIN] = "admin\n",
3768         };
3769         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3770
3771         if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
3772                 return sysfs_emit(buf, "reserved\n");
3773
3774         return sysfs_emit(buf, type[ctrl->cntrltype]);
3775 }
3776 static DEVICE_ATTR_RO(cntrltype);
3777
3778 static ssize_t dctype_show(struct device *dev,
3779                            struct device_attribute *attr, char *buf)
3780 {
3781         static const char * const type[] = {
3782                 [NVME_DCTYPE_NOT_REPORTED] = "none\n",
3783                 [NVME_DCTYPE_DDC] = "ddc\n",
3784                 [NVME_DCTYPE_CDC] = "cdc\n",
3785         };
3786         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3787
3788         if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
3789                 return sysfs_emit(buf, "reserved\n");
3790
3791         return sysfs_emit(buf, type[ctrl->dctype]);
3792 }
3793 static DEVICE_ATTR_RO(dctype);
3794
3795 #ifdef CONFIG_NVME_AUTH
3796 static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev,
3797                 struct device_attribute *attr, char *buf)
3798 {
3799         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3800         struct nvmf_ctrl_options *opts = ctrl->opts;
3801
3802         if (!opts->dhchap_secret)
3803                 return sysfs_emit(buf, "none\n");
3804         return sysfs_emit(buf, "%s\n", opts->dhchap_secret);
3805 }
3806
3807 static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev,
3808                 struct device_attribute *attr, const char *buf, size_t count)
3809 {
3810         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3811         struct nvmf_ctrl_options *opts = ctrl->opts;
3812         char *dhchap_secret;
3813
3814         if (!ctrl->opts->dhchap_secret)
3815                 return -EINVAL;
3816         if (count < 7)
3817                 return -EINVAL;
3818         if (memcmp(buf, "DHHC-1:", 7))
3819                 return -EINVAL;
3820
3821         dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3822         if (!dhchap_secret)
3823                 return -ENOMEM;
3824         memcpy(dhchap_secret, buf, count);
3825         nvme_auth_stop(ctrl);
3826         if (strcmp(dhchap_secret, opts->dhchap_secret)) {
3827                 struct nvme_dhchap_key *key, *host_key;
3828                 int ret;
3829
3830                 ret = nvme_auth_generate_key(dhchap_secret, &key);
3831                 if (ret)
3832                         return ret;
3833                 kfree(opts->dhchap_secret);
3834                 opts->dhchap_secret = dhchap_secret;
3835                 host_key = ctrl->host_key;
3836                 mutex_lock(&ctrl->dhchap_auth_mutex);
3837                 ctrl->host_key = key;
3838                 mutex_unlock(&ctrl->dhchap_auth_mutex);
3839                 nvme_auth_free_key(host_key);
3840         }
3841         /* Start re-authentication */
3842         dev_info(ctrl->device, "re-authenticating controller\n");
3843         queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3844
3845         return count;
3846 }
3847 static DEVICE_ATTR(dhchap_secret, S_IRUGO | S_IWUSR,
3848         nvme_ctrl_dhchap_secret_show, nvme_ctrl_dhchap_secret_store);
3849
3850 static ssize_t nvme_ctrl_dhchap_ctrl_secret_show(struct device *dev,
3851                 struct device_attribute *attr, char *buf)
3852 {
3853         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3854         struct nvmf_ctrl_options *opts = ctrl->opts;
3855
3856         if (!opts->dhchap_ctrl_secret)
3857                 return sysfs_emit(buf, "none\n");
3858         return sysfs_emit(buf, "%s\n", opts->dhchap_ctrl_secret);
3859 }
3860
3861 static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev,
3862                 struct device_attribute *attr, const char *buf, size_t count)
3863 {
3864         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3865         struct nvmf_ctrl_options *opts = ctrl->opts;
3866         char *dhchap_secret;
3867
3868         if (!ctrl->opts->dhchap_ctrl_secret)
3869                 return -EINVAL;
3870         if (count < 7)
3871                 return -EINVAL;
3872         if (memcmp(buf, "DHHC-1:", 7))
3873                 return -EINVAL;
3874
3875         dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3876         if (!dhchap_secret)
3877                 return -ENOMEM;
3878         memcpy(dhchap_secret, buf, count);
3879         nvme_auth_stop(ctrl);
3880         if (strcmp(dhchap_secret, opts->dhchap_ctrl_secret)) {
3881                 struct nvme_dhchap_key *key, *ctrl_key;
3882                 int ret;
3883
3884                 ret = nvme_auth_generate_key(dhchap_secret, &key);
3885                 if (ret)
3886                         return ret;
3887                 kfree(opts->dhchap_ctrl_secret);
3888                 opts->dhchap_ctrl_secret = dhchap_secret;
3889                 ctrl_key = ctrl->ctrl_key;
3890                 mutex_lock(&ctrl->dhchap_auth_mutex);
3891                 ctrl->ctrl_key = key;
3892                 mutex_unlock(&ctrl->dhchap_auth_mutex);
3893                 nvme_auth_free_key(ctrl_key);
3894         }
3895         /* Start re-authentication */
3896         dev_info(ctrl->device, "re-authenticating controller\n");
3897         queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3898
3899         return count;
3900 }
3901 static DEVICE_ATTR(dhchap_ctrl_secret, S_IRUGO | S_IWUSR,
3902         nvme_ctrl_dhchap_ctrl_secret_show, nvme_ctrl_dhchap_ctrl_secret_store);
3903 #endif
3904
3905 static struct attribute *nvme_dev_attrs[] = {
3906         &dev_attr_reset_controller.attr,
3907         &dev_attr_rescan_controller.attr,
3908         &dev_attr_model.attr,
3909         &dev_attr_serial.attr,
3910         &dev_attr_firmware_rev.attr,
3911         &dev_attr_cntlid.attr,
3912         &dev_attr_delete_controller.attr,
3913         &dev_attr_transport.attr,
3914         &dev_attr_subsysnqn.attr,
3915         &dev_attr_address.attr,
3916         &dev_attr_state.attr,
3917         &dev_attr_numa_node.attr,
3918         &dev_attr_queue_count.attr,
3919         &dev_attr_sqsize.attr,
3920         &dev_attr_hostnqn.attr,
3921         &dev_attr_hostid.attr,
3922         &dev_attr_ctrl_loss_tmo.attr,
3923         &dev_attr_reconnect_delay.attr,
3924         &dev_attr_fast_io_fail_tmo.attr,
3925         &dev_attr_kato.attr,
3926         &dev_attr_cntrltype.attr,
3927         &dev_attr_dctype.attr,
3928 #ifdef CONFIG_NVME_AUTH
3929         &dev_attr_dhchap_secret.attr,
3930         &dev_attr_dhchap_ctrl_secret.attr,
3931 #endif
3932         NULL
3933 };
3934
3935 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3936                 struct attribute *a, int n)
3937 {
3938         struct device *dev = container_of(kobj, struct device, kobj);
3939         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3940
3941         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3942                 return 0;
3943         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3944                 return 0;
3945         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3946                 return 0;
3947         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3948                 return 0;
3949         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3950                 return 0;
3951         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3952                 return 0;
3953         if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts)
3954                 return 0;
3955 #ifdef CONFIG_NVME_AUTH
3956         if (a == &dev_attr_dhchap_secret.attr && !ctrl->opts)
3957                 return 0;
3958         if (a == &dev_attr_dhchap_ctrl_secret.attr && !ctrl->opts)
3959                 return 0;
3960 #endif
3961
3962         return a->mode;
3963 }
3964
3965 const struct attribute_group nvme_dev_attrs_group = {
3966         .attrs          = nvme_dev_attrs,
3967         .is_visible     = nvme_dev_attrs_are_visible,
3968 };
3969 EXPORT_SYMBOL_GPL(nvme_dev_attrs_group);
3970
3971 static const struct attribute_group *nvme_dev_attr_groups[] = {
3972         &nvme_dev_attrs_group,
3973         NULL,
3974 };
3975
3976 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3977                 unsigned nsid)
3978 {
3979         struct nvme_ns_head *h;
3980
3981         lockdep_assert_held(&ctrl->subsys->lock);
3982
3983         list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3984                 /*
3985                  * Private namespaces can share NSIDs under some conditions.
3986                  * In that case we can't use the same ns_head for namespaces
3987                  * with the same NSID.
3988                  */
3989                 if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3990                         continue;
3991                 if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3992                         return h;
3993         }
3994
3995         return NULL;
3996 }
3997
3998 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3999                 struct nvme_ns_ids *ids)
4000 {
4001         bool has_uuid = !uuid_is_null(&ids->uuid);
4002         bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
4003         bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
4004         struct nvme_ns_head *h;
4005
4006         lockdep_assert_held(&subsys->lock);
4007
4008         list_for_each_entry(h, &subsys->nsheads, entry) {
4009                 if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
4010                         return -EINVAL;
4011                 if (has_nguid &&
4012                     memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
4013                         return -EINVAL;
4014                 if (has_eui64 &&
4015                     memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
4016                         return -EINVAL;
4017         }
4018
4019         return 0;
4020 }
4021
4022 static void nvme_cdev_rel(struct device *dev)
4023 {
4024         ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
4025 }
4026
4027 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
4028 {
4029         cdev_device_del(cdev, cdev_device);
4030         put_device(cdev_device);
4031 }
4032
4033 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
4034                 const struct file_operations *fops, struct module *owner)
4035 {
4036         int minor, ret;
4037
4038         minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
4039         if (minor < 0)
4040                 return minor;
4041         cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
4042         cdev_device->class = nvme_ns_chr_class;
4043         cdev_device->release = nvme_cdev_rel;
4044         device_initialize(cdev_device);
4045         cdev_init(cdev, fops);
4046         cdev->owner = owner;
4047         ret = cdev_device_add(cdev, cdev_device);
4048         if (ret)
4049                 put_device(cdev_device);
4050
4051         return ret;
4052 }
4053
4054 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
4055 {
4056         return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
4057 }
4058
4059 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
4060 {
4061         nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
4062         return 0;
4063 }
4064
4065 static const struct file_operations nvme_ns_chr_fops = {
4066         .owner          = THIS_MODULE,
4067         .open           = nvme_ns_chr_open,
4068         .release        = nvme_ns_chr_release,
4069         .unlocked_ioctl = nvme_ns_chr_ioctl,
4070         .compat_ioctl   = compat_ptr_ioctl,
4071         .uring_cmd      = nvme_ns_chr_uring_cmd,
4072         .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
4073 };
4074
4075 static int nvme_add_ns_cdev(struct nvme_ns *ns)
4076 {
4077         int ret;
4078
4079         ns->cdev_device.parent = ns->ctrl->device;
4080         ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
4081                            ns->ctrl->instance, ns->head->instance);
4082         if (ret)
4083                 return ret;
4084
4085         return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
4086                              ns->ctrl->ops->module);
4087 }
4088
4089 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
4090                 struct nvme_ns_info *info)
4091 {
4092         struct nvme_ns_head *head;
4093         size_t size = sizeof(*head);
4094         int ret = -ENOMEM;
4095
4096 #ifdef CONFIG_NVME_MULTIPATH
4097         size += num_possible_nodes() * sizeof(struct nvme_ns *);
4098 #endif
4099
4100         head = kzalloc(size, GFP_KERNEL);
4101         if (!head)
4102                 goto out;
4103         ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
4104         if (ret < 0)
4105                 goto out_free_head;
4106         head->instance = ret;
4107         INIT_LIST_HEAD(&head->list);
4108         ret = init_srcu_struct(&head->srcu);
4109         if (ret)
4110                 goto out_ida_remove;
4111         head->subsys = ctrl->subsys;
4112         head->ns_id = info->nsid;
4113         head->ids = info->ids;
4114         head->shared = info->is_shared;
4115         kref_init(&head->ref);
4116
4117         if (head->ids.csi) {
4118                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
4119                 if (ret)
4120                         goto out_cleanup_srcu;
4121         } else
4122                 head->effects = ctrl->effects;
4123
4124         ret = nvme_mpath_alloc_disk(ctrl, head);
4125         if (ret)
4126                 goto out_cleanup_srcu;
4127
4128         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
4129
4130         kref_get(&ctrl->subsys->ref);
4131
4132         return head;
4133 out_cleanup_srcu:
4134         cleanup_srcu_struct(&head->srcu);
4135 out_ida_remove:
4136         ida_free(&ctrl->subsys->ns_ida, head->instance);
4137 out_free_head:
4138         kfree(head);
4139 out:
4140         if (ret > 0)
4141                 ret = blk_status_to_errno(nvme_error_status(ret));
4142         return ERR_PTR(ret);
4143 }
4144
4145 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
4146                 struct nvme_ns_ids *ids)
4147 {
4148         struct nvme_subsystem *s;
4149         int ret = 0;
4150
4151         /*
4152          * Note that this check is racy as we try to avoid holding the global
4153          * lock over the whole ns_head creation.  But it is only intended as
4154          * a sanity check anyway.
4155          */
4156         mutex_lock(&nvme_subsystems_lock);
4157         list_for_each_entry(s, &nvme_subsystems, entry) {
4158                 if (s == this)
4159                         continue;
4160                 mutex_lock(&s->lock);
4161                 ret = nvme_subsys_check_duplicate_ids(s, ids);
4162                 mutex_unlock(&s->lock);
4163                 if (ret)
4164                         break;
4165         }
4166         mutex_unlock(&nvme_subsystems_lock);
4167
4168         return ret;
4169 }
4170
4171 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
4172 {
4173         struct nvme_ctrl *ctrl = ns->ctrl;
4174         struct nvme_ns_head *head = NULL;
4175         int ret;
4176
4177         ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
4178         if (ret) {
4179                 dev_err(ctrl->device,
4180                         "globally duplicate IDs for nsid %d\n", info->nsid);
4181                 nvme_print_device_info(ctrl);
4182                 return ret;
4183         }
4184
4185         mutex_lock(&ctrl->subsys->lock);
4186         head = nvme_find_ns_head(ctrl, info->nsid);
4187         if (!head) {
4188                 ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids);
4189                 if (ret) {
4190                         dev_err(ctrl->device,
4191                                 "duplicate IDs in subsystem for nsid %d\n",
4192                                 info->nsid);
4193                         goto out_unlock;
4194                 }
4195                 head = nvme_alloc_ns_head(ctrl, info);
4196                 if (IS_ERR(head)) {
4197                         ret = PTR_ERR(head);
4198                         goto out_unlock;
4199                 }
4200         } else {
4201                 ret = -EINVAL;
4202                 if (!info->is_shared || !head->shared) {
4203                         dev_err(ctrl->device,
4204                                 "Duplicate unshared namespace %d\n",
4205                                 info->nsid);
4206                         goto out_put_ns_head;
4207                 }
4208                 if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
4209                         dev_err(ctrl->device,
4210                                 "IDs don't match for shared namespace %d\n",
4211                                         info->nsid);
4212                         goto out_put_ns_head;
4213                 }
4214
4215                 if (!multipath && !list_empty(&head->list)) {
4216                         dev_warn(ctrl->device,
4217                                 "Found shared namespace %d, but multipathing not supported.\n",
4218                                 info->nsid);
4219                         dev_warn_once(ctrl->device,
4220                                 "Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0\n.");
4221                 }
4222         }
4223
4224         list_add_tail_rcu(&ns->siblings, &head->list);
4225         ns->head = head;
4226         mutex_unlock(&ctrl->subsys->lock);
4227         return 0;
4228
4229 out_put_ns_head:
4230         nvme_put_ns_head(head);
4231 out_unlock:
4232         mutex_unlock(&ctrl->subsys->lock);
4233         return ret;
4234 }
4235
4236 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4237 {
4238         struct nvme_ns *ns, *ret = NULL;
4239
4240         down_read(&ctrl->namespaces_rwsem);
4241         list_for_each_entry(ns, &ctrl->namespaces, list) {
4242                 if (ns->head->ns_id == nsid) {
4243                         if (!nvme_get_ns(ns))
4244                                 continue;
4245                         ret = ns;
4246                         break;
4247                 }
4248                 if (ns->head->ns_id > nsid)
4249                         break;
4250         }
4251         up_read(&ctrl->namespaces_rwsem);
4252         return ret;
4253 }
4254 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
4255
4256 /*
4257  * Add the namespace to the controller list while keeping the list ordered.
4258  */
4259 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
4260 {
4261         struct nvme_ns *tmp;
4262
4263         list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
4264                 if (tmp->head->ns_id < ns->head->ns_id) {
4265                         list_add(&ns->list, &tmp->list);
4266                         return;
4267                 }
4268         }
4269         list_add(&ns->list, &ns->ctrl->namespaces);
4270 }
4271
4272 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
4273 {
4274         struct nvme_ns *ns;
4275         struct gendisk *disk;
4276         int node = ctrl->numa_node;
4277
4278         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
4279         if (!ns)
4280                 return;
4281
4282         disk = blk_mq_alloc_disk(ctrl->tagset, ns);
4283         if (IS_ERR(disk))
4284                 goto out_free_ns;
4285         disk->fops = &nvme_bdev_ops;
4286         disk->private_data = ns;
4287
4288         ns->disk = disk;
4289         ns->queue = disk->queue;
4290
4291         if (ctrl->opts && ctrl->opts->data_digest)
4292                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
4293
4294         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
4295         if (ctrl->ops->supports_pci_p2pdma &&
4296             ctrl->ops->supports_pci_p2pdma(ctrl))
4297                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
4298
4299         ns->ctrl = ctrl;
4300         kref_init(&ns->kref);
4301
4302         if (nvme_init_ns_head(ns, info))
4303                 goto out_cleanup_disk;
4304
4305         /*
4306          * If multipathing is enabled, the device name for all disks and not
4307          * just those that represent shared namespaces needs to be based on the
4308          * subsystem instance.  Using the controller instance for private
4309          * namespaces could lead to naming collisions between shared and private
4310          * namespaces if they don't use a common numbering scheme.
4311          *
4312          * If multipathing is not enabled, disk names must use the controller
4313          * instance as shared namespaces will show up as multiple block
4314          * devices.
4315          */
4316         if (ns->head->disk) {
4317                 sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
4318                         ctrl->instance, ns->head->instance);
4319                 disk->flags |= GENHD_FL_HIDDEN;
4320         } else if (multipath) {
4321                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
4322                         ns->head->instance);
4323         } else {
4324                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
4325                         ns->head->instance);
4326         }
4327
4328         if (nvme_update_ns_info(ns, info))
4329                 goto out_unlink_ns;
4330
4331         down_write(&ctrl->namespaces_rwsem);
4332         nvme_ns_add_to_ctrl_list(ns);
4333         up_write(&ctrl->namespaces_rwsem);
4334         nvme_get_ctrl(ctrl);
4335
4336         if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups))
4337                 goto out_cleanup_ns_from_list;
4338
4339         if (!nvme_ns_head_multipath(ns->head))
4340                 nvme_add_ns_cdev(ns);
4341
4342         nvme_mpath_add_disk(ns, info->anagrpid);
4343         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
4344
4345         return;
4346
4347  out_cleanup_ns_from_list:
4348         nvme_put_ctrl(ctrl);
4349         down_write(&ctrl->namespaces_rwsem);
4350         list_del_init(&ns->list);
4351         up_write(&ctrl->namespaces_rwsem);
4352  out_unlink_ns:
4353         mutex_lock(&ctrl->subsys->lock);
4354         list_del_rcu(&ns->siblings);
4355         if (list_empty(&ns->head->list))
4356                 list_del_init(&ns->head->entry);
4357         mutex_unlock(&ctrl->subsys->lock);
4358         nvme_put_ns_head(ns->head);
4359  out_cleanup_disk:
4360         put_disk(disk);
4361  out_free_ns:
4362         kfree(ns);
4363 }
4364
4365 static void nvme_ns_remove(struct nvme_ns *ns)
4366 {
4367         bool last_path = false;
4368
4369         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4370                 return;
4371
4372         clear_bit(NVME_NS_READY, &ns->flags);
4373         set_capacity(ns->disk, 0);
4374         nvme_fault_inject_fini(&ns->fault_inject);
4375
4376         /*
4377          * Ensure that !NVME_NS_READY is seen by other threads to prevent
4378          * this ns going back into current_path.
4379          */
4380         synchronize_srcu(&ns->head->srcu);
4381
4382         /* wait for concurrent submissions */
4383         if (nvme_mpath_clear_current_path(ns))
4384                 synchronize_srcu(&ns->head->srcu);
4385
4386         mutex_lock(&ns->ctrl->subsys->lock);
4387         list_del_rcu(&ns->siblings);
4388         if (list_empty(&ns->head->list)) {
4389                 list_del_init(&ns->head->entry);
4390                 last_path = true;
4391         }
4392         mutex_unlock(&ns->ctrl->subsys->lock);
4393
4394         /* guarantee not available in head->list */
4395         synchronize_srcu(&ns->head->srcu);
4396
4397         if (!nvme_ns_head_multipath(ns->head))
4398                 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
4399         del_gendisk(ns->disk);
4400
4401         down_write(&ns->ctrl->namespaces_rwsem);
4402         list_del_init(&ns->list);
4403         up_write(&ns->ctrl->namespaces_rwsem);
4404
4405         if (last_path)
4406                 nvme_mpath_shutdown_disk(ns->head);
4407         nvme_put_ns(ns);
4408 }
4409
4410 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4411 {
4412         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4413
4414         if (ns) {
4415                 nvme_ns_remove(ns);
4416                 nvme_put_ns(ns);
4417         }
4418 }
4419
4420 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info)
4421 {
4422         int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4423
4424         if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) {
4425                 dev_err(ns->ctrl->device,
4426                         "identifiers changed for nsid %d\n", ns->head->ns_id);
4427                 goto out;
4428         }
4429
4430         ret = nvme_update_ns_info(ns, info);
4431 out:
4432         /*
4433          * Only remove the namespace if we got a fatal error back from the
4434          * device, otherwise ignore the error and just move on.
4435          *
4436          * TODO: we should probably schedule a delayed retry here.
4437          */
4438         if (ret > 0 && (ret & NVME_SC_DNR))
4439                 nvme_ns_remove(ns);
4440 }
4441
4442 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4443 {
4444         struct nvme_ns_info info = { .nsid = nsid };
4445         struct nvme_ns *ns;
4446         int ret;
4447
4448         if (nvme_identify_ns_descs(ctrl, &info))
4449                 return;
4450
4451         if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) {
4452                 dev_warn(ctrl->device,
4453                         "command set not reported for nsid: %d\n", nsid);
4454                 return;
4455         }
4456
4457         /*
4458          * If available try to use the Command Set Idependent Identify Namespace
4459          * data structure to find all the generic information that is needed to
4460          * set up a namespace.  If not fall back to the legacy version.
4461          */
4462         if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) ||
4463             (info.ids.csi != NVME_CSI_NVM && info.ids.csi != NVME_CSI_ZNS))
4464                 ret = nvme_ns_info_from_id_cs_indep(ctrl, &info);
4465         else
4466                 ret = nvme_ns_info_from_identify(ctrl, &info);
4467
4468         if (info.is_removed)
4469                 nvme_ns_remove_by_nsid(ctrl, nsid);
4470
4471         /*
4472          * Ignore the namespace if it is not ready. We will get an AEN once it
4473          * becomes ready and restart the scan.
4474          */
4475         if (ret || !info.is_ready)
4476                 return;
4477
4478         ns = nvme_find_get_ns(ctrl, nsid);
4479         if (ns) {
4480                 nvme_validate_ns(ns, &info);
4481                 nvme_put_ns(ns);
4482         } else {
4483                 nvme_alloc_ns(ctrl, &info);
4484         }
4485 }
4486
4487 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4488                                         unsigned nsid)
4489 {
4490         struct nvme_ns *ns, *next;
4491         LIST_HEAD(rm_list);
4492
4493         down_write(&ctrl->namespaces_rwsem);
4494         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4495                 if (ns->head->ns_id > nsid)
4496                         list_move_tail(&ns->list, &rm_list);
4497         }
4498         up_write(&ctrl->namespaces_rwsem);
4499
4500         list_for_each_entry_safe(ns, next, &rm_list, list)
4501                 nvme_ns_remove(ns);
4502
4503 }
4504
4505 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4506 {
4507         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4508         __le32 *ns_list;
4509         u32 prev = 0;
4510         int ret = 0, i;
4511
4512         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4513         if (!ns_list)
4514                 return -ENOMEM;
4515
4516         for (;;) {
4517                 struct nvme_command cmd = {
4518                         .identify.opcode        = nvme_admin_identify,
4519                         .identify.cns           = NVME_ID_CNS_NS_ACTIVE_LIST,
4520                         .identify.nsid          = cpu_to_le32(prev),
4521                 };
4522
4523                 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4524                                             NVME_IDENTIFY_DATA_SIZE);
4525                 if (ret) {
4526                         dev_warn(ctrl->device,
4527                                 "Identify NS List failed (status=0x%x)\n", ret);
4528                         goto free;
4529                 }
4530
4531                 for (i = 0; i < nr_entries; i++) {
4532                         u32 nsid = le32_to_cpu(ns_list[i]);
4533
4534                         if (!nsid)      /* end of the list? */
4535                                 goto out;
4536                         nvme_scan_ns(ctrl, nsid);
4537                         while (++prev < nsid)
4538                                 nvme_ns_remove_by_nsid(ctrl, prev);
4539                 }
4540         }
4541  out:
4542         nvme_remove_invalid_namespaces(ctrl, prev);
4543  free:
4544         kfree(ns_list);
4545         return ret;
4546 }
4547
4548 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4549 {
4550         struct nvme_id_ctrl *id;
4551         u32 nn, i;
4552
4553         if (nvme_identify_ctrl(ctrl, &id))
4554                 return;
4555         nn = le32_to_cpu(id->nn);
4556         kfree(id);
4557
4558         for (i = 1; i <= nn; i++)
4559                 nvme_scan_ns(ctrl, i);
4560
4561         nvme_remove_invalid_namespaces(ctrl, nn);
4562 }
4563
4564 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4565 {
4566         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4567         __le32 *log;
4568         int error;
4569
4570         log = kzalloc(log_size, GFP_KERNEL);
4571         if (!log)
4572                 return;
4573
4574         /*
4575          * We need to read the log to clear the AEN, but we don't want to rely
4576          * on it for the changed namespace information as userspace could have
4577          * raced with us in reading the log page, which could cause us to miss
4578          * updates.
4579          */
4580         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4581                         NVME_CSI_NVM, log, log_size, 0);
4582         if (error)
4583                 dev_warn(ctrl->device,
4584                         "reading changed ns log failed: %d\n", error);
4585
4586         kfree(log);
4587 }
4588
4589 static void nvme_scan_work(struct work_struct *work)
4590 {
4591         struct nvme_ctrl *ctrl =
4592                 container_of(work, struct nvme_ctrl, scan_work);
4593         int ret;
4594
4595         /* No tagset on a live ctrl means IO queues could not created */
4596         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4597                 return;
4598
4599         /*
4600          * Identify controller limits can change at controller reset due to
4601          * new firmware download, even though it is not common we cannot ignore
4602          * such scenario. Controller's non-mdts limits are reported in the unit
4603          * of logical blocks that is dependent on the format of attached
4604          * namespace. Hence re-read the limits at the time of ns allocation.
4605          */
4606         ret = nvme_init_non_mdts_limits(ctrl);
4607         if (ret < 0) {
4608                 dev_warn(ctrl->device,
4609                         "reading non-mdts-limits failed: %d\n", ret);
4610                 return;
4611         }
4612
4613         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4614                 dev_info(ctrl->device, "rescanning namespaces.\n");
4615                 nvme_clear_changed_ns_log(ctrl);
4616         }
4617
4618         mutex_lock(&ctrl->scan_lock);
4619         if (nvme_ctrl_limited_cns(ctrl)) {
4620                 nvme_scan_ns_sequential(ctrl);
4621         } else {
4622                 /*
4623                  * Fall back to sequential scan if DNR is set to handle broken
4624                  * devices which should support Identify NS List (as per the VS
4625                  * they report) but don't actually support it.
4626                  */
4627                 ret = nvme_scan_ns_list(ctrl);
4628                 if (ret > 0 && ret & NVME_SC_DNR)
4629                         nvme_scan_ns_sequential(ctrl);
4630         }
4631         mutex_unlock(&ctrl->scan_lock);
4632 }
4633
4634 /*
4635  * This function iterates the namespace list unlocked to allow recovery from
4636  * controller failure. It is up to the caller to ensure the namespace list is
4637  * not modified by scan work while this function is executing.
4638  */
4639 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4640 {
4641         struct nvme_ns *ns, *next;
4642         LIST_HEAD(ns_list);
4643
4644         /*
4645          * make sure to requeue I/O to all namespaces as these
4646          * might result from the scan itself and must complete
4647          * for the scan_work to make progress
4648          */
4649         nvme_mpath_clear_ctrl_paths(ctrl);
4650
4651         /* prevent racing with ns scanning */
4652         flush_work(&ctrl->scan_work);
4653
4654         /*
4655          * The dead states indicates the controller was not gracefully
4656          * disconnected. In that case, we won't be able to flush any data while
4657          * removing the namespaces' disks; fail all the queues now to avoid
4658          * potentially having to clean up the failed sync later.
4659          */
4660         if (ctrl->state == NVME_CTRL_DEAD) {
4661                 nvme_mark_namespaces_dead(ctrl);
4662                 nvme_unquiesce_io_queues(ctrl);
4663         }
4664
4665         /* this is a no-op when called from the controller reset handler */
4666         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4667
4668         down_write(&ctrl->namespaces_rwsem);
4669         list_splice_init(&ctrl->namespaces, &ns_list);
4670         up_write(&ctrl->namespaces_rwsem);
4671
4672         list_for_each_entry_safe(ns, next, &ns_list, list)
4673                 nvme_ns_remove(ns);
4674 }
4675 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4676
4677 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)
4678 {
4679         const struct nvme_ctrl *ctrl =
4680                 container_of(dev, struct nvme_ctrl, ctrl_device);
4681         struct nvmf_ctrl_options *opts = ctrl->opts;
4682         int ret;
4683
4684         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4685         if (ret)
4686                 return ret;
4687
4688         if (opts) {
4689                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4690                 if (ret)
4691                         return ret;
4692
4693                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4694                                 opts->trsvcid ?: "none");
4695                 if (ret)
4696                         return ret;
4697
4698                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4699                                 opts->host_traddr ?: "none");
4700                 if (ret)
4701                         return ret;
4702
4703                 ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4704                                 opts->host_iface ?: "none");
4705         }
4706         return ret;
4707 }
4708
4709 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4710 {
4711         char *envp[2] = { envdata, NULL };
4712
4713         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4714 }
4715
4716 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4717 {
4718         char *envp[2] = { NULL, NULL };
4719         u32 aen_result = ctrl->aen_result;
4720
4721         ctrl->aen_result = 0;
4722         if (!aen_result)
4723                 return;
4724
4725         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4726         if (!envp[0])
4727                 return;
4728         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4729         kfree(envp[0]);
4730 }
4731
4732 static void nvme_async_event_work(struct work_struct *work)
4733 {
4734         struct nvme_ctrl *ctrl =
4735                 container_of(work, struct nvme_ctrl, async_event_work);
4736
4737         nvme_aen_uevent(ctrl);
4738
4739         /*
4740          * The transport drivers must guarantee AER submission here is safe by
4741          * flushing ctrl async_event_work after changing the controller state
4742          * from LIVE and before freeing the admin queue.
4743         */
4744         if (ctrl->state == NVME_CTRL_LIVE)
4745                 ctrl->ops->submit_async_event(ctrl);
4746 }
4747
4748 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4749 {
4750
4751         u32 csts;
4752
4753         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4754                 return false;
4755
4756         if (csts == ~0)
4757                 return false;
4758
4759         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4760 }
4761
4762 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4763 {
4764         struct nvme_fw_slot_info_log *log;
4765
4766         log = kmalloc(sizeof(*log), GFP_KERNEL);
4767         if (!log)
4768                 return;
4769
4770         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4771                         log, sizeof(*log), 0))
4772                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4773         kfree(log);
4774 }
4775
4776 static void nvme_fw_act_work(struct work_struct *work)
4777 {
4778         struct nvme_ctrl *ctrl = container_of(work,
4779                                 struct nvme_ctrl, fw_act_work);
4780         unsigned long fw_act_timeout;
4781
4782         if (ctrl->mtfa)
4783                 fw_act_timeout = jiffies +
4784                                 msecs_to_jiffies(ctrl->mtfa * 100);
4785         else
4786                 fw_act_timeout = jiffies +
4787                                 msecs_to_jiffies(admin_timeout * 1000);
4788
4789         nvme_quiesce_io_queues(ctrl);
4790         while (nvme_ctrl_pp_status(ctrl)) {
4791                 if (time_after(jiffies, fw_act_timeout)) {
4792                         dev_warn(ctrl->device,
4793                                 "Fw activation timeout, reset controller\n");
4794                         nvme_try_sched_reset(ctrl);
4795                         return;
4796                 }
4797                 msleep(100);
4798         }
4799
4800         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4801                 return;
4802
4803         nvme_unquiesce_io_queues(ctrl);
4804         /* read FW slot information to clear the AER */
4805         nvme_get_fw_slot_info(ctrl);
4806
4807         queue_work(nvme_wq, &ctrl->async_event_work);
4808 }
4809
4810 static u32 nvme_aer_type(u32 result)
4811 {
4812         return result & 0x7;
4813 }
4814
4815 static u32 nvme_aer_subtype(u32 result)
4816 {
4817         return (result & 0xff00) >> 8;
4818 }
4819
4820 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4821 {
4822         u32 aer_notice_type = nvme_aer_subtype(result);
4823         bool requeue = true;
4824
4825         switch (aer_notice_type) {
4826         case NVME_AER_NOTICE_NS_CHANGED:
4827                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4828                 nvme_queue_scan(ctrl);
4829                 break;
4830         case NVME_AER_NOTICE_FW_ACT_STARTING:
4831                 /*
4832                  * We are (ab)using the RESETTING state to prevent subsequent
4833                  * recovery actions from interfering with the controller's
4834                  * firmware activation.
4835                  */
4836                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
4837                         nvme_auth_stop(ctrl);
4838                         requeue = false;
4839                         queue_work(nvme_wq, &ctrl->fw_act_work);
4840                 }
4841                 break;
4842 #ifdef CONFIG_NVME_MULTIPATH
4843         case NVME_AER_NOTICE_ANA:
4844                 if (!ctrl->ana_log_buf)
4845                         break;
4846                 queue_work(nvme_wq, &ctrl->ana_work);
4847                 break;
4848 #endif
4849         case NVME_AER_NOTICE_DISC_CHANGED:
4850                 ctrl->aen_result = result;
4851                 break;
4852         default:
4853                 dev_warn(ctrl->device, "async event result %08x\n", result);
4854         }
4855         return requeue;
4856 }
4857
4858 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4859 {
4860         dev_warn(ctrl->device, "resetting controller due to AER\n");
4861         nvme_reset_ctrl(ctrl);
4862 }
4863
4864 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4865                 volatile union nvme_result *res)
4866 {
4867         u32 result = le32_to_cpu(res->u32);
4868         u32 aer_type = nvme_aer_type(result);
4869         u32 aer_subtype = nvme_aer_subtype(result);
4870         bool requeue = true;
4871
4872         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4873                 return;
4874
4875         trace_nvme_async_event(ctrl, result);
4876         switch (aer_type) {
4877         case NVME_AER_NOTICE:
4878                 requeue = nvme_handle_aen_notice(ctrl, result);
4879                 break;
4880         case NVME_AER_ERROR:
4881                 /*
4882                  * For a persistent internal error, don't run async_event_work
4883                  * to submit a new AER. The controller reset will do it.
4884                  */
4885                 if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4886                         nvme_handle_aer_persistent_error(ctrl);
4887                         return;
4888                 }
4889                 fallthrough;
4890         case NVME_AER_SMART:
4891         case NVME_AER_CSS:
4892         case NVME_AER_VS:
4893                 ctrl->aen_result = result;
4894                 break;
4895         default:
4896                 break;
4897         }
4898
4899         if (requeue)
4900                 queue_work(nvme_wq, &ctrl->async_event_work);
4901 }
4902 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4903
4904 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4905                 const struct blk_mq_ops *ops, unsigned int cmd_size)
4906 {
4907         int ret;
4908
4909         memset(set, 0, sizeof(*set));
4910         set->ops = ops;
4911         set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
4912         if (ctrl->ops->flags & NVME_F_FABRICS)
4913                 set->reserved_tags = NVMF_RESERVED_TAGS;
4914         set->numa_node = ctrl->numa_node;
4915         set->flags = BLK_MQ_F_NO_SCHED;
4916         if (ctrl->ops->flags & NVME_F_BLOCKING)
4917                 set->flags |= BLK_MQ_F_BLOCKING;
4918         set->cmd_size = cmd_size;
4919         set->driver_data = ctrl;
4920         set->nr_hw_queues = 1;
4921         set->timeout = NVME_ADMIN_TIMEOUT;
4922         ret = blk_mq_alloc_tag_set(set);
4923         if (ret)
4924                 return ret;
4925
4926         ctrl->admin_q = blk_mq_init_queue(set);
4927         if (IS_ERR(ctrl->admin_q)) {
4928                 ret = PTR_ERR(ctrl->admin_q);
4929                 goto out_free_tagset;
4930         }
4931
4932         if (ctrl->ops->flags & NVME_F_FABRICS) {
4933                 ctrl->fabrics_q = blk_mq_init_queue(set);
4934                 if (IS_ERR(ctrl->fabrics_q)) {
4935                         ret = PTR_ERR(ctrl->fabrics_q);
4936                         goto out_cleanup_admin_q;
4937                 }
4938         }
4939
4940         ctrl->admin_tagset = set;
4941         return 0;
4942
4943 out_cleanup_admin_q:
4944         blk_mq_destroy_queue(ctrl->admin_q);
4945         blk_put_queue(ctrl->admin_q);
4946 out_free_tagset:
4947         blk_mq_free_tag_set(set);
4948         ctrl->admin_q = NULL;
4949         ctrl->fabrics_q = NULL;
4950         return ret;
4951 }
4952 EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set);
4953
4954 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
4955 {
4956         blk_mq_destroy_queue(ctrl->admin_q);
4957         blk_put_queue(ctrl->admin_q);
4958         if (ctrl->ops->flags & NVME_F_FABRICS) {
4959                 blk_mq_destroy_queue(ctrl->fabrics_q);
4960                 blk_put_queue(ctrl->fabrics_q);
4961         }
4962         blk_mq_free_tag_set(ctrl->admin_tagset);
4963 }
4964 EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
4965
4966 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4967                 const struct blk_mq_ops *ops, unsigned int nr_maps,
4968                 unsigned int cmd_size)
4969 {
4970         int ret;
4971
4972         memset(set, 0, sizeof(*set));
4973         set->ops = ops;
4974         set->queue_depth = min_t(unsigned, ctrl->sqsize, BLK_MQ_MAX_DEPTH - 1);
4975         /*
4976          * Some Apple controllers requires tags to be unique across admin and
4977          * the (only) I/O queue, so reserve the first 32 tags of the I/O queue.
4978          */
4979         if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
4980                 set->reserved_tags = NVME_AQ_DEPTH;
4981         else if (ctrl->ops->flags & NVME_F_FABRICS)
4982                 set->reserved_tags = NVMF_RESERVED_TAGS;
4983         set->numa_node = ctrl->numa_node;
4984         set->flags = BLK_MQ_F_SHOULD_MERGE;
4985         if (ctrl->ops->flags & NVME_F_BLOCKING)
4986                 set->flags |= BLK_MQ_F_BLOCKING;
4987         set->cmd_size = cmd_size,
4988         set->driver_data = ctrl;
4989         set->nr_hw_queues = ctrl->queue_count - 1;
4990         set->timeout = NVME_IO_TIMEOUT;
4991         set->nr_maps = nr_maps;
4992         ret = blk_mq_alloc_tag_set(set);
4993         if (ret)
4994                 return ret;
4995
4996         if (ctrl->ops->flags & NVME_F_FABRICS) {
4997                 ctrl->connect_q = blk_mq_init_queue(set);
4998                 if (IS_ERR(ctrl->connect_q)) {
4999                         ret = PTR_ERR(ctrl->connect_q);
5000                         goto out_free_tag_set;
5001                 }
5002                 blk_queue_flag_set(QUEUE_FLAG_SKIP_TAGSET_QUIESCE,
5003                                    ctrl->connect_q);
5004         }
5005
5006         ctrl->tagset = set;
5007         return 0;
5008
5009 out_free_tag_set:
5010         blk_mq_free_tag_set(set);
5011         ctrl->connect_q = NULL;
5012         return ret;
5013 }
5014 EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set);
5015
5016 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)
5017 {
5018         if (ctrl->ops->flags & NVME_F_FABRICS) {
5019                 blk_mq_destroy_queue(ctrl->connect_q);
5020                 blk_put_queue(ctrl->connect_q);
5021         }
5022         blk_mq_free_tag_set(ctrl->tagset);
5023 }
5024 EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
5025
5026 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
5027 {
5028         nvme_mpath_stop(ctrl);
5029         nvme_auth_stop(ctrl);
5030         nvme_stop_keep_alive(ctrl);
5031         nvme_stop_failfast_work(ctrl);
5032         flush_work(&ctrl->async_event_work);
5033         cancel_work_sync(&ctrl->fw_act_work);
5034         if (ctrl->ops->stop_ctrl)
5035                 ctrl->ops->stop_ctrl(ctrl);
5036 }
5037 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
5038
5039 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
5040 {
5041         nvme_start_keep_alive(ctrl);
5042
5043         nvme_enable_aen(ctrl);
5044
5045         /*
5046          * persistent discovery controllers need to send indication to userspace
5047          * to re-read the discovery log page to learn about possible changes
5048          * that were missed. We identify persistent discovery controllers by
5049          * checking that they started once before, hence are reconnecting back.
5050          */
5051         if (test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
5052             nvme_discovery_ctrl(ctrl))
5053                 nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
5054
5055         if (ctrl->queue_count > 1) {
5056                 nvme_queue_scan(ctrl);
5057                 nvme_unquiesce_io_queues(ctrl);
5058                 nvme_mpath_update(ctrl);
5059         }
5060
5061         nvme_change_uevent(ctrl, "NVME_EVENT=connected");
5062         set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags);
5063 }
5064 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
5065
5066 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
5067 {
5068         nvme_hwmon_exit(ctrl);
5069         nvme_fault_inject_fini(&ctrl->fault_inject);
5070         dev_pm_qos_hide_latency_tolerance(ctrl->device);
5071         cdev_device_del(&ctrl->cdev, ctrl->device);
5072         nvme_put_ctrl(ctrl);
5073 }
5074 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
5075
5076 static void nvme_free_cels(struct nvme_ctrl *ctrl)
5077 {
5078         struct nvme_effects_log *cel;
5079         unsigned long i;
5080
5081         xa_for_each(&ctrl->cels, i, cel) {
5082                 xa_erase(&ctrl->cels, i);
5083                 kfree(cel);
5084         }
5085
5086         xa_destroy(&ctrl->cels);
5087 }
5088
5089 static void nvme_free_ctrl(struct device *dev)
5090 {
5091         struct nvme_ctrl *ctrl =
5092                 container_of(dev, struct nvme_ctrl, ctrl_device);
5093         struct nvme_subsystem *subsys = ctrl->subsys;
5094
5095         if (!subsys || ctrl->instance != subsys->instance)
5096                 ida_free(&nvme_instance_ida, ctrl->instance);
5097
5098         nvme_free_cels(ctrl);
5099         nvme_mpath_uninit(ctrl);
5100         nvme_auth_stop(ctrl);
5101         nvme_auth_free(ctrl);
5102         __free_page(ctrl->discard_page);
5103         free_opal_dev(ctrl->opal_dev);
5104
5105         if (subsys) {
5106                 mutex_lock(&nvme_subsystems_lock);
5107                 list_del(&ctrl->subsys_entry);
5108                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
5109                 mutex_unlock(&nvme_subsystems_lock);
5110         }
5111
5112         ctrl->ops->free_ctrl(ctrl);
5113
5114         if (subsys)
5115                 nvme_put_subsystem(subsys);
5116 }
5117
5118 /*
5119  * Initialize a NVMe controller structures.  This needs to be called during
5120  * earliest initialization so that we have the initialized structured around
5121  * during probing.
5122  */
5123 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
5124                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
5125 {
5126         int ret;
5127
5128         ctrl->state = NVME_CTRL_NEW;
5129         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
5130         spin_lock_init(&ctrl->lock);
5131         mutex_init(&ctrl->scan_lock);
5132         INIT_LIST_HEAD(&ctrl->namespaces);
5133         xa_init(&ctrl->cels);
5134         init_rwsem(&ctrl->namespaces_rwsem);
5135         ctrl->dev = dev;
5136         ctrl->ops = ops;
5137         ctrl->quirks = quirks;
5138         ctrl->numa_node = NUMA_NO_NODE;
5139         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
5140         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
5141         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
5142         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
5143         init_waitqueue_head(&ctrl->state_wq);
5144
5145         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
5146         INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
5147         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
5148         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
5149
5150         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
5151                         PAGE_SIZE);
5152         ctrl->discard_page = alloc_page(GFP_KERNEL);
5153         if (!ctrl->discard_page) {
5154                 ret = -ENOMEM;
5155                 goto out;
5156         }
5157
5158         ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
5159         if (ret < 0)
5160                 goto out;
5161         ctrl->instance = ret;
5162
5163         device_initialize(&ctrl->ctrl_device);
5164         ctrl->device = &ctrl->ctrl_device;
5165         ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
5166                         ctrl->instance);
5167         ctrl->device->class = nvme_class;
5168         ctrl->device->parent = ctrl->dev;
5169         if (ops->dev_attr_groups)
5170                 ctrl->device->groups = ops->dev_attr_groups;
5171         else
5172                 ctrl->device->groups = nvme_dev_attr_groups;
5173         ctrl->device->release = nvme_free_ctrl;
5174         dev_set_drvdata(ctrl->device, ctrl);
5175         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
5176         if (ret)
5177                 goto out_release_instance;
5178
5179         nvme_get_ctrl(ctrl);
5180         cdev_init(&ctrl->cdev, &nvme_dev_fops);
5181         ctrl->cdev.owner = ops->module;
5182         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
5183         if (ret)
5184                 goto out_free_name;
5185
5186         /*
5187          * Initialize latency tolerance controls.  The sysfs files won't
5188          * be visible to userspace unless the device actually supports APST.
5189          */
5190         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
5191         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
5192                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
5193
5194         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
5195         nvme_mpath_init_ctrl(ctrl);
5196         ret = nvme_auth_init_ctrl(ctrl);
5197         if (ret)
5198                 goto out_free_cdev;
5199
5200         return 0;
5201 out_free_cdev:
5202         cdev_device_del(&ctrl->cdev, ctrl->device);
5203 out_free_name:
5204         nvme_put_ctrl(ctrl);
5205         kfree_const(ctrl->device->kobj.name);
5206 out_release_instance:
5207         ida_free(&nvme_instance_ida, ctrl->instance);
5208 out:
5209         if (ctrl->discard_page)
5210                 __free_page(ctrl->discard_page);
5211         return ret;
5212 }
5213 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
5214
5215 /* let I/O to all namespaces fail in preparation for surprise removal */
5216 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl)
5217 {
5218         struct nvme_ns *ns;
5219
5220         down_read(&ctrl->namespaces_rwsem);
5221         list_for_each_entry(ns, &ctrl->namespaces, list)
5222                 blk_mark_disk_dead(ns->disk);
5223         up_read(&ctrl->namespaces_rwsem);
5224 }
5225 EXPORT_SYMBOL_GPL(nvme_mark_namespaces_dead);
5226
5227 void nvme_unfreeze(struct nvme_ctrl *ctrl)
5228 {
5229         struct nvme_ns *ns;
5230
5231         down_read(&ctrl->namespaces_rwsem);
5232         list_for_each_entry(ns, &ctrl->namespaces, list)
5233                 blk_mq_unfreeze_queue(ns->queue);
5234         up_read(&ctrl->namespaces_rwsem);
5235 }
5236 EXPORT_SYMBOL_GPL(nvme_unfreeze);
5237
5238 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
5239 {
5240         struct nvme_ns *ns;
5241
5242         down_read(&ctrl->namespaces_rwsem);
5243         list_for_each_entry(ns, &ctrl->namespaces, list) {
5244                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
5245                 if (timeout <= 0)
5246                         break;
5247         }
5248         up_read(&ctrl->namespaces_rwsem);
5249         return timeout;
5250 }
5251 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
5252
5253 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
5254 {
5255         struct nvme_ns *ns;
5256
5257         down_read(&ctrl->namespaces_rwsem);
5258         list_for_each_entry(ns, &ctrl->namespaces, list)
5259                 blk_mq_freeze_queue_wait(ns->queue);
5260         up_read(&ctrl->namespaces_rwsem);
5261 }
5262 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
5263
5264 void nvme_start_freeze(struct nvme_ctrl *ctrl)
5265 {
5266         struct nvme_ns *ns;
5267
5268         down_read(&ctrl->namespaces_rwsem);
5269         list_for_each_entry(ns, &ctrl->namespaces, list)
5270                 blk_freeze_queue_start(ns->queue);
5271         up_read(&ctrl->namespaces_rwsem);
5272 }
5273 EXPORT_SYMBOL_GPL(nvme_start_freeze);
5274
5275 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)
5276 {
5277         if (!ctrl->tagset)
5278                 return;
5279         if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))
5280                 blk_mq_quiesce_tagset(ctrl->tagset);
5281         else
5282                 blk_mq_wait_quiesce_done(ctrl->tagset);
5283 }
5284 EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);
5285
5286 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)
5287 {
5288         if (!ctrl->tagset)
5289                 return;
5290         if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))
5291                 blk_mq_unquiesce_tagset(ctrl->tagset);
5292 }
5293 EXPORT_SYMBOL_GPL(nvme_unquiesce_io_queues);
5294
5295 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl)
5296 {
5297         if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5298                 blk_mq_quiesce_queue(ctrl->admin_q);
5299         else
5300                 blk_mq_wait_quiesce_done(ctrl->admin_q->tag_set);
5301 }
5302 EXPORT_SYMBOL_GPL(nvme_quiesce_admin_queue);
5303
5304 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl)
5305 {
5306         if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5307                 blk_mq_unquiesce_queue(ctrl->admin_q);
5308 }
5309 EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);
5310
5311 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
5312 {
5313         struct nvme_ns *ns;
5314
5315         down_read(&ctrl->namespaces_rwsem);
5316         list_for_each_entry(ns, &ctrl->namespaces, list)
5317                 blk_sync_queue(ns->queue);
5318         up_read(&ctrl->namespaces_rwsem);
5319 }
5320 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
5321
5322 void nvme_sync_queues(struct nvme_ctrl *ctrl)
5323 {
5324         nvme_sync_io_queues(ctrl);
5325         if (ctrl->admin_q)
5326                 blk_sync_queue(ctrl->admin_q);
5327 }
5328 EXPORT_SYMBOL_GPL(nvme_sync_queues);
5329
5330 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
5331 {
5332         if (file->f_op != &nvme_dev_fops)
5333                 return NULL;
5334         return file->private_data;
5335 }
5336 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
5337
5338 /*
5339  * Check we didn't inadvertently grow the command structure sizes:
5340  */
5341 static inline void _nvme_check_size(void)
5342 {
5343         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
5344         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
5345         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
5346         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
5347         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
5348         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
5349         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
5350         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
5351         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
5352         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
5353         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
5354         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
5355         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
5356         BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
5357                         NVME_IDENTIFY_DATA_SIZE);
5358         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
5359         BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
5360         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
5361         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
5362         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
5363         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
5364         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
5365         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
5366         BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
5367 }
5368
5369
5370 static int __init nvme_core_init(void)
5371 {
5372         int result = -ENOMEM;
5373
5374         _nvme_check_size();
5375
5376         nvme_wq = alloc_workqueue("nvme-wq",
5377                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5378         if (!nvme_wq)
5379                 goto out;
5380
5381         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
5382                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5383         if (!nvme_reset_wq)
5384                 goto destroy_wq;
5385
5386         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
5387                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5388         if (!nvme_delete_wq)
5389                 goto destroy_reset_wq;
5390
5391         result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
5392                         NVME_MINORS, "nvme");
5393         if (result < 0)
5394                 goto destroy_delete_wq;
5395
5396         nvme_class = class_create("nvme");
5397         if (IS_ERR(nvme_class)) {
5398                 result = PTR_ERR(nvme_class);
5399                 goto unregister_chrdev;
5400         }
5401         nvme_class->dev_uevent = nvme_class_uevent;
5402
5403         nvme_subsys_class = class_create("nvme-subsystem");
5404         if (IS_ERR(nvme_subsys_class)) {
5405                 result = PTR_ERR(nvme_subsys_class);
5406                 goto destroy_class;
5407         }
5408
5409         result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
5410                                      "nvme-generic");
5411         if (result < 0)
5412                 goto destroy_subsys_class;
5413
5414         nvme_ns_chr_class = class_create("nvme-generic");
5415         if (IS_ERR(nvme_ns_chr_class)) {
5416                 result = PTR_ERR(nvme_ns_chr_class);
5417                 goto unregister_generic_ns;
5418         }
5419
5420         result = nvme_init_auth();
5421         if (result)
5422                 goto destroy_ns_chr;
5423         return 0;
5424
5425 destroy_ns_chr:
5426         class_destroy(nvme_ns_chr_class);
5427 unregister_generic_ns:
5428         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5429 destroy_subsys_class:
5430         class_destroy(nvme_subsys_class);
5431 destroy_class:
5432         class_destroy(nvme_class);
5433 unregister_chrdev:
5434         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5435 destroy_delete_wq:
5436         destroy_workqueue(nvme_delete_wq);
5437 destroy_reset_wq:
5438         destroy_workqueue(nvme_reset_wq);
5439 destroy_wq:
5440         destroy_workqueue(nvme_wq);
5441 out:
5442         return result;
5443 }
5444
5445 static void __exit nvme_core_exit(void)
5446 {
5447         nvme_exit_auth();
5448         class_destroy(nvme_ns_chr_class);
5449         class_destroy(nvme_subsys_class);
5450         class_destroy(nvme_class);
5451         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5452         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5453         destroy_workqueue(nvme_delete_wq);
5454         destroy_workqueue(nvme_reset_wq);
5455         destroy_workqueue(nvme_wq);
5456         ida_destroy(&nvme_ns_chr_minor_ida);
5457         ida_destroy(&nvme_instance_ida);
5458 }
5459
5460 MODULE_LICENSE("GPL");
5461 MODULE_VERSION("1.0");
5462 module_init(nvme_core_init);
5463 module_exit(nvme_core_exit);