drivers/nvme/host/multipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2017-2018 Christoph Hellwig.
   4  */
   5
   6 #include <linux/backing-dev.h>
   7 #include <linux/moduleparam.h>
   8 #include <trace/events/block.h>
   9 #include "nvme.h"
  10
  11 static bool multipath = true;
  12 module_param(multipath, bool, 0444);
  13 MODULE_PARM_DESC(multipath,
  14         "turn on native support for multiple controllers per subsystem");
  15
  16 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
  17 {
  18         struct nvme_ns_head *h;
  19
  20         lockdep_assert_held(&subsys->lock);
  21         list_for_each_entry(h, &subsys->nsheads, entry)
  22                 if (h->disk)
  23                         blk_mq_unfreeze_queue(h->disk->queue);
  24 }
  25
  26 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
  27 {
  28         struct nvme_ns_head *h;
  29
  30         lockdep_assert_held(&subsys->lock);
  31         list_for_each_entry(h, &subsys->nsheads, entry)
  32                 if (h->disk)
  33                         blk_mq_freeze_queue_wait(h->disk->queue);
  34 }
  35
  36 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
  37 {
  38         struct nvme_ns_head *h;
  39
  40         lockdep_assert_held(&subsys->lock);
  41         list_for_each_entry(h, &subsys->nsheads, entry)
  42                 if (h->disk)
  43                         blk_freeze_queue_start(h->disk->queue);
  44 }
  45
  46 /*
  47  * If multipathing is enabled we need to always use the subsystem instance
  48  * number for numbering our devices to avoid conflicts between subsystems that
  49  * have multiple controllers and thus use the multipath-aware subsystem node
  50  * and those that have a single controller and use the controller node
  51  * directly.
  52  */
  53 bool nvme_mpath_set_disk_name(struct nvme_ns *ns, char *disk_name, int *flags)
  54 {
  55         if (!multipath)
  56                 return false;
  57         if (!ns->head->disk) {
  58                 sprintf(disk_name, "nvme%dn%d", ns->ctrl->subsys->instance,
  59                         ns->head->instance);
  60                 return true;
  61         }
  62         sprintf(disk_name, "nvme%dc%dn%d", ns->ctrl->subsys->instance,
  63                 ns->ctrl->instance, ns->head->instance);
  64         *flags = GENHD_FL_HIDDEN;
  65         return true;
  66 }
  67
  68 void nvme_failover_req(struct request *req)
  69 {
  70         struct nvme_ns *ns = req->q->queuedata;
  71         u16 status = nvme_req(req)->status & 0x7ff;
  72         unsigned long flags;
  73         struct bio *bio;
  74
  75         nvme_mpath_clear_current_path(ns);
  76
  77         /*
  78          * If we got back an ANA error, we know the controller is alive but not
  79          * ready to serve this namespace.  Kick of a re-read of the ANA
  80          * information page, and just try any other available path for now.
  81          */
  82         if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {
  83                 set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  84                 queue_work(nvme_wq, &ns->ctrl->ana_work);
  85         }
  86
  87         spin_lock_irqsave(&ns->head->requeue_lock, flags);
  88         for (bio = req->bio; bio; bio = bio->bi_next)
  89                 bio_set_dev(bio, ns->head->disk->part0);
  90         blk_steal_bios(&ns->head->requeue_list, req);
  91         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
  92
  93         blk_mq_end_request(req, 0);
  94         kblockd_schedule_work(&ns->head->requeue_work);
  95 }
  96
  97 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
  98 {
  99         struct nvme_ns *ns;
 100
 101         down_read(&ctrl->namespaces_rwsem);
 102         list_for_each_entry(ns, &ctrl->namespaces, list) {
 103                 if (ns->head->disk)
 104                         kblockd_schedule_work(&ns->head->requeue_work);
 105         }
 106         up_read(&ctrl->namespaces_rwsem);
 107 }
 108
 109 static const char *nvme_ana_state_names[] = {
 110         [0]                             = "invalid state",
 111         [NVME_ANA_OPTIMIZED]            = "optimized",
 112         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 113         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 114         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 115         [NVME_ANA_CHANGE]               = "change",
 116 };
 117
 118 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 119 {
 120         struct nvme_ns_head *head = ns->head;
 121         bool changed = false;
 122         int node;
 123
 124         if (!head)
 125                 goto out;
 126
 127         for_each_node(node) {
 128                 if (ns == rcu_access_pointer(head->current_path[node])) {
 129                         rcu_assign_pointer(head->current_path[node], NULL);
 130                         changed = true;
 131                 }
 132         }
 133 out:
 134         return changed;
 135 }
 136
 137 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 138 {
 139         struct nvme_ns *ns;
 140
 141         down_read(&ctrl->namespaces_rwsem);
 142         list_for_each_entry(ns, &ctrl->namespaces, list) {
 143                 nvme_mpath_clear_current_path(ns);
 144                 kblockd_schedule_work(&ns->head->requeue_work);
 145         }
 146         up_read(&ctrl->namespaces_rwsem);
 147 }
 148
 149 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
 150 {
 151         struct nvme_ns_head *head = ns->head;
 152         sector_t capacity = get_capacity(head->disk);
 153         int node;
 154
 155         list_for_each_entry_rcu(ns, &head->list, siblings) {
 156                 if (capacity != get_capacity(ns->disk))
 157                         clear_bit(NVME_NS_READY, &ns->flags);
 158         }
 159
 160         for_each_node(node)
 161                 rcu_assign_pointer(head->current_path[node], NULL);
 162 }
 163
 164 static bool nvme_path_is_disabled(struct nvme_ns *ns)
 165 {
 166         /*
 167          * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should
 168          * still be able to complete assuming that the controller is connected.
 169          * Otherwise it will fail immediately and return to the requeue list.
 170          */
 171         if (ns->ctrl->state != NVME_CTRL_LIVE &&
 172             ns->ctrl->state != NVME_CTRL_DELETING)
 173                 return true;
 174         if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
 175             !test_bit(NVME_NS_READY, &ns->flags))
 176                 return true;
 177         return false;
 178 }
 179
 180 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 181 {
 182         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 183         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 184
 185         list_for_each_entry_rcu(ns, &head->list, siblings) {
 186                 if (nvme_path_is_disabled(ns))
 187                         continue;
 188
 189                 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
 190                         distance = node_distance(node, ns->ctrl->numa_node);
 191                 else
 192                         distance = LOCAL_DISTANCE;
 193
 194                 switch (ns->ana_state) {
 195                 case NVME_ANA_OPTIMIZED:
 196                         if (distance < found_distance) {
 197                                 found_distance = distance;
 198                                 found = ns;
 199                         }
 200                         break;
 201                 case NVME_ANA_NONOPTIMIZED:
 202                         if (distance < fallback_distance) {
 203                                 fallback_distance = distance;
 204                                 fallback = ns;
 205                         }
 206                         break;
 207                 default:
 208                         break;
 209                 }
 210         }
 211
 212         if (!found)
 213                 found = fallback;
 214         if (found)
 215                 rcu_assign_pointer(head->current_path[node], found);
 216         return found;
 217 }
 218
 219 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
 220                 struct nvme_ns *ns)
 221 {
 222         ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
 223                         siblings);
 224         if (ns)
 225                 return ns;
 226         return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
 227 }
 228
 229 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 230                 int node, struct nvme_ns *old)
 231 {
 232         struct nvme_ns *ns, *found = NULL;
 233
 234         if (list_is_singular(&head->list)) {
 235                 if (nvme_path_is_disabled(old))
 236                         return NULL;
 237                 return old;
 238         }
 239
 240         for (ns = nvme_next_ns(head, old);
 241              ns && ns != old;
 242              ns = nvme_next_ns(head, ns)) {
 243                 if (nvme_path_is_disabled(ns))
 244                         continue;
 245
 246                 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 247                         found = ns;
 248                         goto out;
 249                 }
 250                 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 251                         found = ns;
 252         }
 253
 254         /*
 255          * The loop above skips the current path for round-robin semantics.
 256          * Fall back to the current path if either:
 257          *  - no other optimized path found and current is optimized,
 258          *  - no other usable path found and current is usable.
 259          */
 260         if (!nvme_path_is_disabled(old) &&
 261             (old->ana_state == NVME_ANA_OPTIMIZED ||
 262              (!found && old->ana_state == NVME_ANA_NONOPTIMIZED)))
 263                 return old;
 264
 265         if (!found)
 266                 return NULL;
 267 out:
 268         rcu_assign_pointer(head->current_path[node], found);
 269         return found;
 270 }
 271
 272 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 273 {
 274         return ns->ctrl->state == NVME_CTRL_LIVE &&
 275                 ns->ana_state == NVME_ANA_OPTIMIZED;
 276 }
 277
 278 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 279 {
 280         int node = numa_node_id();
 281         struct nvme_ns *ns;
 282
 283         ns = srcu_dereference(head->current_path[node], &head->srcu);
 284         if (unlikely(!ns))
 285                 return __nvme_find_path(head, node);
 286
 287         if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR)
 288                 return nvme_round_robin_path(head, node, ns);
 289         if (unlikely(!nvme_path_is_optimized(ns)))
 290                 return __nvme_find_path(head, node);
 291         return ns;
 292 }
 293
 294 static bool nvme_available_path(struct nvme_ns_head *head)
 295 {
 296         struct nvme_ns *ns;
 297
 298         list_for_each_entry_rcu(ns, &head->list, siblings) {
 299                 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))
 300                         continue;
 301                 switch (ns->ctrl->state) {
 302                 case NVME_CTRL_LIVE:
 303                 case NVME_CTRL_RESETTING:
 304                 case NVME_CTRL_CONNECTING:
 305                         /* fallthru */
 306                         return true;
 307                 default:
 308                         break;
 309                 }
 310         }
 311         return false;
 312 }
 313
 314 static blk_qc_t nvme_ns_head_submit_bio(struct bio *bio)
 315 {
 316         struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
 317         struct device *dev = disk_to_dev(head->disk);
 318         struct nvme_ns *ns;
 319         blk_qc_t ret = BLK_QC_T_NONE;
 320         int srcu_idx;
 321
 322         /*
 323          * The namespace might be going away and the bio might be moved to a
 324          * different queue via blk_steal_bios(), so we need to use the bio_split
 325          * pool from the original queue to allocate the bvecs from.
 326          */
 327         blk_queue_split(&bio);
 328
 329         srcu_idx = srcu_read_lock(&head->srcu);
 330         ns = nvme_find_path(head);
 331         if (likely(ns)) {
 332                 bio_set_dev(bio, ns->disk->part0);
 333                 bio->bi_opf |= REQ_NVME_MPATH;
 334                 trace_block_bio_remap(bio, disk_devt(ns->head->disk),
 335                                       bio->bi_iter.bi_sector);
 336                 ret = submit_bio_noacct(bio);
 337         } else if (nvme_available_path(head)) {
 338                 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 339
 340                 spin_lock_irq(&head->requeue_lock);
 341                 bio_list_add(&head->requeue_list, bio);
 342                 spin_unlock_irq(&head->requeue_lock);
 343         } else {
 344                 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 345
 346                 bio->bi_status = BLK_STS_IOERR;
 347                 bio_endio(bio);
 348         }
 349
 350         srcu_read_unlock(&head->srcu, srcu_idx);
 351         return ret;
 352 }
 353
 354 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
 355 {
 356         if (!nvme_tryget_ns_head(bdev->bd_disk->private_data))
 357                 return -ENXIO;
 358         return 0;
 359 }
 360
 361 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
 362 {
 363         nvme_put_ns_head(disk->private_data);
 364 }
 365
 366 #ifdef CONFIG_BLK_DEV_ZONED
 367 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector,
 368                 unsigned int nr_zones, report_zones_cb cb, void *data)
 369 {
 370         struct nvme_ns_head *head = disk->private_data;
 371         struct nvme_ns *ns;
 372         int srcu_idx, ret = -EWOULDBLOCK;
 373
 374         srcu_idx = srcu_read_lock(&head->srcu);
 375         ns = nvme_find_path(head);
 376         if (ns)
 377                 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);
 378         srcu_read_unlock(&head->srcu, srcu_idx);
 379         return ret;
 380 }
 381 #else
 382 #define nvme_ns_head_report_zones       NULL
 383 #endif /* CONFIG_BLK_DEV_ZONED */
 384
 385 const struct block_device_operations nvme_ns_head_ops = {
 386         .owner          = THIS_MODULE,
 387         .submit_bio     = nvme_ns_head_submit_bio,
 388         .open           = nvme_ns_head_open,
 389         .release        = nvme_ns_head_release,
 390         .ioctl          = nvme_ns_head_ioctl,
 391         .getgeo         = nvme_getgeo,
 392         .report_zones   = nvme_ns_head_report_zones,
 393         .pr_ops         = &nvme_pr_ops,
 394 };
 395
 396 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev)
 397 {
 398         return container_of(cdev, struct nvme_ns_head, cdev);
 399 }
 400
 401 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file)
 402 {
 403         if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev)))
 404                 return -ENXIO;
 405         return 0;
 406 }
 407
 408 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file)
 409 {
 410         nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev));
 411         return 0;
 412 }
 413
 414 static const struct file_operations nvme_ns_head_chr_fops = {
 415         .owner          = THIS_MODULE,
 416         .open           = nvme_ns_head_chr_open,
 417         .release        = nvme_ns_head_chr_release,
 418         .unlocked_ioctl = nvme_ns_head_chr_ioctl,
 419         .compat_ioctl   = compat_ptr_ioctl,
 420 };
 421
 422 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)
 423 {
 424         int ret;
 425
 426         head->cdev_device.parent = &head->subsys->dev;
 427         ret = dev_set_name(&head->cdev_device, "ng%dn%d",
 428                            head->subsys->instance, head->instance);
 429         if (ret)
 430                 return ret;
 431         ret = nvme_cdev_add(&head->cdev, &head->cdev_device,
 432                             &nvme_ns_head_chr_fops, THIS_MODULE);
 433         return ret;
 434 }
 435
 436 static void nvme_requeue_work(struct work_struct *work)
 437 {
 438         struct nvme_ns_head *head =
 439                 container_of(work, struct nvme_ns_head, requeue_work);
 440         struct bio *bio, *next;
 441
 442         spin_lock_irq(&head->requeue_lock);
 443         next = bio_list_get(&head->requeue_list);
 444         spin_unlock_irq(&head->requeue_lock);
 445
 446         while ((bio = next) != NULL) {
 447                 next = bio->bi_next;
 448                 bio->bi_next = NULL;
 449
 450                 submit_bio_noacct(bio);
 451         }
 452 }
 453
 454 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 455 {
 456         bool vwc = false;
 457
 458         mutex_init(&head->lock);
 459         bio_list_init(&head->requeue_list);
 460         spin_lock_init(&head->requeue_lock);
 461         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 462
 463         /*
 464          * Add a multipath node if the subsystems supports multiple controllers.
 465          * We also do this for private namespaces as the namespace sharing data could
 466          * change after a rescan.
 467          */
 468         if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || !multipath)
 469                 return 0;
 470
 471         head->disk = blk_alloc_disk(ctrl->numa_node);
 472         if (!head->disk)
 473                 return -ENOMEM;
 474         head->disk->fops = &nvme_ns_head_ops;
 475         head->disk->private_data = head;
 476         sprintf(head->disk->disk_name, "nvme%dn%d",
 477                         ctrl->subsys->instance, head->instance);
 478
 479         blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue);
 480         blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue);
 481
 482         /* set to a default value of 512 until the disk is validated */
 483         blk_queue_logical_block_size(head->disk->queue, 512);
 484         blk_set_stacking_limits(&head->disk->queue->limits);
 485
 486         /* we need to propagate up the VMC settings */
 487         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 488                 vwc = true;
 489         blk_queue_write_cache(head->disk->queue, vwc, vwc);
 490         return 0;
 491 }
 492
 493 static void nvme_mpath_set_live(struct nvme_ns *ns)
 494 {
 495         struct nvme_ns_head *head = ns->head;
 496
 497         if (!head->disk)
 498                 return;
 499
 500         if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 501                 device_add_disk(&head->subsys->dev, head->disk,
 502                                 nvme_ns_id_attr_groups);
 503                 nvme_add_ns_head_cdev(head);
 504         }
 505
 506         mutex_lock(&head->lock);
 507         if (nvme_path_is_optimized(ns)) {
 508                 int node, srcu_idx;
 509
 510                 srcu_idx = srcu_read_lock(&head->srcu);
 511                 for_each_node(node)
 512                         __nvme_find_path(head, node);
 513                 srcu_read_unlock(&head->srcu, srcu_idx);
 514         }
 515         mutex_unlock(&head->lock);
 516
 517         synchronize_srcu(&head->srcu);
 518         kblockd_schedule_work(&head->requeue_work);
 519 }
 520
 521 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 522                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 523                         void *))
 524 {
 525         void *base = ctrl->ana_log_buf;
 526         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 527         int error, i;
 528
 529         lockdep_assert_held(&ctrl->ana_lock);
 530
 531         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 532                 struct nvme_ana_group_desc *desc = base + offset;
 533                 u32 nr_nsids;
 534                 size_t nsid_buf_size;
 535
 536                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 537                         return -EINVAL;
 538
 539                 nr_nsids = le32_to_cpu(desc->nnsids);
 540                 nsid_buf_size = nr_nsids * sizeof(__le32);
 541
 542                 if (WARN_ON_ONCE(desc->grpid == 0))
 543                         return -EINVAL;
 544                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 545                         return -EINVAL;
 546                 if (WARN_ON_ONCE(desc->state == 0))
 547                         return -EINVAL;
 548                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 549                         return -EINVAL;
 550
 551                 offset += sizeof(*desc);
 552                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 553                         return -EINVAL;
 554
 555                 error = cb(ctrl, desc, data);
 556                 if (error)
 557                         return error;
 558
 559                 offset += nsid_buf_size;
 560         }
 561
 562         return 0;
 563 }
 564
 565 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 566 {
 567         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 568 }
 569
 570 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 571                 struct nvme_ns *ns)
 572 {
 573         ns->ana_grpid = le32_to_cpu(desc->grpid);
 574         ns->ana_state = desc->state;
 575         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 576
 577         if (nvme_state_is_live(ns->ana_state))
 578                 nvme_mpath_set_live(ns);
 579 }
 580
 581 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 582                 struct nvme_ana_group_desc *desc, void *data)
 583 {
 584         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 585         unsigned *nr_change_groups = data;
 586         struct nvme_ns *ns;
 587
 588         dev_dbg(ctrl->device, "ANA group %d: %s.\n",
 589                         le32_to_cpu(desc->grpid),
 590                         nvme_ana_state_names[desc->state]);
 591
 592         if (desc->state == NVME_ANA_CHANGE)
 593                 (*nr_change_groups)++;
 594
 595         if (!nr_nsids)
 596                 return 0;
 597
 598         down_read(&ctrl->namespaces_rwsem);
 599         list_for_each_entry(ns, &ctrl->namespaces, list) {
 600                 unsigned nsid;
 601 again:
 602                 nsid = le32_to_cpu(desc->nsids[n]);
 603                 if (ns->head->ns_id < nsid)
 604                         continue;
 605                 if (ns->head->ns_id == nsid)
 606                         nvme_update_ns_ana_state(desc, ns);
 607                 if (++n == nr_nsids)
 608                         break;
 609                 if (ns->head->ns_id > nsid)
 610                         goto again;
 611         }
 612         up_read(&ctrl->namespaces_rwsem);
 613         return 0;
 614 }
 615
 616 static int nvme_read_ana_log(struct nvme_ctrl *ctrl)
 617 {
 618         u32 nr_change_groups = 0;
 619         int error;
 620
 621         mutex_lock(&ctrl->ana_lock);
 622         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM,
 623                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 624         if (error) {
 625                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 626                 goto out_unlock;
 627         }
 628
 629         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 630                         nvme_update_ana_state);
 631         if (error)
 632                 goto out_unlock;
 633
 634         /*
 635          * In theory we should have an ANATT timer per group as they might enter
 636          * the change state at different times.  But that is a lot of overhead
 637          * just to protect against a target that keeps entering new changes
 638          * states while never finishing previous ones.  But we'll still
 639          * eventually time out once all groups are in change state, so this
 640          * isn't a big deal.
 641          *
 642          * We also double the ANATT value to provide some slack for transports
 643          * or AEN processing overhead.
 644          */
 645         if (nr_change_groups)
 646                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 647         else
 648                 del_timer_sync(&ctrl->anatt_timer);
 649 out_unlock:
 650         mutex_unlock(&ctrl->ana_lock);
 651         return error;
 652 }
 653
 654 static void nvme_ana_work(struct work_struct *work)
 655 {
 656         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 657
 658         if (ctrl->state != NVME_CTRL_LIVE)
 659                 return;
 660
 661         nvme_read_ana_log(ctrl);
 662 }
 663
 664 static void nvme_anatt_timeout(struct timer_list *t)
 665 {
 666         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 667
 668         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 669         nvme_reset_ctrl(ctrl);
 670 }
 671
 672 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 673 {
 674         if (!nvme_ctrl_use_ana(ctrl))
 675                 return;
 676         del_timer_sync(&ctrl->anatt_timer);
 677         cancel_work_sync(&ctrl->ana_work);
 678 }
 679
 680 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store)  \
 681         struct device_attribute subsys_attr_##_name =   \
 682                 __ATTR(_name, _mode, _show, _store)
 683
 684 static const char *nvme_iopolicy_names[] = {
 685         [NVME_IOPOLICY_NUMA]    = "numa",
 686         [NVME_IOPOLICY_RR]      = "round-robin",
 687 };
 688
 689 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 690                 struct device_attribute *attr, char *buf)
 691 {
 692         struct nvme_subsystem *subsys =
 693                 container_of(dev, struct nvme_subsystem, dev);
 694
 695         return sysfs_emit(buf, "%s\n",
 696                           nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 697 }
 698
 699 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 700                 struct device_attribute *attr, const char *buf, size_t count)
 701 {
 702         struct nvme_subsystem *subsys =
 703                 container_of(dev, struct nvme_subsystem, dev);
 704         int i;
 705
 706         for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 707                 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
 708                         WRITE_ONCE(subsys->iopolicy, i);
 709                         return count;
 710                 }
 711         }
 712
 713         return -EINVAL;
 714 }
 715 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
 716                       nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
 717
 718 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 719                 char *buf)
 720 {
 721         return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 722 }
 723 DEVICE_ATTR_RO(ana_grpid);
 724
 725 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 726                 char *buf)
 727 {
 728         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 729
 730         return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 731 }
 732 DEVICE_ATTR_RO(ana_state);
 733
 734 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
 735                 struct nvme_ana_group_desc *desc, void *data)
 736 {
 737         struct nvme_ana_group_desc *dst = data;
 738
 739         if (desc->grpid != dst->grpid)
 740                 return 0;
 741
 742         *dst = *desc;
 743         return -ENXIO; /* just break out of the loop */
 744 }
 745
 746 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 747 {
 748         if (nvme_ctrl_use_ana(ns->ctrl)) {
 749                 struct nvme_ana_group_desc desc = {
 750                         .grpid = id->anagrpid,
 751                         .state = 0,
 752                 };
 753
 754                 mutex_lock(&ns->ctrl->ana_lock);
 755                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
 756                 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);
 757                 mutex_unlock(&ns->ctrl->ana_lock);
 758                 if (desc.state) {
 759                         /* found the group desc: update */
 760                         nvme_update_ns_ana_state(&desc, ns);
 761                 } else {
 762                         /* group desc not found: trigger a re-read */
 763                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
 764                         queue_work(nvme_wq, &ns->ctrl->ana_work);
 765                 }
 766         } else {
 767                 ns->ana_state = NVME_ANA_OPTIMIZED;
 768                 nvme_mpath_set_live(ns);
 769         }
 770
 771         if (blk_queue_stable_writes(ns->queue) && ns->head->disk)
 772                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
 773                                    ns->head->disk->queue);
 774 #ifdef CONFIG_BLK_DEV_ZONED
 775         if (blk_queue_is_zoned(ns->queue) && ns->head->disk)
 776                 ns->head->disk->queue->nr_zones = ns->queue->nr_zones;
 777 #endif
 778 }
 779
 780 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head)
 781 {
 782         if (!head->disk)
 783                 return;
 784         kblockd_schedule_work(&head->requeue_work);
 785         if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 786                 nvme_cdev_del(&head->cdev, &head->cdev_device);
 787                 del_gendisk(head->disk);
 788         }
 789 }
 790
 791 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 792 {
 793         if (!head->disk)
 794                 return;
 795         blk_set_queue_dying(head->disk->queue);
 796         /* make sure all pending bios are cleaned up */
 797         kblockd_schedule_work(&head->requeue_work);
 798         flush_work(&head->requeue_work);
 799         blk_cleanup_disk(head->disk);
 800 }
 801
 802 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
 803 {
 804         mutex_init(&ctrl->ana_lock);
 805         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 806         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 807 }
 808
 809 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 810 {
 811         size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT;
 812         size_t ana_log_size;
 813         int error = 0;
 814
 815         /* check if multipath is enabled and we have the capability */
 816         if (!multipath || !ctrl->subsys ||
 817             !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA))
 818                 return 0;
 819
 820         if (!ctrl->max_namespaces ||
 821             ctrl->max_namespaces > le32_to_cpu(id->nn)) {
 822                 dev_err(ctrl->device,
 823                         "Invalid MNAN value %u\n", ctrl->max_namespaces);
 824                 return -EINVAL;
 825         }
 826
 827         ctrl->anacap = id->anacap;
 828         ctrl->anatt = id->anatt;
 829         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 830         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 831
 832         ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 833                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) +
 834                 ctrl->max_namespaces * sizeof(__le32);
 835         if (ana_log_size > max_transfer_size) {
 836                 dev_err(ctrl->device,
 837                         "ANA log page size (%zd) larger than MDTS (%zd).\n",
 838                         ana_log_size, max_transfer_size);
 839                 dev_err(ctrl->device, "disabling ANA support.\n");
 840                 goto out_uninit;
 841         }
 842         if (ana_log_size > ctrl->ana_log_size) {
 843                 nvme_mpath_stop(ctrl);
 844                 kfree(ctrl->ana_log_buf);
 845                 ctrl->ana_log_buf = kmalloc(ana_log_size, GFP_KERNEL);
 846                 if (!ctrl->ana_log_buf)
 847                         return -ENOMEM;
 848         }
 849         ctrl->ana_log_size = ana_log_size;
 850         error = nvme_read_ana_log(ctrl);
 851         if (error)
 852                 goto out_uninit;
 853         return 0;
 854
 855 out_uninit:
 856         nvme_mpath_uninit(ctrl);
 857         return error;
 858 }
 859
 860 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 861 {
 862         kfree(ctrl->ana_log_buf);
 863         ctrl->ana_log_buf = NULL;
 864 }