drivers/nvme/host/multipath.c

   1 /*
   2  * Copyright (c) 2017-2018 Christoph Hellwig.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  */
  13
  14 #include <linux/moduleparam.h>
  15 #include <trace/events/block.h>
  16 #include "nvme.h"
  17
  18 static bool multipath = true;
  19 module_param(multipath, bool, 0444);
  20 MODULE_PARM_DESC(multipath,
  21         "turn on native support for multiple controllers per subsystem");
  22
  23 inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
  24 {
  25         return multipath && ctrl->subsys && (ctrl->subsys->cmic & (1 << 3));
  26 }
  27
  28 /*
  29  * If multipathing is enabled we need to always use the subsystem instance
  30  * number for numbering our devices to avoid conflicts between subsystems that
  31  * have multiple controllers and thus use the multipath-aware subsystem node
  32  * and those that have a single controller and use the controller node
  33  * directly.
  34  */
  35 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
  36                         struct nvme_ctrl *ctrl, int *flags)
  37 {
  38         if (!multipath) {
  39                 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
  40         } else if (ns->head->disk) {
  41                 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
  42                                 ctrl->cntlid, ns->head->instance);
  43                 *flags = GENHD_FL_HIDDEN;
  44         } else {
  45                 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
  46                                 ns->head->instance);
  47         }
  48 }
  49
  50 void nvme_failover_req(struct request *req)
  51 {
  52         struct nvme_ns *ns = req->q->queuedata;
  53         u16 status = nvme_req(req)->status;
  54         unsigned long flags;
  55
  56         spin_lock_irqsave(&ns->head->requeue_lock, flags);
  57         blk_steal_bios(&ns->head->requeue_list, req);
  58         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
  59         blk_mq_end_request(req, 0);
  60
  61         switch (status & 0x7ff) {
  62         case NVME_SC_ANA_TRANSITION:
  63         case NVME_SC_ANA_INACCESSIBLE:
  64         case NVME_SC_ANA_PERSISTENT_LOSS:
  65                 /*
  66                  * If we got back an ANA error we know the controller is alive,
  67                  * but not ready to serve this namespaces.  The spec suggests
  68                  * we should update our general state here, but due to the fact
  69                  * that the admin and I/O queues are not serialized that is
  70                  * fundamentally racy.  So instead just clear the current path,
  71                  * mark the the path as pending and kick of a re-read of the ANA
  72                  * log page ASAP.
  73                  */
  74                 nvme_mpath_clear_current_path(ns);
  75                 if (ns->ctrl->ana_log_buf) {
  76                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  77                         queue_work(nvme_wq, &ns->ctrl->ana_work);
  78                 }
  79                 break;
  80         case NVME_SC_HOST_PATH_ERROR:
  81                 /*
  82                  * Temporary transport disruption in talking to the controller.
  83                  * Try to send on a new path.
  84                  */
  85                 nvme_mpath_clear_current_path(ns);
  86                 break;
  87         default:
  88                 /*
  89                  * Reset the controller for any non-ANA error as we don't know
  90                  * what caused the error.
  91                  */
  92                 nvme_reset_ctrl(ns->ctrl);
  93                 break;
  94         }
  95
  96         kblockd_schedule_work(&ns->head->requeue_work);
  97 }
  98
  99 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 100 {
 101         struct nvme_ns *ns;
 102
 103         down_read(&ctrl->namespaces_rwsem);
 104         list_for_each_entry(ns, &ctrl->namespaces, list) {
 105                 if (ns->head->disk)
 106                         kblockd_schedule_work(&ns->head->requeue_work);
 107         }
 108         up_read(&ctrl->namespaces_rwsem);
 109 }
 110
 111 static const char *nvme_ana_state_names[] = {
 112         [0]                             = "invalid state",
 113         [NVME_ANA_OPTIMIZED]            = "optimized",
 114         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 115         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 116         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 117         [NVME_ANA_CHANGE]               = "change",
 118 };
 119
 120 void nvme_mpath_clear_current_path(struct nvme_ns *ns)
 121 {
 122         struct nvme_ns_head *head = ns->head;
 123         int node;
 124
 125         if (!head)
 126                 return;
 127
 128         for_each_node(node) {
 129                 if (ns == rcu_access_pointer(head->current_path[node]))
 130                         rcu_assign_pointer(head->current_path[node], NULL);
 131         }
 132 }
 133
 134 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 135 {
 136         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 137         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 138
 139         list_for_each_entry_rcu(ns, &head->list, siblings) {
 140                 if (ns->ctrl->state != NVME_CTRL_LIVE ||
 141                     test_bit(NVME_NS_ANA_PENDING, &ns->flags))
 142                         continue;
 143
 144                 distance = node_distance(node, dev_to_node(ns->ctrl->dev));
 145
 146                 switch (ns->ana_state) {
 147                 case NVME_ANA_OPTIMIZED:
 148                         if (distance < found_distance) {
 149                                 found_distance = distance;
 150                                 found = ns;
 151                         }
 152                         break;
 153                 case NVME_ANA_NONOPTIMIZED:
 154                         if (distance < fallback_distance) {
 155                                 fallback_distance = distance;
 156                                 fallback = ns;
 157                         }
 158                         break;
 159                 default:
 160                         break;
 161                 }
 162         }
 163
 164         if (!found)
 165                 found = fallback;
 166         if (found)
 167                 rcu_assign_pointer(head->current_path[node], found);
 168         return found;
 169 }
 170
 171 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 172 {
 173         return ns->ctrl->state == NVME_CTRL_LIVE &&
 174                 ns->ana_state == NVME_ANA_OPTIMIZED;
 175 }
 176
 177 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 178 {
 179         int node = numa_node_id();
 180         struct nvme_ns *ns;
 181
 182         ns = srcu_dereference(head->current_path[node], &head->srcu);
 183         if (unlikely(!ns || !nvme_path_is_optimized(ns)))
 184                 ns = __nvme_find_path(head, node);
 185         return ns;
 186 }
 187
 188 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
 189                 struct bio *bio)
 190 {
 191         struct nvme_ns_head *head = q->queuedata;
 192         struct device *dev = disk_to_dev(head->disk);
 193         struct nvme_ns *ns;
 194         blk_qc_t ret = BLK_QC_T_NONE;
 195         int srcu_idx;
 196
 197         srcu_idx = srcu_read_lock(&head->srcu);
 198         ns = nvme_find_path(head);
 199         if (likely(ns)) {
 200                 bio->bi_disk = ns->disk;
 201                 bio->bi_opf |= REQ_NVME_MPATH;
 202                 trace_block_bio_remap(bio->bi_disk->queue, bio,
 203                                       disk_devt(ns->head->disk),
 204                                       bio->bi_iter.bi_sector);
 205                 ret = direct_make_request(bio);
 206         } else if (!list_empty_careful(&head->list)) {
 207                 dev_warn_ratelimited(dev, "no path available - requeuing I/O\n");
 208
 209                 spin_lock_irq(&head->requeue_lock);
 210                 bio_list_add(&head->requeue_list, bio);
 211                 spin_unlock_irq(&head->requeue_lock);
 212         } else {
 213                 dev_warn_ratelimited(dev, "no path - failing I/O\n");
 214
 215                 bio->bi_status = BLK_STS_IOERR;
 216                 bio_endio(bio);
 217         }
 218
 219         srcu_read_unlock(&head->srcu, srcu_idx);
 220         return ret;
 221 }
 222
 223 static bool nvme_ns_head_poll(struct request_queue *q, blk_qc_t qc)
 224 {
 225         struct nvme_ns_head *head = q->queuedata;
 226         struct nvme_ns *ns;
 227         bool found = false;
 228         int srcu_idx;
 229
 230         srcu_idx = srcu_read_lock(&head->srcu);
 231         ns = srcu_dereference(head->current_path[numa_node_id()], &head->srcu);
 232         if (likely(ns && nvme_path_is_optimized(ns)))
 233                 found = ns->queue->poll_fn(q, qc);
 234         srcu_read_unlock(&head->srcu, srcu_idx);
 235         return found;
 236 }
 237
 238 static void nvme_requeue_work(struct work_struct *work)
 239 {
 240         struct nvme_ns_head *head =
 241                 container_of(work, struct nvme_ns_head, requeue_work);
 242         struct bio *bio, *next;
 243
 244         spin_lock_irq(&head->requeue_lock);
 245         next = bio_list_get(&head->requeue_list);
 246         spin_unlock_irq(&head->requeue_lock);
 247
 248         while ((bio = next) != NULL) {
 249                 next = bio->bi_next;
 250                 bio->bi_next = NULL;
 251
 252                 /*
 253                  * Reset disk to the mpath node and resubmit to select a new
 254                  * path.
 255                  */
 256                 bio->bi_disk = head->disk;
 257                 generic_make_request(bio);
 258         }
 259 }
 260
 261 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 262 {
 263         struct request_queue *q;
 264         bool vwc = false;
 265
 266         mutex_init(&head->lock);
 267         bio_list_init(&head->requeue_list);
 268         spin_lock_init(&head->requeue_lock);
 269         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 270
 271         /*
 272          * Add a multipath node if the subsystems supports multiple controllers.
 273          * We also do this for private namespaces as the namespace sharing data could
 274          * change after a rescan.
 275          */
 276         if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
 277                 return 0;
 278
 279         q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE, NULL);
 280         if (!q)
 281                 goto out;
 282         q->queuedata = head;
 283         blk_queue_make_request(q, nvme_ns_head_make_request);
 284         q->poll_fn = nvme_ns_head_poll;
 285         blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 286         /* set to a default value for 512 until disk is validated */
 287         blk_queue_logical_block_size(q, 512);
 288         blk_set_stacking_limits(&q->limits);
 289
 290         /* we need to propagate up the VMC settings */
 291         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 292                 vwc = true;
 293         blk_queue_write_cache(q, vwc, vwc);
 294
 295         head->disk = alloc_disk(0);
 296         if (!head->disk)
 297                 goto out_cleanup_queue;
 298         head->disk->fops = &nvme_ns_head_ops;
 299         head->disk->private_data = head;
 300         head->disk->queue = q;
 301         head->disk->flags = GENHD_FL_EXT_DEVT;
 302         sprintf(head->disk->disk_name, "nvme%dn%d",
 303                         ctrl->subsys->instance, head->instance);
 304         return 0;
 305
 306 out_cleanup_queue:
 307         blk_cleanup_queue(q);
 308 out:
 309         return -ENOMEM;
 310 }
 311
 312 static void nvme_mpath_set_live(struct nvme_ns *ns)
 313 {
 314         struct nvme_ns_head *head = ns->head;
 315
 316         lockdep_assert_held(&ns->head->lock);
 317
 318         if (!head->disk)
 319                 return;
 320
 321         if (!(head->disk->flags & GENHD_FL_UP))
 322                 device_add_disk(&head->subsys->dev, head->disk,
 323                                 nvme_ns_id_attr_groups);
 324
 325         if (nvme_path_is_optimized(ns)) {
 326                 int node, srcu_idx;
 327
 328                 srcu_idx = srcu_read_lock(&head->srcu);
 329                 for_each_node(node)
 330                         __nvme_find_path(head, node);
 331                 srcu_read_unlock(&head->srcu, srcu_idx);
 332         }
 333
 334         kblockd_schedule_work(&ns->head->requeue_work);
 335 }
 336
 337 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 338                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 339                         void *))
 340 {
 341         void *base = ctrl->ana_log_buf;
 342         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 343         int error, i;
 344
 345         lockdep_assert_held(&ctrl->ana_lock);
 346
 347         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 348                 struct nvme_ana_group_desc *desc = base + offset;
 349                 u32 nr_nsids = le32_to_cpu(desc->nnsids);
 350                 size_t nsid_buf_size = nr_nsids * sizeof(__le32);
 351
 352                 if (WARN_ON_ONCE(desc->grpid == 0))
 353                         return -EINVAL;
 354                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 355                         return -EINVAL;
 356                 if (WARN_ON_ONCE(desc->state == 0))
 357                         return -EINVAL;
 358                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 359                         return -EINVAL;
 360
 361                 offset += sizeof(*desc);
 362                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 363                         return -EINVAL;
 364
 365                 error = cb(ctrl, desc, data);
 366                 if (error)
 367                         return error;
 368
 369                 offset += nsid_buf_size;
 370                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 371                         return -EINVAL;
 372         }
 373
 374         return 0;
 375 }
 376
 377 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 378 {
 379         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 380 }
 381
 382 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 383                 struct nvme_ns *ns)
 384 {
 385         enum nvme_ana_state old;
 386
 387         mutex_lock(&ns->head->lock);
 388         old = ns->ana_state;
 389         ns->ana_grpid = le32_to_cpu(desc->grpid);
 390         ns->ana_state = desc->state;
 391         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 392
 393         if (nvme_state_is_live(ns->ana_state) && !nvme_state_is_live(old))
 394                 nvme_mpath_set_live(ns);
 395         mutex_unlock(&ns->head->lock);
 396 }
 397
 398 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 399                 struct nvme_ana_group_desc *desc, void *data)
 400 {
 401         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 402         unsigned *nr_change_groups = data;
 403         struct nvme_ns *ns;
 404
 405         dev_info(ctrl->device, "ANA group %d: %s.\n",
 406                         le32_to_cpu(desc->grpid),
 407                         nvme_ana_state_names[desc->state]);
 408
 409         if (desc->state == NVME_ANA_CHANGE)
 410                 (*nr_change_groups)++;
 411
 412         if (!nr_nsids)
 413                 return 0;
 414
 415         down_write(&ctrl->namespaces_rwsem);
 416         list_for_each_entry(ns, &ctrl->namespaces, list) {
 417                 if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))
 418                         continue;
 419                 nvme_update_ns_ana_state(desc, ns);
 420                 if (++n == nr_nsids)
 421                         break;
 422         }
 423         up_write(&ctrl->namespaces_rwsem);
 424         WARN_ON_ONCE(n < nr_nsids);
 425         return 0;
 426 }
 427
 428 static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 429 {
 430         u32 nr_change_groups = 0;
 431         int error;
 432
 433         mutex_lock(&ctrl->ana_lock);
 434         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
 435                         groups_only ? NVME_ANA_LOG_RGO : 0,
 436                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 437         if (error) {
 438                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 439                 goto out_unlock;
 440         }
 441
 442         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 443                         nvme_update_ana_state);
 444         if (error)
 445                 goto out_unlock;
 446
 447         /*
 448          * In theory we should have an ANATT timer per group as they might enter
 449          * the change state at different times.  But that is a lot of overhead
 450          * just to protect against a target that keeps entering new changes
 451          * states while never finishing previous ones.  But we'll still
 452          * eventually time out once all groups are in change state, so this
 453          * isn't a big deal.
 454          *
 455          * We also double the ANATT value to provide some slack for transports
 456          * or AEN processing overhead.
 457          */
 458         if (nr_change_groups)
 459                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 460         else
 461                 del_timer_sync(&ctrl->anatt_timer);
 462 out_unlock:
 463         mutex_unlock(&ctrl->ana_lock);
 464         return error;
 465 }
 466
 467 static void nvme_ana_work(struct work_struct *work)
 468 {
 469         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 470
 471         nvme_read_ana_log(ctrl, false);
 472 }
 473
 474 static void nvme_anatt_timeout(struct timer_list *t)
 475 {
 476         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 477
 478         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 479         nvme_reset_ctrl(ctrl);
 480 }
 481
 482 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 483 {
 484         if (!nvme_ctrl_use_ana(ctrl))
 485                 return;
 486         del_timer_sync(&ctrl->anatt_timer);
 487         cancel_work_sync(&ctrl->ana_work);
 488 }
 489
 490 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 491                 char *buf)
 492 {
 493         return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 494 }
 495 DEVICE_ATTR_RO(ana_grpid);
 496
 497 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 498                 char *buf)
 499 {
 500         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 501
 502         return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 503 }
 504 DEVICE_ATTR_RO(ana_state);
 505
 506 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
 507                 struct nvme_ana_group_desc *desc, void *data)
 508 {
 509         struct nvme_ns *ns = data;
 510
 511         if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
 512                 nvme_update_ns_ana_state(desc, ns);
 513                 return -ENXIO; /* just break out of the loop */
 514         }
 515
 516         return 0;
 517 }
 518
 519 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 520 {
 521         if (nvme_ctrl_use_ana(ns->ctrl)) {
 522                 mutex_lock(&ns->ctrl->ana_lock);
 523                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
 524                 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
 525                 mutex_unlock(&ns->ctrl->ana_lock);
 526         } else {
 527                 mutex_lock(&ns->head->lock);
 528                 ns->ana_state = NVME_ANA_OPTIMIZED;
 529                 nvme_mpath_set_live(ns);
 530                 mutex_unlock(&ns->head->lock);
 531         }
 532 }
 533
 534 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 535 {
 536         if (!head->disk)
 537                 return;
 538         if (head->disk->flags & GENHD_FL_UP)
 539                 del_gendisk(head->disk);
 540         blk_set_queue_dying(head->disk->queue);
 541         /* make sure all pending bios are cleaned up */
 542         kblockd_schedule_work(&head->requeue_work);
 543         flush_work(&head->requeue_work);
 544         blk_cleanup_queue(head->disk->queue);
 545         put_disk(head->disk);
 546 }
 547
 548 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 549 {
 550         int error;
 551
 552         if (!nvme_ctrl_use_ana(ctrl))
 553                 return 0;
 554
 555         ctrl->anacap = id->anacap;
 556         ctrl->anatt = id->anatt;
 557         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 558         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 559
 560         mutex_init(&ctrl->ana_lock);
 561         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 562         ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 563                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
 564         if (!(ctrl->anacap & (1 << 6)))
 565                 ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
 566
 567         if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
 568                 dev_err(ctrl->device,
 569                         "ANA log page size (%zd) larger than MDTS (%d).\n",
 570                         ctrl->ana_log_size,
 571                         ctrl->max_hw_sectors << SECTOR_SHIFT);
 572                 dev_err(ctrl->device, "disabling ANA support.\n");
 573                 return 0;
 574         }
 575
 576         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 577         ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
 578         if (!ctrl->ana_log_buf) {
 579                 error = -ENOMEM;
 580                 goto out;
 581         }
 582
 583         error = nvme_read_ana_log(ctrl, true);
 584         if (error)
 585                 goto out_free_ana_log_buf;
 586         return 0;
 587 out_free_ana_log_buf:
 588         kfree(ctrl->ana_log_buf);
 589 out:
 590         return error;
 591 }
 592
 593 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 594 {
 595         kfree(ctrl->ana_log_buf);
 596 }
 597