nvme: fix multipath crash when ANA is deactivated
[platform/kernel/linux-rpi.git] / drivers / nvme / host / multipath.c
1 /*
2  * Copyright (c) 2017-2018 Christoph Hellwig.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  */
13
14 #include <linux/moduleparam.h>
15 #include <trace/events/block.h>
16 #include "nvme.h"
17
18 static bool multipath = true;
19 module_param(multipath, bool, 0444);
20 MODULE_PARM_DESC(multipath,
21         "turn on native support for multiple controllers per subsystem");
22
23 /*
24  * If multipathing is enabled we need to always use the subsystem instance
25  * number for numbering our devices to avoid conflicts between subsystems that
26  * have multiple controllers and thus use the multipath-aware subsystem node
27  * and those that have a single controller and use the controller node
28  * directly.
29  */
30 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
31                         struct nvme_ctrl *ctrl, int *flags)
32 {
33         if (!multipath) {
34                 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
35         } else if (ns->head->disk) {
36                 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
37                                 ctrl->cntlid, ns->head->instance);
38                 *flags = GENHD_FL_HIDDEN;
39         } else {
40                 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
41                                 ns->head->instance);
42         }
43 }
44
45 void nvme_failover_req(struct request *req)
46 {
47         struct nvme_ns *ns = req->q->queuedata;
48         u16 status = nvme_req(req)->status;
49         unsigned long flags;
50
51         spin_lock_irqsave(&ns->head->requeue_lock, flags);
52         blk_steal_bios(&ns->head->requeue_list, req);
53         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
54         blk_mq_end_request(req, 0);
55
56         switch (status & 0x7ff) {
57         case NVME_SC_ANA_TRANSITION:
58         case NVME_SC_ANA_INACCESSIBLE:
59         case NVME_SC_ANA_PERSISTENT_LOSS:
60                 /*
61                  * If we got back an ANA error we know the controller is alive,
62                  * but not ready to serve this namespaces.  The spec suggests
63                  * we should update our general state here, but due to the fact
64                  * that the admin and I/O queues are not serialized that is
65                  * fundamentally racy.  So instead just clear the current path,
66                  * mark the the path as pending and kick of a re-read of the ANA
67                  * log page ASAP.
68                  */
69                 nvme_mpath_clear_current_path(ns);
70                 if (ns->ctrl->ana_log_buf) {
71                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
72                         queue_work(nvme_wq, &ns->ctrl->ana_work);
73                 }
74                 break;
75         case NVME_SC_HOST_PATH_ERROR:
76                 /*
77                  * Temporary transport disruption in talking to the controller.
78                  * Try to send on a new path.
79                  */
80                 nvme_mpath_clear_current_path(ns);
81                 break;
82         default:
83                 /*
84                  * Reset the controller for any non-ANA error as we don't know
85                  * what caused the error.
86                  */
87                 nvme_reset_ctrl(ns->ctrl);
88                 break;
89         }
90
91         kblockd_schedule_work(&ns->head->requeue_work);
92 }
93
94 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
95 {
96         struct nvme_ns *ns;
97
98         down_read(&ctrl->namespaces_rwsem);
99         list_for_each_entry(ns, &ctrl->namespaces, list) {
100                 if (ns->head->disk)
101                         kblockd_schedule_work(&ns->head->requeue_work);
102         }
103         up_read(&ctrl->namespaces_rwsem);
104 }
105
106 static const char *nvme_ana_state_names[] = {
107         [0]                             = "invalid state",
108         [NVME_ANA_OPTIMIZED]            = "optimized",
109         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
110         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
111         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
112         [NVME_ANA_CHANGE]               = "change",
113 };
114
115 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head)
116 {
117         struct nvme_ns *ns, *fallback = NULL;
118
119         list_for_each_entry_rcu(ns, &head->list, siblings) {
120                 if (ns->ctrl->state != NVME_CTRL_LIVE ||
121                     test_bit(NVME_NS_ANA_PENDING, &ns->flags))
122                         continue;
123                 switch (ns->ana_state) {
124                 case NVME_ANA_OPTIMIZED:
125                         rcu_assign_pointer(head->current_path, ns);
126                         return ns;
127                 case NVME_ANA_NONOPTIMIZED:
128                         fallback = ns;
129                         break;
130                 default:
131                         break;
132                 }
133         }
134
135         if (fallback)
136                 rcu_assign_pointer(head->current_path, fallback);
137         return fallback;
138 }
139
140 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
141 {
142         return ns->ctrl->state == NVME_CTRL_LIVE &&
143                 ns->ana_state == NVME_ANA_OPTIMIZED;
144 }
145
146 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
147 {
148         struct nvme_ns *ns = srcu_dereference(head->current_path, &head->srcu);
149
150         if (unlikely(!ns || !nvme_path_is_optimized(ns)))
151                 ns = __nvme_find_path(head);
152         return ns;
153 }
154
155 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
156                 struct bio *bio)
157 {
158         struct nvme_ns_head *head = q->queuedata;
159         struct device *dev = disk_to_dev(head->disk);
160         struct nvme_ns *ns;
161         blk_qc_t ret = BLK_QC_T_NONE;
162         int srcu_idx;
163
164         srcu_idx = srcu_read_lock(&head->srcu);
165         ns = nvme_find_path(head);
166         if (likely(ns)) {
167                 bio->bi_disk = ns->disk;
168                 bio->bi_opf |= REQ_NVME_MPATH;
169                 trace_block_bio_remap(bio->bi_disk->queue, bio,
170                                       disk_devt(ns->head->disk),
171                                       bio->bi_iter.bi_sector);
172                 ret = direct_make_request(bio);
173         } else if (!list_empty_careful(&head->list)) {
174                 dev_warn_ratelimited(dev, "no path available - requeuing I/O\n");
175
176                 spin_lock_irq(&head->requeue_lock);
177                 bio_list_add(&head->requeue_list, bio);
178                 spin_unlock_irq(&head->requeue_lock);
179         } else {
180                 dev_warn_ratelimited(dev, "no path - failing I/O\n");
181
182                 bio->bi_status = BLK_STS_IOERR;
183                 bio_endio(bio);
184         }
185
186         srcu_read_unlock(&head->srcu, srcu_idx);
187         return ret;
188 }
189
190 static bool nvme_ns_head_poll(struct request_queue *q, blk_qc_t qc)
191 {
192         struct nvme_ns_head *head = q->queuedata;
193         struct nvme_ns *ns;
194         bool found = false;
195         int srcu_idx;
196
197         srcu_idx = srcu_read_lock(&head->srcu);
198         ns = srcu_dereference(head->current_path, &head->srcu);
199         if (likely(ns && nvme_path_is_optimized(ns)))
200                 found = ns->queue->poll_fn(q, qc);
201         srcu_read_unlock(&head->srcu, srcu_idx);
202         return found;
203 }
204
205 static void nvme_requeue_work(struct work_struct *work)
206 {
207         struct nvme_ns_head *head =
208                 container_of(work, struct nvme_ns_head, requeue_work);
209         struct bio *bio, *next;
210
211         spin_lock_irq(&head->requeue_lock);
212         next = bio_list_get(&head->requeue_list);
213         spin_unlock_irq(&head->requeue_lock);
214
215         while ((bio = next) != NULL) {
216                 next = bio->bi_next;
217                 bio->bi_next = NULL;
218
219                 /*
220                  * Reset disk to the mpath node and resubmit to select a new
221                  * path.
222                  */
223                 bio->bi_disk = head->disk;
224                 generic_make_request(bio);
225         }
226 }
227
228 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
229 {
230         struct request_queue *q;
231         bool vwc = false;
232
233         mutex_init(&head->lock);
234         bio_list_init(&head->requeue_list);
235         spin_lock_init(&head->requeue_lock);
236         INIT_WORK(&head->requeue_work, nvme_requeue_work);
237
238         /*
239          * Add a multipath node if the subsystems supports multiple controllers.
240          * We also do this for private namespaces as the namespace sharing data could
241          * change after a rescan.
242          */
243         if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
244                 return 0;
245
246         q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE, NULL);
247         if (!q)
248                 goto out;
249         q->queuedata = head;
250         blk_queue_make_request(q, nvme_ns_head_make_request);
251         q->poll_fn = nvme_ns_head_poll;
252         blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
253         /* set to a default value for 512 until disk is validated */
254         blk_queue_logical_block_size(q, 512);
255         blk_set_stacking_limits(&q->limits);
256
257         /* we need to propagate up the VMC settings */
258         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
259                 vwc = true;
260         blk_queue_write_cache(q, vwc, vwc);
261
262         head->disk = alloc_disk(0);
263         if (!head->disk)
264                 goto out_cleanup_queue;
265         head->disk->fops = &nvme_ns_head_ops;
266         head->disk->private_data = head;
267         head->disk->queue = q;
268         head->disk->flags = GENHD_FL_EXT_DEVT;
269         sprintf(head->disk->disk_name, "nvme%dn%d",
270                         ctrl->subsys->instance, head->instance);
271         return 0;
272
273 out_cleanup_queue:
274         blk_cleanup_queue(q);
275 out:
276         return -ENOMEM;
277 }
278
279 static void nvme_mpath_set_live(struct nvme_ns *ns)
280 {
281         struct nvme_ns_head *head = ns->head;
282
283         lockdep_assert_held(&ns->head->lock);
284
285         if (!head->disk)
286                 return;
287
288         if (!(head->disk->flags & GENHD_FL_UP)) {
289                 device_add_disk(&head->subsys->dev, head->disk);
290                 if (sysfs_create_group(&disk_to_dev(head->disk)->kobj,
291                                 &nvme_ns_id_attr_group))
292                         dev_warn(&head->subsys->dev,
293                                  "failed to create id group.\n");
294         }
295
296         kblockd_schedule_work(&ns->head->requeue_work);
297 }
298
299 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
300                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
301                         void *))
302 {
303         void *base = ctrl->ana_log_buf;
304         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
305         int error, i;
306
307         lockdep_assert_held(&ctrl->ana_lock);
308
309         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
310                 struct nvme_ana_group_desc *desc = base + offset;
311                 u32 nr_nsids = le32_to_cpu(desc->nnsids);
312                 size_t nsid_buf_size = nr_nsids * sizeof(__le32);
313
314                 if (WARN_ON_ONCE(desc->grpid == 0))
315                         return -EINVAL;
316                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
317                         return -EINVAL;
318                 if (WARN_ON_ONCE(desc->state == 0))
319                         return -EINVAL;
320                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
321                         return -EINVAL;
322
323                 offset += sizeof(*desc);
324                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
325                         return -EINVAL;
326
327                 error = cb(ctrl, desc, data);
328                 if (error)
329                         return error;
330
331                 offset += nsid_buf_size;
332                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
333                         return -EINVAL;
334         }
335
336         return 0;
337 }
338
339 static inline bool nvme_state_is_live(enum nvme_ana_state state)
340 {
341         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
342 }
343
344 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
345                 struct nvme_ns *ns)
346 {
347         mutex_lock(&ns->head->lock);
348         ns->ana_grpid = le32_to_cpu(desc->grpid);
349         ns->ana_state = desc->state;
350         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
351
352         if (nvme_state_is_live(ns->ana_state))
353                 nvme_mpath_set_live(ns);
354         mutex_unlock(&ns->head->lock);
355 }
356
357 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
358                 struct nvme_ana_group_desc *desc, void *data)
359 {
360         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
361         unsigned *nr_change_groups = data;
362         struct nvme_ns *ns;
363
364         dev_info(ctrl->device, "ANA group %d: %s.\n",
365                         le32_to_cpu(desc->grpid),
366                         nvme_ana_state_names[desc->state]);
367
368         if (desc->state == NVME_ANA_CHANGE)
369                 (*nr_change_groups)++;
370
371         if (!nr_nsids)
372                 return 0;
373
374         down_write(&ctrl->namespaces_rwsem);
375         list_for_each_entry(ns, &ctrl->namespaces, list) {
376                 if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))
377                         continue;
378                 nvme_update_ns_ana_state(desc, ns);
379                 if (++n == nr_nsids)
380                         break;
381         }
382         up_write(&ctrl->namespaces_rwsem);
383         WARN_ON_ONCE(n < nr_nsids);
384         return 0;
385 }
386
387 static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
388 {
389         u32 nr_change_groups = 0;
390         int error;
391
392         mutex_lock(&ctrl->ana_lock);
393         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
394                         groups_only ? NVME_ANA_LOG_RGO : 0,
395                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
396         if (error) {
397                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
398                 goto out_unlock;
399         }
400
401         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
402                         nvme_update_ana_state);
403         if (error)
404                 goto out_unlock;
405
406         /*
407          * In theory we should have an ANATT timer per group as they might enter
408          * the change state at different times.  But that is a lot of overhead
409          * just to protect against a target that keeps entering new changes
410          * states while never finishing previous ones.  But we'll still
411          * eventually time out once all groups are in change state, so this
412          * isn't a big deal.
413          *
414          * We also double the ANATT value to provide some slack for transports
415          * or AEN processing overhead.
416          */
417         if (nr_change_groups)
418                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
419         else
420                 del_timer_sync(&ctrl->anatt_timer);
421 out_unlock:
422         mutex_unlock(&ctrl->ana_lock);
423         return error;
424 }
425
426 static void nvme_ana_work(struct work_struct *work)
427 {
428         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
429
430         nvme_read_ana_log(ctrl, false);
431 }
432
433 static void nvme_anatt_timeout(struct timer_list *t)
434 {
435         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
436
437         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
438         nvme_reset_ctrl(ctrl);
439 }
440
441 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
442 {
443         if (!nvme_ctrl_use_ana(ctrl))
444                 return;
445         del_timer_sync(&ctrl->anatt_timer);
446         cancel_work_sync(&ctrl->ana_work);
447 }
448
449 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
450                 char *buf)
451 {
452         return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
453 }
454 DEVICE_ATTR_RO(ana_grpid);
455
456 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
457                 char *buf)
458 {
459         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
460
461         return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
462 }
463 DEVICE_ATTR_RO(ana_state);
464
465 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
466                 struct nvme_ana_group_desc *desc, void *data)
467 {
468         struct nvme_ns *ns = data;
469
470         if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
471                 nvme_update_ns_ana_state(desc, ns);
472                 return -ENXIO; /* just break out of the loop */
473         }
474
475         return 0;
476 }
477
478 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
479 {
480         if (nvme_ctrl_use_ana(ns->ctrl)) {
481                 mutex_lock(&ns->ctrl->ana_lock);
482                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
483                 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
484                 mutex_unlock(&ns->ctrl->ana_lock);
485         } else {
486                 mutex_lock(&ns->head->lock);
487                 ns->ana_state = NVME_ANA_OPTIMIZED; 
488                 nvme_mpath_set_live(ns);
489                 mutex_unlock(&ns->head->lock);
490         }
491 }
492
493 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
494 {
495         if (!head->disk)
496                 return;
497         if (head->disk->flags & GENHD_FL_UP) {
498                 sysfs_remove_group(&disk_to_dev(head->disk)->kobj,
499                                    &nvme_ns_id_attr_group);
500                 del_gendisk(head->disk);
501         }
502         blk_set_queue_dying(head->disk->queue);
503         /* make sure all pending bios are cleaned up */
504         kblockd_schedule_work(&head->requeue_work);
505         flush_work(&head->requeue_work);
506         blk_cleanup_queue(head->disk->queue);
507         put_disk(head->disk);
508 }
509
510 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
511 {
512         int error;
513
514         /* check if multipath is enabled and we have the capability */
515         if (!multipath || !ctrl->subsys || !(ctrl->subsys->cmic & (1 << 3)))
516                 return 0;
517
518         ctrl->anacap = id->anacap;
519         ctrl->anatt = id->anatt;
520         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
521         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
522
523         mutex_init(&ctrl->ana_lock);
524         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
525         ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
526                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
527         ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
528
529         if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
530                 dev_err(ctrl->device,
531                         "ANA log page size (%zd) larger than MDTS (%d).\n",
532                         ctrl->ana_log_size,
533                         ctrl->max_hw_sectors << SECTOR_SHIFT);
534                 dev_err(ctrl->device, "disabling ANA support.\n");
535                 return 0;
536         }
537
538         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
539         ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
540         if (!ctrl->ana_log_buf) {
541                 error = -ENOMEM;
542                 goto out;
543         }
544
545         error = nvme_read_ana_log(ctrl, true);
546         if (error)
547                 goto out_free_ana_log_buf;
548         return 0;
549 out_free_ana_log_buf:
550         kfree(ctrl->ana_log_buf);
551         ctrl->ana_log_buf = NULL;
552 out:
553         return error;
554 }
555
556 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
557 {
558         kfree(ctrl->ana_log_buf);
559         ctrl->ana_log_buf = NULL;
560 }
561