drivers/nvme/host/fc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
   4  */
   5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   6 #include <linux/module.h>
   7 #include <linux/parser.h>
   8 #include <uapi/scsi/fc/fc_fs.h>
   9 #include <uapi/scsi/fc/fc_els.h>
  10 #include <linux/delay.h>
  11 #include <linux/overflow.h>
  12 #include <linux/blk-cgroup.h>
  13 #include "nvme.h"
  14 #include "fabrics.h"
  15 #include <linux/nvme-fc-driver.h>
  16 #include <linux/nvme-fc.h>
  17 #include "fc.h"
  18 #include <scsi/scsi_transport_fc.h>
  19 #include <linux/blk-mq-pci.h>
  20
  21 /* *************************** Data Structures/Defines ****************** */
  22
  23
  24 enum nvme_fc_queue_flags {
  25         NVME_FC_Q_CONNECTED = 0,
  26         NVME_FC_Q_LIVE,
  27 };
  28
  29 #define NVME_FC_DEFAULT_DEV_LOSS_TMO    60      /* seconds */
  30 #define NVME_FC_DEFAULT_RECONNECT_TMO   2       /* delay between reconnects
  31                                                  * when connected and a
  32                                                  * connection failure.
  33                                                  */
  34
  35 struct nvme_fc_queue {
  36         struct nvme_fc_ctrl     *ctrl;
  37         struct device           *dev;
  38         struct blk_mq_hw_ctx    *hctx;
  39         void                    *lldd_handle;
  40         size_t                  cmnd_capsule_len;
  41         u32                     qnum;
  42         u32                     rqcnt;
  43         u32                     seqno;
  44
  45         u64                     connection_id;
  46         atomic_t                csn;
  47
  48         unsigned long           flags;
  49 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
  50
  51 enum nvme_fcop_flags {
  52         FCOP_FLAGS_TERMIO       = (1 << 0),
  53         FCOP_FLAGS_AEN          = (1 << 1),
  54 };
  55
  56 struct nvmefc_ls_req_op {
  57         struct nvmefc_ls_req    ls_req;
  58
  59         struct nvme_fc_rport    *rport;
  60         struct nvme_fc_queue    *queue;
  61         struct request          *rq;
  62         u32                     flags;
  63
  64         int                     ls_error;
  65         struct completion       ls_done;
  66         struct list_head        lsreq_list;     /* rport->ls_req_list */
  67         bool                    req_queued;
  68 };
  69
  70 struct nvmefc_ls_rcv_op {
  71         struct nvme_fc_rport            *rport;
  72         struct nvmefc_ls_rsp            *lsrsp;
  73         union nvmefc_ls_requests        *rqstbuf;
  74         union nvmefc_ls_responses       *rspbuf;
  75         u16                             rqstdatalen;
  76         bool                            handled;
  77         dma_addr_t                      rspdma;
  78         struct list_head                lsrcv_list;     /* rport->ls_rcv_list */
  79 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
  80
  81 enum nvme_fcpop_state {
  82         FCPOP_STATE_UNINIT      = 0,
  83         FCPOP_STATE_IDLE        = 1,
  84         FCPOP_STATE_ACTIVE      = 2,
  85         FCPOP_STATE_ABORTED     = 3,
  86         FCPOP_STATE_COMPLETE    = 4,
  87 };
  88
  89 struct nvme_fc_fcp_op {
  90         struct nvme_request     nreq;           /*
  91                                                  * nvme/host/core.c
  92                                                  * requires this to be
  93                                                  * the 1st element in the
  94                                                  * private structure
  95                                                  * associated with the
  96                                                  * request.
  97                                                  */
  98         struct nvmefc_fcp_req   fcp_req;
  99
 100         struct nvme_fc_ctrl     *ctrl;
 101         struct nvme_fc_queue    *queue;
 102         struct request          *rq;
 103
 104         atomic_t                state;
 105         u32                     flags;
 106         u32                     rqno;
 107         u32                     nents;
 108
 109         struct nvme_fc_cmd_iu   cmd_iu;
 110         struct nvme_fc_ersp_iu  rsp_iu;
 111 };
 112
 113 struct nvme_fcp_op_w_sgl {
 114         struct nvme_fc_fcp_op   op;
 115         struct scatterlist      sgl[NVME_INLINE_SG_CNT];
 116         uint8_t                 priv[];
 117 };
 118
 119 struct nvme_fc_lport {
 120         struct nvme_fc_local_port       localport;
 121
 122         struct ida                      endp_cnt;
 123         struct list_head                port_list;      /* nvme_fc_port_list */
 124         struct list_head                endp_list;
 125         struct device                   *dev;   /* physical device for dma */
 126         struct nvme_fc_port_template    *ops;
 127         struct kref                     ref;
 128         atomic_t                        act_rport_cnt;
 129 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
 130
 131 struct nvme_fc_rport {
 132         struct nvme_fc_remote_port      remoteport;
 133
 134         struct list_head                endp_list; /* for lport->endp_list */
 135         struct list_head                ctrl_list;
 136         struct list_head                ls_req_list;
 137         struct list_head                ls_rcv_list;
 138         struct list_head                disc_list;
 139         struct device                   *dev;   /* physical device for dma */
 140         struct nvme_fc_lport            *lport;
 141         spinlock_t                      lock;
 142         struct kref                     ref;
 143         atomic_t                        act_ctrl_cnt;
 144         unsigned long                   dev_loss_end;
 145         struct work_struct              lsrcv_work;
 146 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
 147
 148 /* fc_ctrl flags values - specified as bit positions */
 149 #define ASSOC_ACTIVE            0
 150 #define ASSOC_FAILED            1
 151 #define FCCTRL_TERMIO           2
 152
 153 struct nvme_fc_ctrl {
 154         spinlock_t              lock;
 155         struct nvme_fc_queue    *queues;
 156         struct device           *dev;
 157         struct nvme_fc_lport    *lport;
 158         struct nvme_fc_rport    *rport;
 159         u32                     cnum;
 160
 161         bool                    ioq_live;
 162         u64                     association_id;
 163         struct nvmefc_ls_rcv_op *rcv_disconn;
 164
 165         struct list_head        ctrl_list;      /* rport->ctrl_list */
 166
 167         struct blk_mq_tag_set   admin_tag_set;
 168         struct blk_mq_tag_set   tag_set;
 169
 170         struct work_struct      ioerr_work;
 171         struct delayed_work     connect_work;
 172
 173         struct kref             ref;
 174         unsigned long           flags;
 175         u32                     iocnt;
 176         wait_queue_head_t       ioabort_wait;
 177
 178         struct nvme_fc_fcp_op   aen_ops[NVME_NR_AEN_COMMANDS];
 179
 180         struct nvme_ctrl        ctrl;
 181 };
 182
 183 static inline struct nvme_fc_ctrl *
 184 to_fc_ctrl(struct nvme_ctrl *ctrl)
 185 {
 186         return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
 187 }
 188
 189 static inline struct nvme_fc_lport *
 190 localport_to_lport(struct nvme_fc_local_port *portptr)
 191 {
 192         return container_of(portptr, struct nvme_fc_lport, localport);
 193 }
 194
 195 static inline struct nvme_fc_rport *
 196 remoteport_to_rport(struct nvme_fc_remote_port *portptr)
 197 {
 198         return container_of(portptr, struct nvme_fc_rport, remoteport);
 199 }
 200
 201 static inline struct nvmefc_ls_req_op *
 202 ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
 203 {
 204         return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
 205 }
 206
 207 static inline struct nvme_fc_fcp_op *
 208 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
 209 {
 210         return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
 211 }
 212
 213
 214
 215 /* *************************** Globals **************************** */
 216
 217
 218 static DEFINE_SPINLOCK(nvme_fc_lock);
 219
 220 static LIST_HEAD(nvme_fc_lport_list);
 221 static DEFINE_IDA(nvme_fc_local_port_cnt);
 222 static DEFINE_IDA(nvme_fc_ctrl_cnt);
 223
 224 static struct workqueue_struct *nvme_fc_wq;
 225
 226 static bool nvme_fc_waiting_to_unload;
 227 static DECLARE_COMPLETION(nvme_fc_unload_proceed);
 228
 229 /*
 230  * These items are short-term. They will eventually be moved into
 231  * a generic FC class. See comments in module init.
 232  */
 233 static struct device *fc_udev_device;
 234
 235 static void nvme_fc_complete_rq(struct request *rq);
 236
 237 /* *********************** FC-NVME Port Management ************************ */
 238
 239 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
 240                         struct nvme_fc_queue *, unsigned int);
 241
 242 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
 243
 244
 245 static void
 246 nvme_fc_free_lport(struct kref *ref)
 247 {
 248         struct nvme_fc_lport *lport =
 249                 container_of(ref, struct nvme_fc_lport, ref);
 250         unsigned long flags;
 251
 252         WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
 253         WARN_ON(!list_empty(&lport->endp_list));
 254
 255         /* remove from transport list */
 256         spin_lock_irqsave(&nvme_fc_lock, flags);
 257         list_del(&lport->port_list);
 258         if (nvme_fc_waiting_to_unload && list_empty(&nvme_fc_lport_list))
 259                 complete(&nvme_fc_unload_proceed);
 260         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 261
 262         ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
 263         ida_destroy(&lport->endp_cnt);
 264
 265         put_device(lport->dev);
 266
 267         kfree(lport);
 268 }
 269
 270 static void
 271 nvme_fc_lport_put(struct nvme_fc_lport *lport)
 272 {
 273         kref_put(&lport->ref, nvme_fc_free_lport);
 274 }
 275
 276 static int
 277 nvme_fc_lport_get(struct nvme_fc_lport *lport)
 278 {
 279         return kref_get_unless_zero(&lport->ref);
 280 }
 281
 282
 283 static struct nvme_fc_lport *
 284 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
 285                         struct nvme_fc_port_template *ops,
 286                         struct device *dev)
 287 {
 288         struct nvme_fc_lport *lport;
 289         unsigned long flags;
 290
 291         spin_lock_irqsave(&nvme_fc_lock, flags);
 292
 293         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
 294                 if (lport->localport.node_name != pinfo->node_name ||
 295                     lport->localport.port_name != pinfo->port_name)
 296                         continue;
 297
 298                 if (lport->dev != dev) {
 299                         lport = ERR_PTR(-EXDEV);
 300                         goto out_done;
 301                 }
 302
 303                 if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
 304                         lport = ERR_PTR(-EEXIST);
 305                         goto out_done;
 306                 }
 307
 308                 if (!nvme_fc_lport_get(lport)) {
 309                         /*
 310                          * fails if ref cnt already 0. If so,
 311                          * act as if lport already deleted
 312                          */
 313                         lport = NULL;
 314                         goto out_done;
 315                 }
 316
 317                 /* resume the lport */
 318
 319                 lport->ops = ops;
 320                 lport->localport.port_role = pinfo->port_role;
 321                 lport->localport.port_id = pinfo->port_id;
 322                 lport->localport.port_state = FC_OBJSTATE_ONLINE;
 323
 324                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 325
 326                 return lport;
 327         }
 328
 329         lport = NULL;
 330
 331 out_done:
 332         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 333
 334         return lport;
 335 }
 336
 337 /**
 338  * nvme_fc_register_localport - transport entry point called by an
 339  *                              LLDD to register the existence of a NVME
 340  *                              host FC port.
 341  * @pinfo:     pointer to information about the port to be registered
 342  * @template:  LLDD entrypoints and operational parameters for the port
 343  * @dev:       physical hardware device node port corresponds to. Will be
 344  *             used for DMA mappings
 345  * @portptr:   pointer to a local port pointer. Upon success, the routine
 346  *             will allocate a nvme_fc_local_port structure and place its
 347  *             address in the local port pointer. Upon failure, local port
 348  *             pointer will be set to 0.
 349  *
 350  * Returns:
 351  * a completion status. Must be 0 upon success; a negative errno
 352  * (ex: -ENXIO) upon failure.
 353  */
 354 int
 355 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
 356                         struct nvme_fc_port_template *template,
 357                         struct device *dev,
 358                         struct nvme_fc_local_port **portptr)
 359 {
 360         struct nvme_fc_lport *newrec;
 361         unsigned long flags;
 362         int ret, idx;
 363
 364         if (!template->localport_delete || !template->remoteport_delete ||
 365             !template->ls_req || !template->fcp_io ||
 366             !template->ls_abort || !template->fcp_abort ||
 367             !template->max_hw_queues || !template->max_sgl_segments ||
 368             !template->max_dif_sgl_segments || !template->dma_boundary) {
 369                 ret = -EINVAL;
 370                 goto out_reghost_failed;
 371         }
 372
 373         /*
 374          * look to see if there is already a localport that had been
 375          * deregistered and in the process of waiting for all the
 376          * references to fully be removed.  If the references haven't
 377          * expired, we can simply re-enable the localport. Remoteports
 378          * and controller reconnections should resume naturally.
 379          */
 380         newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
 381
 382         /* found an lport, but something about its state is bad */
 383         if (IS_ERR(newrec)) {
 384                 ret = PTR_ERR(newrec);
 385                 goto out_reghost_failed;
 386
 387         /* found existing lport, which was resumed */
 388         } else if (newrec) {
 389                 *portptr = &newrec->localport;
 390                 return 0;
 391         }
 392
 393         /* nothing found - allocate a new localport struct */
 394
 395         newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
 396                          GFP_KERNEL);
 397         if (!newrec) {
 398                 ret = -ENOMEM;
 399                 goto out_reghost_failed;
 400         }
 401
 402         idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
 403         if (idx < 0) {
 404                 ret = -ENOSPC;
 405                 goto out_fail_kfree;
 406         }
 407
 408         if (!get_device(dev) && dev) {
 409                 ret = -ENODEV;
 410                 goto out_ida_put;
 411         }
 412
 413         INIT_LIST_HEAD(&newrec->port_list);
 414         INIT_LIST_HEAD(&newrec->endp_list);
 415         kref_init(&newrec->ref);
 416         atomic_set(&newrec->act_rport_cnt, 0);
 417         newrec->ops = template;
 418         newrec->dev = dev;
 419         ida_init(&newrec->endp_cnt);
 420         if (template->local_priv_sz)
 421                 newrec->localport.private = &newrec[1];
 422         else
 423                 newrec->localport.private = NULL;
 424         newrec->localport.node_name = pinfo->node_name;
 425         newrec->localport.port_name = pinfo->port_name;
 426         newrec->localport.port_role = pinfo->port_role;
 427         newrec->localport.port_id = pinfo->port_id;
 428         newrec->localport.port_state = FC_OBJSTATE_ONLINE;
 429         newrec->localport.port_num = idx;
 430
 431         spin_lock_irqsave(&nvme_fc_lock, flags);
 432         list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
 433         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 434
 435         if (dev)
 436                 dma_set_seg_boundary(dev, template->dma_boundary);
 437
 438         *portptr = &newrec->localport;
 439         return 0;
 440
 441 out_ida_put:
 442         ida_free(&nvme_fc_local_port_cnt, idx);
 443 out_fail_kfree:
 444         kfree(newrec);
 445 out_reghost_failed:
 446         *portptr = NULL;
 447
 448         return ret;
 449 }
 450 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
 451
 452 /**
 453  * nvme_fc_unregister_localport - transport entry point called by an
 454  *                              LLDD to deregister/remove a previously
 455  *                              registered a NVME host FC port.
 456  * @portptr: pointer to the (registered) local port that is to be deregistered.
 457  *
 458  * Returns:
 459  * a completion status. Must be 0 upon success; a negative errno
 460  * (ex: -ENXIO) upon failure.
 461  */
 462 int
 463 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
 464 {
 465         struct nvme_fc_lport *lport = localport_to_lport(portptr);
 466         unsigned long flags;
 467
 468         if (!portptr)
 469                 return -EINVAL;
 470
 471         spin_lock_irqsave(&nvme_fc_lock, flags);
 472
 473         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 474                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 475                 return -EINVAL;
 476         }
 477         portptr->port_state = FC_OBJSTATE_DELETED;
 478
 479         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 480
 481         if (atomic_read(&lport->act_rport_cnt) == 0)
 482                 lport->ops->localport_delete(&lport->localport);
 483
 484         nvme_fc_lport_put(lport);
 485
 486         return 0;
 487 }
 488 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
 489
 490 /*
 491  * TRADDR strings, per FC-NVME are fixed format:
 492  *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
 493  * udev event will only differ by prefix of what field is
 494  * being specified:
 495  *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
 496  *  19 + 43 + null_fudge = 64 characters
 497  */
 498 #define FCNVME_TRADDR_LENGTH            64
 499
 500 static void
 501 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
 502                 struct nvme_fc_rport *rport)
 503 {
 504         char hostaddr[FCNVME_TRADDR_LENGTH];    /* NVMEFC_HOST_TRADDR=...*/
 505         char tgtaddr[FCNVME_TRADDR_LENGTH];     /* NVMEFC_TRADDR=...*/
 506         char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
 507
 508         if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
 509                 return;
 510
 511         snprintf(hostaddr, sizeof(hostaddr),
 512                 "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
 513                 lport->localport.node_name, lport->localport.port_name);
 514         snprintf(tgtaddr, sizeof(tgtaddr),
 515                 "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
 516                 rport->remoteport.node_name, rport->remoteport.port_name);
 517         kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
 518 }
 519
 520 static void
 521 nvme_fc_free_rport(struct kref *ref)
 522 {
 523         struct nvme_fc_rport *rport =
 524                 container_of(ref, struct nvme_fc_rport, ref);
 525         struct nvme_fc_lport *lport =
 526                         localport_to_lport(rport->remoteport.localport);
 527         unsigned long flags;
 528
 529         WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
 530         WARN_ON(!list_empty(&rport->ctrl_list));
 531
 532         /* remove from lport list */
 533         spin_lock_irqsave(&nvme_fc_lock, flags);
 534         list_del(&rport->endp_list);
 535         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 536
 537         WARN_ON(!list_empty(&rport->disc_list));
 538         ida_free(&lport->endp_cnt, rport->remoteport.port_num);
 539
 540         kfree(rport);
 541
 542         nvme_fc_lport_put(lport);
 543 }
 544
 545 static void
 546 nvme_fc_rport_put(struct nvme_fc_rport *rport)
 547 {
 548         kref_put(&rport->ref, nvme_fc_free_rport);
 549 }
 550
 551 static int
 552 nvme_fc_rport_get(struct nvme_fc_rport *rport)
 553 {
 554         return kref_get_unless_zero(&rport->ref);
 555 }
 556
 557 static void
 558 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
 559 {
 560         switch (ctrl->ctrl.state) {
 561         case NVME_CTRL_NEW:
 562         case NVME_CTRL_CONNECTING:
 563                 /*
 564                  * As all reconnects were suppressed, schedule a
 565                  * connect.
 566                  */
 567                 dev_info(ctrl->ctrl.device,
 568                         "NVME-FC{%d}: connectivity re-established. "
 569                         "Attempting reconnect\n", ctrl->cnum);
 570
 571                 queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
 572                 break;
 573
 574         case NVME_CTRL_RESETTING:
 575                 /*
 576                  * Controller is already in the process of terminating the
 577                  * association. No need to do anything further. The reconnect
 578                  * step will naturally occur after the reset completes.
 579                  */
 580                 break;
 581
 582         default:
 583                 /* no action to take - let it delete */
 584                 break;
 585         }
 586 }
 587
 588 static struct nvme_fc_rport *
 589 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
 590                                 struct nvme_fc_port_info *pinfo)
 591 {
 592         struct nvme_fc_rport *rport;
 593         struct nvme_fc_ctrl *ctrl;
 594         unsigned long flags;
 595
 596         spin_lock_irqsave(&nvme_fc_lock, flags);
 597
 598         list_for_each_entry(rport, &lport->endp_list, endp_list) {
 599                 if (rport->remoteport.node_name != pinfo->node_name ||
 600                     rport->remoteport.port_name != pinfo->port_name)
 601                         continue;
 602
 603                 if (!nvme_fc_rport_get(rport)) {
 604                         rport = ERR_PTR(-ENOLCK);
 605                         goto out_done;
 606                 }
 607
 608                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 609
 610                 spin_lock_irqsave(&rport->lock, flags);
 611
 612                 /* has it been unregistered */
 613                 if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
 614                         /* means lldd called us twice */
 615                         spin_unlock_irqrestore(&rport->lock, flags);
 616                         nvme_fc_rport_put(rport);
 617                         return ERR_PTR(-ESTALE);
 618                 }
 619
 620                 rport->remoteport.port_role = pinfo->port_role;
 621                 rport->remoteport.port_id = pinfo->port_id;
 622                 rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
 623                 rport->dev_loss_end = 0;
 624
 625                 /*
 626                  * kick off a reconnect attempt on all associations to the
 627                  * remote port. A successful reconnects will resume i/o.
 628                  */
 629                 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
 630                         nvme_fc_resume_controller(ctrl);
 631
 632                 spin_unlock_irqrestore(&rport->lock, flags);
 633
 634                 return rport;
 635         }
 636
 637         rport = NULL;
 638
 639 out_done:
 640         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 641
 642         return rport;
 643 }
 644
 645 static inline void
 646 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
 647                         struct nvme_fc_port_info *pinfo)
 648 {
 649         if (pinfo->dev_loss_tmo)
 650                 rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
 651         else
 652                 rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
 653 }
 654
 655 /**
 656  * nvme_fc_register_remoteport - transport entry point called by an
 657  *                              LLDD to register the existence of a NVME
 658  *                              subsystem FC port on its fabric.
 659  * @localport: pointer to the (registered) local port that the remote
 660  *             subsystem port is connected to.
 661  * @pinfo:     pointer to information about the port to be registered
 662  * @portptr:   pointer to a remote port pointer. Upon success, the routine
 663  *             will allocate a nvme_fc_remote_port structure and place its
 664  *             address in the remote port pointer. Upon failure, remote port
 665  *             pointer will be set to 0.
 666  *
 667  * Returns:
 668  * a completion status. Must be 0 upon success; a negative errno
 669  * (ex: -ENXIO) upon failure.
 670  */
 671 int
 672 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
 673                                 struct nvme_fc_port_info *pinfo,
 674                                 struct nvme_fc_remote_port **portptr)
 675 {
 676         struct nvme_fc_lport *lport = localport_to_lport(localport);
 677         struct nvme_fc_rport *newrec;
 678         unsigned long flags;
 679         int ret, idx;
 680
 681         if (!nvme_fc_lport_get(lport)) {
 682                 ret = -ESHUTDOWN;
 683                 goto out_reghost_failed;
 684         }
 685
 686         /*
 687          * look to see if there is already a remoteport that is waiting
 688          * for a reconnect (within dev_loss_tmo) with the same WWN's.
 689          * If so, transition to it and reconnect.
 690          */
 691         newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
 692
 693         /* found an rport, but something about its state is bad */
 694         if (IS_ERR(newrec)) {
 695                 ret = PTR_ERR(newrec);
 696                 goto out_lport_put;
 697
 698         /* found existing rport, which was resumed */
 699         } else if (newrec) {
 700                 nvme_fc_lport_put(lport);
 701                 __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 702                 nvme_fc_signal_discovery_scan(lport, newrec);
 703                 *portptr = &newrec->remoteport;
 704                 return 0;
 705         }
 706
 707         /* nothing found - allocate a new remoteport struct */
 708
 709         newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
 710                          GFP_KERNEL);
 711         if (!newrec) {
 712                 ret = -ENOMEM;
 713                 goto out_lport_put;
 714         }
 715
 716         idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
 717         if (idx < 0) {
 718                 ret = -ENOSPC;
 719                 goto out_kfree_rport;
 720         }
 721
 722         INIT_LIST_HEAD(&newrec->endp_list);
 723         INIT_LIST_HEAD(&newrec->ctrl_list);
 724         INIT_LIST_HEAD(&newrec->ls_req_list);
 725         INIT_LIST_HEAD(&newrec->disc_list);
 726         kref_init(&newrec->ref);
 727         atomic_set(&newrec->act_ctrl_cnt, 0);
 728         spin_lock_init(&newrec->lock);
 729         newrec->remoteport.localport = &lport->localport;
 730         INIT_LIST_HEAD(&newrec->ls_rcv_list);
 731         newrec->dev = lport->dev;
 732         newrec->lport = lport;
 733         if (lport->ops->remote_priv_sz)
 734                 newrec->remoteport.private = &newrec[1];
 735         else
 736                 newrec->remoteport.private = NULL;
 737         newrec->remoteport.port_role = pinfo->port_role;
 738         newrec->remoteport.node_name = pinfo->node_name;
 739         newrec->remoteport.port_name = pinfo->port_name;
 740         newrec->remoteport.port_id = pinfo->port_id;
 741         newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
 742         newrec->remoteport.port_num = idx;
 743         __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 744         INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
 745
 746         spin_lock_irqsave(&nvme_fc_lock, flags);
 747         list_add_tail(&newrec->endp_list, &lport->endp_list);
 748         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 749
 750         nvme_fc_signal_discovery_scan(lport, newrec);
 751
 752         *portptr = &newrec->remoteport;
 753         return 0;
 754
 755 out_kfree_rport:
 756         kfree(newrec);
 757 out_lport_put:
 758         nvme_fc_lport_put(lport);
 759 out_reghost_failed:
 760         *portptr = NULL;
 761         return ret;
 762 }
 763 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
 764
 765 static int
 766 nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
 767 {
 768         struct nvmefc_ls_req_op *lsop;
 769         unsigned long flags;
 770
 771 restart:
 772         spin_lock_irqsave(&rport->lock, flags);
 773
 774         list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
 775                 if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
 776                         lsop->flags |= FCOP_FLAGS_TERMIO;
 777                         spin_unlock_irqrestore(&rport->lock, flags);
 778                         rport->lport->ops->ls_abort(&rport->lport->localport,
 779                                                 &rport->remoteport,
 780                                                 &lsop->ls_req);
 781                         goto restart;
 782                 }
 783         }
 784         spin_unlock_irqrestore(&rport->lock, flags);
 785
 786         return 0;
 787 }
 788
 789 static void
 790 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
 791 {
 792         dev_info(ctrl->ctrl.device,
 793                 "NVME-FC{%d}: controller connectivity lost. Awaiting "
 794                 "Reconnect", ctrl->cnum);
 795
 796         switch (ctrl->ctrl.state) {
 797         case NVME_CTRL_NEW:
 798         case NVME_CTRL_LIVE:
 799                 /*
 800                  * Schedule a controller reset. The reset will terminate the
 801                  * association and schedule the reconnect timer.  Reconnects
 802                  * will be attempted until either the ctlr_loss_tmo
 803                  * (max_retries * connect_delay) expires or the remoteport's
 804                  * dev_loss_tmo expires.
 805                  */
 806                 if (nvme_reset_ctrl(&ctrl->ctrl)) {
 807                         dev_warn(ctrl->ctrl.device,
 808                                 "NVME-FC{%d}: Couldn't schedule reset.\n",
 809                                 ctrl->cnum);
 810                         nvme_delete_ctrl(&ctrl->ctrl);
 811                 }
 812                 break;
 813
 814         case NVME_CTRL_CONNECTING:
 815                 /*
 816                  * The association has already been terminated and the
 817                  * controller is attempting reconnects.  No need to do anything
 818                  * futher.  Reconnects will be attempted until either the
 819                  * ctlr_loss_tmo (max_retries * connect_delay) expires or the
 820                  * remoteport's dev_loss_tmo expires.
 821                  */
 822                 break;
 823
 824         case NVME_CTRL_RESETTING:
 825                 /*
 826                  * Controller is already in the process of terminating the
 827                  * association.  No need to do anything further. The reconnect
 828                  * step will kick in naturally after the association is
 829                  * terminated.
 830                  */
 831                 break;
 832
 833         case NVME_CTRL_DELETING:
 834         case NVME_CTRL_DELETING_NOIO:
 835         default:
 836                 /* no action to take - let it delete */
 837                 break;
 838         }
 839 }
 840
 841 /**
 842  * nvme_fc_unregister_remoteport - transport entry point called by an
 843  *                              LLDD to deregister/remove a previously
 844  *                              registered a NVME subsystem FC port.
 845  * @portptr: pointer to the (registered) remote port that is to be
 846  *           deregistered.
 847  *
 848  * Returns:
 849  * a completion status. Must be 0 upon success; a negative errno
 850  * (ex: -ENXIO) upon failure.
 851  */
 852 int
 853 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
 854 {
 855         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 856         struct nvme_fc_ctrl *ctrl;
 857         unsigned long flags;
 858
 859         if (!portptr)
 860                 return -EINVAL;
 861
 862         spin_lock_irqsave(&rport->lock, flags);
 863
 864         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 865                 spin_unlock_irqrestore(&rport->lock, flags);
 866                 return -EINVAL;
 867         }
 868         portptr->port_state = FC_OBJSTATE_DELETED;
 869
 870         rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
 871
 872         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
 873                 /* if dev_loss_tmo==0, dev loss is immediate */
 874                 if (!portptr->dev_loss_tmo) {
 875                         dev_warn(ctrl->ctrl.device,
 876                                 "NVME-FC{%d}: controller connectivity lost.\n",
 877                                 ctrl->cnum);
 878                         nvme_delete_ctrl(&ctrl->ctrl);
 879                 } else
 880                         nvme_fc_ctrl_connectivity_loss(ctrl);
 881         }
 882
 883         spin_unlock_irqrestore(&rport->lock, flags);
 884
 885         nvme_fc_abort_lsops(rport);
 886
 887         if (atomic_read(&rport->act_ctrl_cnt) == 0)
 888                 rport->lport->ops->remoteport_delete(portptr);
 889
 890         /*
 891          * release the reference, which will allow, if all controllers
 892          * go away, which should only occur after dev_loss_tmo occurs,
 893          * for the rport to be torn down.
 894          */
 895         nvme_fc_rport_put(rport);
 896
 897         return 0;
 898 }
 899 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
 900
 901 /**
 902  * nvme_fc_rescan_remoteport - transport entry point called by an
 903  *                              LLDD to request a nvme device rescan.
 904  * @remoteport: pointer to the (registered) remote port that is to be
 905  *              rescanned.
 906  *
 907  * Returns: N/A
 908  */
 909 void
 910 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
 911 {
 912         struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
 913
 914         nvme_fc_signal_discovery_scan(rport->lport, rport);
 915 }
 916 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
 917
 918 int
 919 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
 920                         u32 dev_loss_tmo)
 921 {
 922         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 923         unsigned long flags;
 924
 925         spin_lock_irqsave(&rport->lock, flags);
 926
 927         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 928                 spin_unlock_irqrestore(&rport->lock, flags);
 929                 return -EINVAL;
 930         }
 931
 932         /* a dev_loss_tmo of 0 (immediate) is allowed to be set */
 933         rport->remoteport.dev_loss_tmo = dev_loss_tmo;
 934
 935         spin_unlock_irqrestore(&rport->lock, flags);
 936
 937         return 0;
 938 }
 939 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
 940
 941
 942 /* *********************** FC-NVME DMA Handling **************************** */
 943
 944 /*
 945  * The fcloop device passes in a NULL device pointer. Real LLD's will
 946  * pass in a valid device pointer. If NULL is passed to the dma mapping
 947  * routines, depending on the platform, it may or may not succeed, and
 948  * may crash.
 949  *
 950  * As such:
 951  * Wrapper all the dma routines and check the dev pointer.
 952  *
 953  * If simple mappings (return just a dma address, we'll noop them,
 954  * returning a dma address of 0.
 955  *
 956  * On more complex mappings (dma_map_sg), a pseudo routine fills
 957  * in the scatter list, setting all dma addresses to 0.
 958  */
 959
 960 static inline dma_addr_t
 961 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
 962                 enum dma_data_direction dir)
 963 {
 964         return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
 965 }
 966
 967 static inline int
 968 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 969 {
 970         return dev ? dma_mapping_error(dev, dma_addr) : 0;
 971 }
 972
 973 static inline void
 974 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
 975         enum dma_data_direction dir)
 976 {
 977         if (dev)
 978                 dma_unmap_single(dev, addr, size, dir);
 979 }
 980
 981 static inline void
 982 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 983                 enum dma_data_direction dir)
 984 {
 985         if (dev)
 986                 dma_sync_single_for_cpu(dev, addr, size, dir);
 987 }
 988
 989 static inline void
 990 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
 991                 enum dma_data_direction dir)
 992 {
 993         if (dev)
 994                 dma_sync_single_for_device(dev, addr, size, dir);
 995 }
 996
 997 /* pseudo dma_map_sg call */
 998 static int
 999 fc_map_sg(struct scatterlist *sg, int nents)
1000 {
1001         struct scatterlist *s;
1002         int i;
1003
1004         WARN_ON(nents == 0 || sg[0].length == 0);
1005
1006         for_each_sg(sg, s, nents, i) {
1007                 s->dma_address = 0L;
1008 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1009                 s->dma_length = s->length;
1010 #endif
1011         }
1012         return nents;
1013 }
1014
1015 static inline int
1016 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1017                 enum dma_data_direction dir)
1018 {
1019         return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1020 }
1021
1022 static inline void
1023 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1024                 enum dma_data_direction dir)
1025 {
1026         if (dev)
1027                 dma_unmap_sg(dev, sg, nents, dir);
1028 }
1029
1030 /* *********************** FC-NVME LS Handling **************************** */
1031
1032 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1033 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1034
1035 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1036
1037 static void
1038 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1039 {
1040         struct nvme_fc_rport *rport = lsop->rport;
1041         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1042         unsigned long flags;
1043
1044         spin_lock_irqsave(&rport->lock, flags);
1045
1046         if (!lsop->req_queued) {
1047                 spin_unlock_irqrestore(&rport->lock, flags);
1048                 return;
1049         }
1050
1051         list_del(&lsop->lsreq_list);
1052
1053         lsop->req_queued = false;
1054
1055         spin_unlock_irqrestore(&rport->lock, flags);
1056
1057         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1058                                   (lsreq->rqstlen + lsreq->rsplen),
1059                                   DMA_BIDIRECTIONAL);
1060
1061         nvme_fc_rport_put(rport);
1062 }
1063
1064 static int
1065 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1066                 struct nvmefc_ls_req_op *lsop,
1067                 void (*done)(struct nvmefc_ls_req *req, int status))
1068 {
1069         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1070         unsigned long flags;
1071         int ret = 0;
1072
1073         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1074                 return -ECONNREFUSED;
1075
1076         if (!nvme_fc_rport_get(rport))
1077                 return -ESHUTDOWN;
1078
1079         lsreq->done = done;
1080         lsop->rport = rport;
1081         lsop->req_queued = false;
1082         INIT_LIST_HEAD(&lsop->lsreq_list);
1083         init_completion(&lsop->ls_done);
1084
1085         lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1086                                   lsreq->rqstlen + lsreq->rsplen,
1087                                   DMA_BIDIRECTIONAL);
1088         if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1089                 ret = -EFAULT;
1090                 goto out_putrport;
1091         }
1092         lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1093
1094         spin_lock_irqsave(&rport->lock, flags);
1095
1096         list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1097
1098         lsop->req_queued = true;
1099
1100         spin_unlock_irqrestore(&rport->lock, flags);
1101
1102         ret = rport->lport->ops->ls_req(&rport->lport->localport,
1103                                         &rport->remoteport, lsreq);
1104         if (ret)
1105                 goto out_unlink;
1106
1107         return 0;
1108
1109 out_unlink:
1110         lsop->ls_error = ret;
1111         spin_lock_irqsave(&rport->lock, flags);
1112         lsop->req_queued = false;
1113         list_del(&lsop->lsreq_list);
1114         spin_unlock_irqrestore(&rport->lock, flags);
1115         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1116                                   (lsreq->rqstlen + lsreq->rsplen),
1117                                   DMA_BIDIRECTIONAL);
1118 out_putrport:
1119         nvme_fc_rport_put(rport);
1120
1121         return ret;
1122 }
1123
1124 static void
1125 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1126 {
1127         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1128
1129         lsop->ls_error = status;
1130         complete(&lsop->ls_done);
1131 }
1132
1133 static int
1134 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1135 {
1136         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1137         struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1138         int ret;
1139
1140         ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1141
1142         if (!ret) {
1143                 /*
1144                  * No timeout/not interruptible as we need the struct
1145                  * to exist until the lldd calls us back. Thus mandate
1146                  * wait until driver calls back. lldd responsible for
1147                  * the timeout action
1148                  */
1149                 wait_for_completion(&lsop->ls_done);
1150
1151                 __nvme_fc_finish_ls_req(lsop);
1152
1153                 ret = lsop->ls_error;
1154         }
1155
1156         if (ret)
1157                 return ret;
1158
1159         /* ACC or RJT payload ? */
1160         if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1161                 return -ENXIO;
1162
1163         return 0;
1164 }
1165
1166 static int
1167 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1168                 struct nvmefc_ls_req_op *lsop,
1169                 void (*done)(struct nvmefc_ls_req *req, int status))
1170 {
1171         /* don't wait for completion */
1172
1173         return __nvme_fc_send_ls_req(rport, lsop, done);
1174 }
1175
1176 static int
1177 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1178         struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1179 {
1180         struct nvmefc_ls_req_op *lsop;
1181         struct nvmefc_ls_req *lsreq;
1182         struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1183         struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1184         unsigned long flags;
1185         int ret, fcret = 0;
1186
1187         lsop = kzalloc((sizeof(*lsop) +
1188                          sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1189                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1190         if (!lsop) {
1191                 dev_info(ctrl->ctrl.device,
1192                         "NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1193                         ctrl->cnum);
1194                 ret = -ENOMEM;
1195                 goto out_no_memory;
1196         }
1197
1198         assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1199         assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1200         lsreq = &lsop->ls_req;
1201         if (ctrl->lport->ops->lsrqst_priv_sz)
1202                 lsreq->private = &assoc_acc[1];
1203         else
1204                 lsreq->private = NULL;
1205
1206         assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1207         assoc_rqst->desc_list_len =
1208                         cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1209
1210         assoc_rqst->assoc_cmd.desc_tag =
1211                         cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1212         assoc_rqst->assoc_cmd.desc_len =
1213                         fcnvme_lsdesc_len(
1214                                 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1215
1216         assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1217         assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1218         /* Linux supports only Dynamic controllers */
1219         assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1220         uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1221         strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1222                 min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE));
1223         strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1224                 min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE));
1225
1226         lsop->queue = queue;
1227         lsreq->rqstaddr = assoc_rqst;
1228         lsreq->rqstlen = sizeof(*assoc_rqst);
1229         lsreq->rspaddr = assoc_acc;
1230         lsreq->rsplen = sizeof(*assoc_acc);
1231         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1232
1233         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1234         if (ret)
1235                 goto out_free_buffer;
1236
1237         /* process connect LS completion */
1238
1239         /* validate the ACC response */
1240         if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1241                 fcret = VERR_LSACC;
1242         else if (assoc_acc->hdr.desc_list_len !=
1243                         fcnvme_lsdesc_len(
1244                                 sizeof(struct fcnvme_ls_cr_assoc_acc)))
1245                 fcret = VERR_CR_ASSOC_ACC_LEN;
1246         else if (assoc_acc->hdr.rqst.desc_tag !=
1247                         cpu_to_be32(FCNVME_LSDESC_RQST))
1248                 fcret = VERR_LSDESC_RQST;
1249         else if (assoc_acc->hdr.rqst.desc_len !=
1250                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1251                 fcret = VERR_LSDESC_RQST_LEN;
1252         else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1253                 fcret = VERR_CR_ASSOC;
1254         else if (assoc_acc->associd.desc_tag !=
1255                         cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1256                 fcret = VERR_ASSOC_ID;
1257         else if (assoc_acc->associd.desc_len !=
1258                         fcnvme_lsdesc_len(
1259                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1260                 fcret = VERR_ASSOC_ID_LEN;
1261         else if (assoc_acc->connectid.desc_tag !=
1262                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1263                 fcret = VERR_CONN_ID;
1264         else if (assoc_acc->connectid.desc_len !=
1265                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1266                 fcret = VERR_CONN_ID_LEN;
1267
1268         if (fcret) {
1269                 ret = -EBADF;
1270                 dev_err(ctrl->dev,
1271                         "q %d Create Association LS failed: %s\n",
1272                         queue->qnum, validation_errors[fcret]);
1273         } else {
1274                 spin_lock_irqsave(&ctrl->lock, flags);
1275                 ctrl->association_id =
1276                         be64_to_cpu(assoc_acc->associd.association_id);
1277                 queue->connection_id =
1278                         be64_to_cpu(assoc_acc->connectid.connection_id);
1279                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1280                 spin_unlock_irqrestore(&ctrl->lock, flags);
1281         }
1282
1283 out_free_buffer:
1284         kfree(lsop);
1285 out_no_memory:
1286         if (ret)
1287                 dev_err(ctrl->dev,
1288                         "queue %d connect admin queue failed (%d).\n",
1289                         queue->qnum, ret);
1290         return ret;
1291 }
1292
1293 static int
1294 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1295                         u16 qsize, u16 ersp_ratio)
1296 {
1297         struct nvmefc_ls_req_op *lsop;
1298         struct nvmefc_ls_req *lsreq;
1299         struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1300         struct fcnvme_ls_cr_conn_acc *conn_acc;
1301         int ret, fcret = 0;
1302
1303         lsop = kzalloc((sizeof(*lsop) +
1304                          sizeof(*conn_rqst) + sizeof(*conn_acc) +
1305                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1306         if (!lsop) {
1307                 dev_info(ctrl->ctrl.device,
1308                         "NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1309                         ctrl->cnum);
1310                 ret = -ENOMEM;
1311                 goto out_no_memory;
1312         }
1313
1314         conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1315         conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1316         lsreq = &lsop->ls_req;
1317         if (ctrl->lport->ops->lsrqst_priv_sz)
1318                 lsreq->private = (void *)&conn_acc[1];
1319         else
1320                 lsreq->private = NULL;
1321
1322         conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1323         conn_rqst->desc_list_len = cpu_to_be32(
1324                                 sizeof(struct fcnvme_lsdesc_assoc_id) +
1325                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1326
1327         conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1328         conn_rqst->associd.desc_len =
1329                         fcnvme_lsdesc_len(
1330                                 sizeof(struct fcnvme_lsdesc_assoc_id));
1331         conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1332         conn_rqst->connect_cmd.desc_tag =
1333                         cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1334         conn_rqst->connect_cmd.desc_len =
1335                         fcnvme_lsdesc_len(
1336                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1337         conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1338         conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1339         conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1340
1341         lsop->queue = queue;
1342         lsreq->rqstaddr = conn_rqst;
1343         lsreq->rqstlen = sizeof(*conn_rqst);
1344         lsreq->rspaddr = conn_acc;
1345         lsreq->rsplen = sizeof(*conn_acc);
1346         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1347
1348         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1349         if (ret)
1350                 goto out_free_buffer;
1351
1352         /* process connect LS completion */
1353
1354         /* validate the ACC response */
1355         if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1356                 fcret = VERR_LSACC;
1357         else if (conn_acc->hdr.desc_list_len !=
1358                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1359                 fcret = VERR_CR_CONN_ACC_LEN;
1360         else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1361                 fcret = VERR_LSDESC_RQST;
1362         else if (conn_acc->hdr.rqst.desc_len !=
1363                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1364                 fcret = VERR_LSDESC_RQST_LEN;
1365         else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1366                 fcret = VERR_CR_CONN;
1367         else if (conn_acc->connectid.desc_tag !=
1368                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1369                 fcret = VERR_CONN_ID;
1370         else if (conn_acc->connectid.desc_len !=
1371                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1372                 fcret = VERR_CONN_ID_LEN;
1373
1374         if (fcret) {
1375                 ret = -EBADF;
1376                 dev_err(ctrl->dev,
1377                         "q %d Create I/O Connection LS failed: %s\n",
1378                         queue->qnum, validation_errors[fcret]);
1379         } else {
1380                 queue->connection_id =
1381                         be64_to_cpu(conn_acc->connectid.connection_id);
1382                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1383         }
1384
1385 out_free_buffer:
1386         kfree(lsop);
1387 out_no_memory:
1388         if (ret)
1389                 dev_err(ctrl->dev,
1390                         "queue %d connect I/O queue failed (%d).\n",
1391                         queue->qnum, ret);
1392         return ret;
1393 }
1394
1395 static void
1396 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1397 {
1398         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1399
1400         __nvme_fc_finish_ls_req(lsop);
1401
1402         /* fc-nvme initiator doesn't care about success or failure of cmd */
1403
1404         kfree(lsop);
1405 }
1406
1407 /*
1408  * This routine sends a FC-NVME LS to disconnect (aka terminate)
1409  * the FC-NVME Association.  Terminating the association also
1410  * terminates the FC-NVME connections (per queue, both admin and io
1411  * queues) that are part of the association. E.g. things are torn
1412  * down, and the related FC-NVME Association ID and Connection IDs
1413  * become invalid.
1414  *
1415  * The behavior of the fc-nvme initiator is such that it's
1416  * understanding of the association and connections will implicitly
1417  * be torn down. The action is implicit as it may be due to a loss of
1418  * connectivity with the fc-nvme target, so you may never get a
1419  * response even if you tried.  As such, the action of this routine
1420  * is to asynchronously send the LS, ignore any results of the LS, and
1421  * continue on with terminating the association. If the fc-nvme target
1422  * is present and receives the LS, it too can tear down.
1423  */
1424 static void
1425 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1426 {
1427         struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1428         struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1429         struct nvmefc_ls_req_op *lsop;
1430         struct nvmefc_ls_req *lsreq;
1431         int ret;
1432
1433         lsop = kzalloc((sizeof(*lsop) +
1434                         sizeof(*discon_rqst) + sizeof(*discon_acc) +
1435                         ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1436         if (!lsop) {
1437                 dev_info(ctrl->ctrl.device,
1438                         "NVME-FC{%d}: send Disconnect Association "
1439                         "failed: ENOMEM\n",
1440                         ctrl->cnum);
1441                 return;
1442         }
1443
1444         discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1445         discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1446         lsreq = &lsop->ls_req;
1447         if (ctrl->lport->ops->lsrqst_priv_sz)
1448                 lsreq->private = (void *)&discon_acc[1];
1449         else
1450                 lsreq->private = NULL;
1451
1452         nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1453                                 ctrl->association_id);
1454
1455         ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1456                                 nvme_fc_disconnect_assoc_done);
1457         if (ret)
1458                 kfree(lsop);
1459 }
1460
1461 static void
1462 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1463 {
1464         struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1465         struct nvme_fc_rport *rport = lsop->rport;
1466         struct nvme_fc_lport *lport = rport->lport;
1467         unsigned long flags;
1468
1469         spin_lock_irqsave(&rport->lock, flags);
1470         list_del(&lsop->lsrcv_list);
1471         spin_unlock_irqrestore(&rport->lock, flags);
1472
1473         fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1474                                 sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1475         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1476                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1477
1478         kfree(lsop->rspbuf);
1479         kfree(lsop->rqstbuf);
1480         kfree(lsop);
1481
1482         nvme_fc_rport_put(rport);
1483 }
1484
1485 static void
1486 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1487 {
1488         struct nvme_fc_rport *rport = lsop->rport;
1489         struct nvme_fc_lport *lport = rport->lport;
1490         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1491         int ret;
1492
1493         fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1494                                   sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1495
1496         ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1497                                      lsop->lsrsp);
1498         if (ret) {
1499                 dev_warn(lport->dev,
1500                         "LLDD rejected LS RSP xmt: LS %d status %d\n",
1501                         w0->ls_cmd, ret);
1502                 nvme_fc_xmt_ls_rsp_done(lsop->lsrsp);
1503                 return;
1504         }
1505 }
1506
1507 static struct nvme_fc_ctrl *
1508 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1509                       struct nvmefc_ls_rcv_op *lsop)
1510 {
1511         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1512                                         &lsop->rqstbuf->rq_dis_assoc;
1513         struct nvme_fc_ctrl *ctrl, *ret = NULL;
1514         struct nvmefc_ls_rcv_op *oldls = NULL;
1515         u64 association_id = be64_to_cpu(rqst->associd.association_id);
1516         unsigned long flags;
1517
1518         spin_lock_irqsave(&rport->lock, flags);
1519
1520         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1521                 if (!nvme_fc_ctrl_get(ctrl))
1522                         continue;
1523                 spin_lock(&ctrl->lock);
1524                 if (association_id == ctrl->association_id) {
1525                         oldls = ctrl->rcv_disconn;
1526                         ctrl->rcv_disconn = lsop;
1527                         ret = ctrl;
1528                 }
1529                 spin_unlock(&ctrl->lock);
1530                 if (ret)
1531                         /* leave the ctrl get reference */
1532                         break;
1533                 nvme_fc_ctrl_put(ctrl);
1534         }
1535
1536         spin_unlock_irqrestore(&rport->lock, flags);
1537
1538         /* transmit a response for anything that was pending */
1539         if (oldls) {
1540                 dev_info(rport->lport->dev,
1541                         "NVME-FC{%d}: Multiple Disconnect Association "
1542                         "LS's received\n", ctrl->cnum);
1543                 /* overwrite good response with bogus failure */
1544                 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1545                                                 sizeof(*oldls->rspbuf),
1546                                                 rqst->w0.ls_cmd,
1547                                                 FCNVME_RJT_RC_UNAB,
1548                                                 FCNVME_RJT_EXP_NONE, 0);
1549                 nvme_fc_xmt_ls_rsp(oldls);
1550         }
1551
1552         return ret;
1553 }
1554
1555 /*
1556  * returns true to mean LS handled and ls_rsp can be sent
1557  * returns false to defer ls_rsp xmt (will be done as part of
1558  *     association termination)
1559  */
1560 static bool
1561 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1562 {
1563         struct nvme_fc_rport *rport = lsop->rport;
1564         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1565                                         &lsop->rqstbuf->rq_dis_assoc;
1566         struct fcnvme_ls_disconnect_assoc_acc *acc =
1567                                         &lsop->rspbuf->rsp_dis_assoc;
1568         struct nvme_fc_ctrl *ctrl = NULL;
1569         int ret = 0;
1570
1571         memset(acc, 0, sizeof(*acc));
1572
1573         ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1574         if (!ret) {
1575                 /* match an active association */
1576                 ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1577                 if (!ctrl)
1578                         ret = VERR_NO_ASSOC;
1579         }
1580
1581         if (ret) {
1582                 dev_info(rport->lport->dev,
1583                         "Disconnect LS failed: %s\n",
1584                         validation_errors[ret]);
1585                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1586                                         sizeof(*acc), rqst->w0.ls_cmd,
1587                                         (ret == VERR_NO_ASSOC) ?
1588                                                 FCNVME_RJT_RC_INV_ASSOC :
1589                                                 FCNVME_RJT_RC_LOGIC,
1590                                         FCNVME_RJT_EXP_NONE, 0);
1591                 return true;
1592         }
1593
1594         /* format an ACCept response */
1595
1596         lsop->lsrsp->rsplen = sizeof(*acc);
1597
1598         nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1599                         fcnvme_lsdesc_len(
1600                                 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1601                         FCNVME_LS_DISCONNECT_ASSOC);
1602
1603         /*
1604          * the transmit of the response will occur after the exchanges
1605          * for the association have been ABTS'd by
1606          * nvme_fc_delete_association().
1607          */
1608
1609         /* fail the association */
1610         nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1611
1612         /* release the reference taken by nvme_fc_match_disconn_ls() */
1613         nvme_fc_ctrl_put(ctrl);
1614
1615         return false;
1616 }
1617
1618 /*
1619  * Actual Processing routine for received FC-NVME LS Requests from the LLD
1620  * returns true if a response should be sent afterward, false if rsp will
1621  * be sent asynchronously.
1622  */
1623 static bool
1624 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1625 {
1626         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1627         bool ret = true;
1628
1629         lsop->lsrsp->nvme_fc_private = lsop;
1630         lsop->lsrsp->rspbuf = lsop->rspbuf;
1631         lsop->lsrsp->rspdma = lsop->rspdma;
1632         lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1633         /* Be preventative. handlers will later set to valid length */
1634         lsop->lsrsp->rsplen = 0;
1635
1636         /*
1637          * handlers:
1638          *   parse request input, execute the request, and format the
1639          *   LS response
1640          */
1641         switch (w0->ls_cmd) {
1642         case FCNVME_LS_DISCONNECT_ASSOC:
1643                 ret = nvme_fc_ls_disconnect_assoc(lsop);
1644                 break;
1645         case FCNVME_LS_DISCONNECT_CONN:
1646                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1647                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1648                                 FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1649                 break;
1650         case FCNVME_LS_CREATE_ASSOCIATION:
1651         case FCNVME_LS_CREATE_CONNECTION:
1652                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1653                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1654                                 FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1655                 break;
1656         default:
1657                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1658                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1659                                 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1660                 break;
1661         }
1662
1663         return(ret);
1664 }
1665
1666 static void
1667 nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1668 {
1669         struct nvme_fc_rport *rport =
1670                 container_of(work, struct nvme_fc_rport, lsrcv_work);
1671         struct fcnvme_ls_rqst_w0 *w0;
1672         struct nvmefc_ls_rcv_op *lsop;
1673         unsigned long flags;
1674         bool sendrsp;
1675
1676 restart:
1677         sendrsp = true;
1678         spin_lock_irqsave(&rport->lock, flags);
1679         list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1680                 if (lsop->handled)
1681                         continue;
1682
1683                 lsop->handled = true;
1684                 if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1685                         spin_unlock_irqrestore(&rport->lock, flags);
1686                         sendrsp = nvme_fc_handle_ls_rqst(lsop);
1687                 } else {
1688                         spin_unlock_irqrestore(&rport->lock, flags);
1689                         w0 = &lsop->rqstbuf->w0;
1690                         lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1691                                                 lsop->rspbuf,
1692                                                 sizeof(*lsop->rspbuf),
1693                                                 w0->ls_cmd,
1694                                                 FCNVME_RJT_RC_UNAB,
1695                                                 FCNVME_RJT_EXP_NONE, 0);
1696                 }
1697                 if (sendrsp)
1698                         nvme_fc_xmt_ls_rsp(lsop);
1699                 goto restart;
1700         }
1701         spin_unlock_irqrestore(&rport->lock, flags);
1702 }
1703
1704 static
1705 void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1706                                 struct fcnvme_ls_rqst_w0 *w0)
1707 {
1708         dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1709                 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1710                         nvmefc_ls_names[w0->ls_cmd] : "");
1711 }
1712
1713 /**
1714  * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1715  *                       upon the reception of a NVME LS request.
1716  *
1717  * The nvme-fc layer will copy payload to an internal structure for
1718  * processing.  As such, upon completion of the routine, the LLDD may
1719  * immediately free/reuse the LS request buffer passed in the call.
1720  *
1721  * If this routine returns error, the LLDD should abort the exchange.
1722  *
1723  * @portptr:    pointer to the (registered) remote port that the LS
1724  *              was received from. The remoteport is associated with
1725  *              a specific localport.
1726  * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1727  *              used to reference the exchange corresponding to the LS
1728  *              when issuing an ls response.
1729  * @lsreqbuf:   pointer to the buffer containing the LS Request
1730  * @lsreqbuf_len: length, in bytes, of the received LS request
1731  */
1732 int
1733 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1734                         struct nvmefc_ls_rsp *lsrsp,
1735                         void *lsreqbuf, u32 lsreqbuf_len)
1736 {
1737         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1738         struct nvme_fc_lport *lport = rport->lport;
1739         struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1740         struct nvmefc_ls_rcv_op *lsop;
1741         unsigned long flags;
1742         int ret;
1743
1744         nvme_fc_rport_get(rport);
1745
1746         /* validate there's a routine to transmit a response */
1747         if (!lport->ops->xmt_ls_rsp) {
1748                 dev_info(lport->dev,
1749                         "RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1750                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1751                                 nvmefc_ls_names[w0->ls_cmd] : "");
1752                 ret = -EINVAL;
1753                 goto out_put;
1754         }
1755
1756         if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1757                 dev_info(lport->dev,
1758                         "RCV %s LS failed: payload too large\n",
1759                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1760                                 nvmefc_ls_names[w0->ls_cmd] : "");
1761                 ret = -E2BIG;
1762                 goto out_put;
1763         }
1764
1765         lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1766         if (!lsop) {
1767                 nvme_fc_rcv_ls_req_err_msg(lport, w0);
1768                 ret = -ENOMEM;
1769                 goto out_put;
1770         }
1771
1772         lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1773         lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1774         if (!lsop->rqstbuf || !lsop->rspbuf) {
1775                 nvme_fc_rcv_ls_req_err_msg(lport, w0);
1776                 ret = -ENOMEM;
1777                 goto out_free;
1778         }
1779
1780         lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1781                                         sizeof(*lsop->rspbuf),
1782                                         DMA_TO_DEVICE);
1783         if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1784                 dev_info(lport->dev,
1785                         "RCV %s LS failed: DMA mapping failure\n",
1786                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1787                                 nvmefc_ls_names[w0->ls_cmd] : "");
1788                 ret = -EFAULT;
1789                 goto out_free;
1790         }
1791
1792         lsop->rport = rport;
1793         lsop->lsrsp = lsrsp;
1794
1795         memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1796         lsop->rqstdatalen = lsreqbuf_len;
1797
1798         spin_lock_irqsave(&rport->lock, flags);
1799         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1800                 spin_unlock_irqrestore(&rport->lock, flags);
1801                 ret = -ENOTCONN;
1802                 goto out_unmap;
1803         }
1804         list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1805         spin_unlock_irqrestore(&rport->lock, flags);
1806
1807         schedule_work(&rport->lsrcv_work);
1808
1809         return 0;
1810
1811 out_unmap:
1812         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1813                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1814 out_free:
1815         kfree(lsop->rspbuf);
1816         kfree(lsop->rqstbuf);
1817         kfree(lsop);
1818 out_put:
1819         nvme_fc_rport_put(rport);
1820         return ret;
1821 }
1822 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1823
1824
1825 /* *********************** NVME Ctrl Routines **************************** */
1826
1827 static void
1828 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1829                 struct nvme_fc_fcp_op *op)
1830 {
1831         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1832                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1833         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1834                                 sizeof(op->cmd_iu), DMA_TO_DEVICE);
1835
1836         atomic_set(&op->state, FCPOP_STATE_UNINIT);
1837 }
1838
1839 static void
1840 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1841                 unsigned int hctx_idx)
1842 {
1843         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1844
1845         return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op);
1846 }
1847
1848 static int
1849 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1850 {
1851         unsigned long flags;
1852         int opstate;
1853
1854         spin_lock_irqsave(&ctrl->lock, flags);
1855         opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1856         if (opstate != FCPOP_STATE_ACTIVE)
1857                 atomic_set(&op->state, opstate);
1858         else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1859                 op->flags |= FCOP_FLAGS_TERMIO;
1860                 ctrl->iocnt++;
1861         }
1862         spin_unlock_irqrestore(&ctrl->lock, flags);
1863
1864         if (opstate != FCPOP_STATE_ACTIVE)
1865                 return -ECANCELED;
1866
1867         ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1868                                         &ctrl->rport->remoteport,
1869                                         op->queue->lldd_handle,
1870                                         &op->fcp_req);
1871
1872         return 0;
1873 }
1874
1875 static void
1876 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1877 {
1878         struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1879         int i;
1880
1881         /* ensure we've initialized the ops once */
1882         if (!(aen_op->flags & FCOP_FLAGS_AEN))
1883                 return;
1884
1885         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1886                 __nvme_fc_abort_op(ctrl, aen_op);
1887 }
1888
1889 static inline void
1890 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1891                 struct nvme_fc_fcp_op *op, int opstate)
1892 {
1893         unsigned long flags;
1894
1895         if (opstate == FCPOP_STATE_ABORTED) {
1896                 spin_lock_irqsave(&ctrl->lock, flags);
1897                 if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1898                     op->flags & FCOP_FLAGS_TERMIO) {
1899                         if (!--ctrl->iocnt)
1900                                 wake_up(&ctrl->ioabort_wait);
1901                 }
1902                 spin_unlock_irqrestore(&ctrl->lock, flags);
1903         }
1904 }
1905
1906 static void
1907 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1908 {
1909         struct nvme_fc_ctrl *ctrl =
1910                         container_of(work, struct nvme_fc_ctrl, ioerr_work);
1911
1912         nvme_fc_error_recovery(ctrl, "transport detected io error");
1913 }
1914
1915 /*
1916  * nvme_fc_io_getuuid - Routine called to get the appid field
1917  * associated with request by the lldd
1918  * @req:IO request from nvme fc to driver
1919  * Returns: UUID if there is an appid associated with VM or
1920  * NULL if the user/libvirt has not set the appid to VM
1921  */
1922 char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1923 {
1924         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1925         struct request *rq = op->rq;
1926
1927         if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq->bio)
1928                 return NULL;
1929         return blkcg_get_fc_appid(rq->bio);
1930 }
1931 EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1932
1933 static void
1934 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1935 {
1936         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1937         struct request *rq = op->rq;
1938         struct nvmefc_fcp_req *freq = &op->fcp_req;
1939         struct nvme_fc_ctrl *ctrl = op->ctrl;
1940         struct nvme_fc_queue *queue = op->queue;
1941         struct nvme_completion *cqe = &op->rsp_iu.cqe;
1942         struct nvme_command *sqe = &op->cmd_iu.sqe;
1943         __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1944         union nvme_result result;
1945         bool terminate_assoc = true;
1946         int opstate;
1947
1948         /*
1949          * WARNING:
1950          * The current linux implementation of a nvme controller
1951          * allocates a single tag set for all io queues and sizes
1952          * the io queues to fully hold all possible tags. Thus, the
1953          * implementation does not reference or care about the sqhd
1954          * value as it never needs to use the sqhd/sqtail pointers
1955          * for submission pacing.
1956          *
1957          * This affects the FC-NVME implementation in two ways:
1958          * 1) As the value doesn't matter, we don't need to waste
1959          *    cycles extracting it from ERSPs and stamping it in the
1960          *    cases where the transport fabricates CQEs on successful
1961          *    completions.
1962          * 2) The FC-NVME implementation requires that delivery of
1963          *    ERSP completions are to go back to the nvme layer in order
1964          *    relative to the rsn, such that the sqhd value will always
1965          *    be "in order" for the nvme layer. As the nvme layer in
1966          *    linux doesn't care about sqhd, there's no need to return
1967          *    them in order.
1968          *
1969          * Additionally:
1970          * As the core nvme layer in linux currently does not look at
1971          * every field in the cqe - in cases where the FC transport must
1972          * fabricate a CQE, the following fields will not be set as they
1973          * are not referenced:
1974          *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1975          *
1976          * Failure or error of an individual i/o, in a transport
1977          * detected fashion unrelated to the nvme completion status,
1978          * potentially cause the initiator and target sides to get out
1979          * of sync on SQ head/tail (aka outstanding io count allowed).
1980          * Per FC-NVME spec, failure of an individual command requires
1981          * the connection to be terminated, which in turn requires the
1982          * association to be terminated.
1983          */
1984
1985         opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1986
1987         fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1988                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1989
1990         if (opstate == FCPOP_STATE_ABORTED)
1991                 status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1992         else if (freq->status) {
1993                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1994                 dev_info(ctrl->ctrl.device,
1995                         "NVME-FC{%d}: io failed due to lldd error %d\n",
1996                         ctrl->cnum, freq->status);
1997         }
1998
1999         /*
2000          * For the linux implementation, if we have an unsuccesful
2001          * status, they blk-mq layer can typically be called with the
2002          * non-zero status and the content of the cqe isn't important.
2003          */
2004         if (status)
2005                 goto done;
2006
2007         /*
2008          * command completed successfully relative to the wire
2009          * protocol. However, validate anything received and
2010          * extract the status and result from the cqe (create it
2011          * where necessary).
2012          */
2013
2014         switch (freq->rcv_rsplen) {
2015
2016         case 0:
2017         case NVME_FC_SIZEOF_ZEROS_RSP:
2018                 /*
2019                  * No response payload or 12 bytes of payload (which
2020                  * should all be zeros) are considered successful and
2021                  * no payload in the CQE by the transport.
2022                  */
2023                 if (freq->transferred_length !=
2024                     be32_to_cpu(op->cmd_iu.data_len)) {
2025                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2026                         dev_info(ctrl->ctrl.device,
2027                                 "NVME-FC{%d}: io failed due to bad transfer "
2028                                 "length: %d vs expected %d\n",
2029                                 ctrl->cnum, freq->transferred_length,
2030                                 be32_to_cpu(op->cmd_iu.data_len));
2031                         goto done;
2032                 }
2033                 result.u64 = 0;
2034                 break;
2035
2036         case sizeof(struct nvme_fc_ersp_iu):
2037                 /*
2038                  * The ERSP IU contains a full completion with CQE.
2039                  * Validate ERSP IU and look at cqe.
2040                  */
2041                 if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2042                                         (freq->rcv_rsplen / 4) ||
2043                              be32_to_cpu(op->rsp_iu.xfrd_len) !=
2044                                         freq->transferred_length ||
2045                              op->rsp_iu.ersp_result ||
2046                              sqe->common.command_id != cqe->command_id)) {
2047                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2048                         dev_info(ctrl->ctrl.device,
2049                                 "NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2050                                 "iu len %d, xfr len %d vs %d, status code "
2051                                 "%d, cmdid %d vs %d\n",
2052                                 ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2053                                 be32_to_cpu(op->rsp_iu.xfrd_len),
2054                                 freq->transferred_length,
2055                                 op->rsp_iu.ersp_result,
2056                                 sqe->common.command_id,
2057                                 cqe->command_id);
2058                         goto done;
2059                 }
2060                 result = cqe->result;
2061                 status = cqe->status;
2062                 break;
2063
2064         default:
2065                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2066                 dev_info(ctrl->ctrl.device,
2067                         "NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2068                         "len %d\n",
2069                         ctrl->cnum, freq->rcv_rsplen);
2070                 goto done;
2071         }
2072
2073         terminate_assoc = false;
2074
2075 done:
2076         if (op->flags & FCOP_FLAGS_AEN) {
2077                 nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2078                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2079                 atomic_set(&op->state, FCPOP_STATE_IDLE);
2080                 op->flags = FCOP_FLAGS_AEN;     /* clear other flags */
2081                 nvme_fc_ctrl_put(ctrl);
2082                 goto check_error;
2083         }
2084
2085         __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2086         if (!nvme_try_complete_req(rq, status, result))
2087                 nvme_fc_complete_rq(rq);
2088
2089 check_error:
2090         if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2091                 queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2092 }
2093
2094 static int
2095 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2096                 struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2097                 struct request *rq, u32 rqno)
2098 {
2099         struct nvme_fcp_op_w_sgl *op_w_sgl =
2100                 container_of(op, typeof(*op_w_sgl), op);
2101         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2102         int ret = 0;
2103
2104         memset(op, 0, sizeof(*op));
2105         op->fcp_req.cmdaddr = &op->cmd_iu;
2106         op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2107         op->fcp_req.rspaddr = &op->rsp_iu;
2108         op->fcp_req.rsplen = sizeof(op->rsp_iu);
2109         op->fcp_req.done = nvme_fc_fcpio_done;
2110         op->ctrl = ctrl;
2111         op->queue = queue;
2112         op->rq = rq;
2113         op->rqno = rqno;
2114
2115         cmdiu->format_id = NVME_CMD_FORMAT_ID;
2116         cmdiu->fc_id = NVME_CMD_FC_ID;
2117         cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2118         if (queue->qnum)
2119                 cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2120                                         (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2121         else
2122                 cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2123
2124         op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2125                                 &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2126         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2127                 dev_err(ctrl->dev,
2128                         "FCP Op failed - cmdiu dma mapping failed.\n");
2129                 ret = -EFAULT;
2130                 goto out_on_error;
2131         }
2132
2133         op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2134                                 &op->rsp_iu, sizeof(op->rsp_iu),
2135                                 DMA_FROM_DEVICE);
2136         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2137                 dev_err(ctrl->dev,
2138                         "FCP Op failed - rspiu dma mapping failed.\n");
2139                 ret = -EFAULT;
2140         }
2141
2142         atomic_set(&op->state, FCPOP_STATE_IDLE);
2143 out_on_error:
2144         return ret;
2145 }
2146
2147 static int
2148 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2149                 unsigned int hctx_idx, unsigned int numa_node)
2150 {
2151         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2152         struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2153         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2154         struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2155         int res;
2156
2157         res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2158         if (res)
2159                 return res;
2160         op->op.fcp_req.first_sgl = op->sgl;
2161         op->op.fcp_req.private = &op->priv[0];
2162         nvme_req(rq)->ctrl = &ctrl->ctrl;
2163         nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2164         return res;
2165 }
2166
2167 static int
2168 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2169 {
2170         struct nvme_fc_fcp_op *aen_op;
2171         struct nvme_fc_cmd_iu *cmdiu;
2172         struct nvme_command *sqe;
2173         void *private = NULL;
2174         int i, ret;
2175
2176         aen_op = ctrl->aen_ops;
2177         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2178                 if (ctrl->lport->ops->fcprqst_priv_sz) {
2179                         private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2180                                                 GFP_KERNEL);
2181                         if (!private)
2182                                 return -ENOMEM;
2183                 }
2184
2185                 cmdiu = &aen_op->cmd_iu;
2186                 sqe = &cmdiu->sqe;
2187                 ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2188                                 aen_op, (struct request *)NULL,
2189                                 (NVME_AQ_BLK_MQ_DEPTH + i));
2190                 if (ret) {
2191                         kfree(private);
2192                         return ret;
2193                 }
2194
2195                 aen_op->flags = FCOP_FLAGS_AEN;
2196                 aen_op->fcp_req.private = private;
2197
2198                 memset(sqe, 0, sizeof(*sqe));
2199                 sqe->common.opcode = nvme_admin_async_event;
2200                 /* Note: core layer may overwrite the sqe.command_id value */
2201                 sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2202         }
2203         return 0;
2204 }
2205
2206 static void
2207 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2208 {
2209         struct nvme_fc_fcp_op *aen_op;
2210         int i;
2211
2212         cancel_work_sync(&ctrl->ctrl.async_event_work);
2213         aen_op = ctrl->aen_ops;
2214         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2215                 __nvme_fc_exit_request(ctrl, aen_op);
2216
2217                 kfree(aen_op->fcp_req.private);
2218                 aen_op->fcp_req.private = NULL;
2219         }
2220 }
2221
2222 static inline int
2223 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2224 {
2225         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data);
2226         struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2227
2228         hctx->driver_data = queue;
2229         queue->hctx = hctx;
2230         return 0;
2231 }
2232
2233 static int
2234 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2235 {
2236         return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1);
2237 }
2238
2239 static int
2240 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2241                 unsigned int hctx_idx)
2242 {
2243         return __nvme_fc_init_hctx(hctx, data, hctx_idx);
2244 }
2245
2246 static void
2247 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2248 {
2249         struct nvme_fc_queue *queue;
2250
2251         queue = &ctrl->queues[idx];
2252         memset(queue, 0, sizeof(*queue));
2253         queue->ctrl = ctrl;
2254         queue->qnum = idx;
2255         atomic_set(&queue->csn, 0);
2256         queue->dev = ctrl->dev;
2257
2258         if (idx > 0)
2259                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2260         else
2261                 queue->cmnd_capsule_len = sizeof(struct nvme_command);
2262
2263         /*
2264          * Considered whether we should allocate buffers for all SQEs
2265          * and CQEs and dma map them - mapping their respective entries
2266          * into the request structures (kernel vm addr and dma address)
2267          * thus the driver could use the buffers/mappings directly.
2268          * It only makes sense if the LLDD would use them for its
2269          * messaging api. It's very unlikely most adapter api's would use
2270          * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2271          * structures were used instead.
2272          */
2273 }
2274
2275 /*
2276  * This routine terminates a queue at the transport level.
2277  * The transport has already ensured that all outstanding ios on
2278  * the queue have been terminated.
2279  * The transport will send a Disconnect LS request to terminate
2280  * the queue's connection. Termination of the admin queue will also
2281  * terminate the association at the target.
2282  */
2283 static void
2284 nvme_fc_free_queue(struct nvme_fc_queue *queue)
2285 {
2286         if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2287                 return;
2288
2289         clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2290         /*
2291          * Current implementation never disconnects a single queue.
2292          * It always terminates a whole association. So there is never
2293          * a disconnect(queue) LS sent to the target.
2294          */
2295
2296         queue->connection_id = 0;
2297         atomic_set(&queue->csn, 0);
2298 }
2299
2300 static void
2301 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2302         struct nvme_fc_queue *queue, unsigned int qidx)
2303 {
2304         if (ctrl->lport->ops->delete_queue)
2305                 ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2306                                 queue->lldd_handle);
2307         queue->lldd_handle = NULL;
2308 }
2309
2310 static void
2311 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2312 {
2313         int i;
2314
2315         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2316                 nvme_fc_free_queue(&ctrl->queues[i]);
2317 }
2318
2319 static int
2320 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2321         struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2322 {
2323         int ret = 0;
2324
2325         queue->lldd_handle = NULL;
2326         if (ctrl->lport->ops->create_queue)
2327                 ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2328                                 qidx, qsize, &queue->lldd_handle);
2329
2330         return ret;
2331 }
2332
2333 static void
2334 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2335 {
2336         struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2337         int i;
2338
2339         for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2340                 __nvme_fc_delete_hw_queue(ctrl, queue, i);
2341 }
2342
2343 static int
2344 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2345 {
2346         struct nvme_fc_queue *queue = &ctrl->queues[1];
2347         int i, ret;
2348
2349         for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2350                 ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2351                 if (ret)
2352                         goto delete_queues;
2353         }
2354
2355         return 0;
2356
2357 delete_queues:
2358         for (; i > 0; i--)
2359                 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2360         return ret;
2361 }
2362
2363 static int
2364 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2365 {
2366         int i, ret = 0;
2367
2368         for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2369                 ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2370                                         (qsize / 5));
2371                 if (ret)
2372                         break;
2373                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2374                 if (ret)
2375                         break;
2376
2377                 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2378         }
2379
2380         return ret;
2381 }
2382
2383 static void
2384 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2385 {
2386         int i;
2387
2388         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2389                 nvme_fc_init_queue(ctrl, i);
2390 }
2391
2392 static void
2393 nvme_fc_ctrl_free(struct kref *ref)
2394 {
2395         struct nvme_fc_ctrl *ctrl =
2396                 container_of(ref, struct nvme_fc_ctrl, ref);
2397         unsigned long flags;
2398
2399         if (ctrl->ctrl.tagset)
2400                 nvme_remove_io_tag_set(&ctrl->ctrl);
2401
2402         /* remove from rport list */
2403         spin_lock_irqsave(&ctrl->rport->lock, flags);
2404         list_del(&ctrl->ctrl_list);
2405         spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2406
2407         nvme_unquiesce_admin_queue(&ctrl->ctrl);
2408         nvme_remove_admin_tag_set(&ctrl->ctrl);
2409
2410         kfree(ctrl->queues);
2411
2412         put_device(ctrl->dev);
2413         nvme_fc_rport_put(ctrl->rport);
2414
2415         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2416         if (ctrl->ctrl.opts)
2417                 nvmf_free_options(ctrl->ctrl.opts);
2418         kfree(ctrl);
2419 }
2420
2421 static void
2422 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2423 {
2424         kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2425 }
2426
2427 static int
2428 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2429 {
2430         return kref_get_unless_zero(&ctrl->ref);
2431 }
2432
2433 /*
2434  * All accesses from nvme core layer done - can now free the
2435  * controller. Called after last nvme_put_ctrl() call
2436  */
2437 static void
2438 nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl)
2439 {
2440         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2441
2442         WARN_ON(nctrl != &ctrl->ctrl);
2443
2444         nvme_fc_ctrl_put(ctrl);
2445 }
2446
2447 /*
2448  * This routine is used by the transport when it needs to find active
2449  * io on a queue that is to be terminated. The transport uses
2450  * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2451  * this routine to kill them on a 1 by 1 basis.
2452  *
2453  * As FC allocates FC exchange for each io, the transport must contact
2454  * the LLDD to terminate the exchange, thus releasing the FC exchange.
2455  * After terminating the exchange the LLDD will call the transport's
2456  * normal io done path for the request, but it will have an aborted
2457  * status. The done path will return the io request back to the block
2458  * layer with an error status.
2459  */
2460 static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2461 {
2462         struct nvme_ctrl *nctrl = data;
2463         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2464         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2465
2466         op->nreq.flags |= NVME_REQ_CANCELLED;
2467         __nvme_fc_abort_op(ctrl, op);
2468         return true;
2469 }
2470
2471 /*
2472  * This routine runs through all outstanding commands on the association
2473  * and aborts them.  This routine is typically be called by the
2474  * delete_association routine. It is also called due to an error during
2475  * reconnect. In that scenario, it is most likely a command that initializes
2476  * the controller, including fabric Connect commands on io queues, that
2477  * may have timed out or failed thus the io must be killed for the connect
2478  * thread to see the error.
2479  */
2480 static void
2481 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2482 {
2483         int q;
2484
2485         /*
2486          * if aborting io, the queues are no longer good, mark them
2487          * all as not live.
2488          */
2489         if (ctrl->ctrl.queue_count > 1) {
2490                 for (q = 1; q < ctrl->ctrl.queue_count; q++)
2491                         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2492         }
2493         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2494
2495         /*
2496          * If io queues are present, stop them and terminate all outstanding
2497          * ios on them. As FC allocates FC exchange for each io, the
2498          * transport must contact the LLDD to terminate the exchange,
2499          * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2500          * to tell us what io's are busy and invoke a transport routine
2501          * to kill them with the LLDD.  After terminating the exchange
2502          * the LLDD will call the transport's normal io done path, but it
2503          * will have an aborted status. The done path will return the
2504          * io requests back to the block layer as part of normal completions
2505          * (but with error status).
2506          */
2507         if (ctrl->ctrl.queue_count > 1) {
2508                 nvme_quiesce_io_queues(&ctrl->ctrl);
2509                 nvme_sync_io_queues(&ctrl->ctrl);
2510                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
2511                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2512                 blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2513                 if (start_queues)
2514                         nvme_unquiesce_io_queues(&ctrl->ctrl);
2515         }
2516
2517         /*
2518          * Other transports, which don't have link-level contexts bound
2519          * to sqe's, would try to gracefully shutdown the controller by
2520          * writing the registers for shutdown and polling (call
2521          * nvme_disable_ctrl()). Given a bunch of i/o was potentially
2522          * just aborted and we will wait on those contexts, and given
2523          * there was no indication of how live the controlelr is on the
2524          * link, don't send more io to create more contexts for the
2525          * shutdown. Let the controller fail via keepalive failure if
2526          * its still present.
2527          */
2528
2529         /*
2530          * clean up the admin queue. Same thing as above.
2531          */
2532         nvme_quiesce_admin_queue(&ctrl->ctrl);
2533         blk_sync_queue(ctrl->ctrl.admin_q);
2534         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2535                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2536         blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2537         if (start_queues)
2538                 nvme_unquiesce_admin_queue(&ctrl->ctrl);
2539 }
2540
2541 static void
2542 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2543 {
2544         /*
2545          * if an error (io timeout, etc) while (re)connecting, the remote
2546          * port requested terminating of the association (disconnect_ls)
2547          * or an error (timeout or abort) occurred on an io while creating
2548          * the controller.  Abort any ios on the association and let the
2549          * create_association error path resolve things.
2550          */
2551         if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
2552                 __nvme_fc_abort_outstanding_ios(ctrl, true);
2553                 set_bit(ASSOC_FAILED, &ctrl->flags);
2554                 return;
2555         }
2556
2557         /* Otherwise, only proceed if in LIVE state - e.g. on first error */
2558         if (ctrl->ctrl.state != NVME_CTRL_LIVE)
2559                 return;
2560
2561         dev_warn(ctrl->ctrl.device,
2562                 "NVME-FC{%d}: transport association event: %s\n",
2563                 ctrl->cnum, errmsg);
2564         dev_warn(ctrl->ctrl.device,
2565                 "NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2566
2567         nvme_reset_ctrl(&ctrl->ctrl);
2568 }
2569
2570 static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2571 {
2572         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2573         struct nvme_fc_ctrl *ctrl = op->ctrl;
2574         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2575         struct nvme_command *sqe = &cmdiu->sqe;
2576
2577         /*
2578          * Attempt to abort the offending command. Command completion
2579          * will detect the aborted io and will fail the connection.
2580          */
2581         dev_info(ctrl->ctrl.device,
2582                 "NVME-FC{%d.%d}: io timeout: opcode %d fctype %d w10/11: "
2583                 "x%08x/x%08x\n",
2584                 ctrl->cnum, op->queue->qnum, sqe->common.opcode,
2585                 sqe->connect.fctype, sqe->common.cdw10, sqe->common.cdw11);
2586         if (__nvme_fc_abort_op(ctrl, op))
2587                 nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2588
2589         /*
2590          * the io abort has been initiated. Have the reset timer
2591          * restarted and the abort completion will complete the io
2592          * shortly. Avoids a synchronous wait while the abort finishes.
2593          */
2594         return BLK_EH_RESET_TIMER;
2595 }
2596
2597 static int
2598 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2599                 struct nvme_fc_fcp_op *op)
2600 {
2601         struct nvmefc_fcp_req *freq = &op->fcp_req;
2602         int ret;
2603
2604         freq->sg_cnt = 0;
2605
2606         if (!blk_rq_nr_phys_segments(rq))
2607                 return 0;
2608
2609         freq->sg_table.sgl = freq->first_sgl;
2610         ret = sg_alloc_table_chained(&freq->sg_table,
2611                         blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2612                         NVME_INLINE_SG_CNT);
2613         if (ret)
2614                 return -ENOMEM;
2615
2616         op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
2617         WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2618         freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2619                                 op->nents, rq_dma_dir(rq));
2620         if (unlikely(freq->sg_cnt <= 0)) {
2621                 sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2622                 freq->sg_cnt = 0;
2623                 return -EFAULT;
2624         }
2625
2626         /*
2627          * TODO: blk_integrity_rq(rq)  for DIF
2628          */
2629         return 0;
2630 }
2631
2632 static void
2633 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2634                 struct nvme_fc_fcp_op *op)
2635 {
2636         struct nvmefc_fcp_req *freq = &op->fcp_req;
2637
2638         if (!freq->sg_cnt)
2639                 return;
2640
2641         fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2642                         rq_dma_dir(rq));
2643
2644         sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2645
2646         freq->sg_cnt = 0;
2647 }
2648
2649 /*
2650  * In FC, the queue is a logical thing. At transport connect, the target
2651  * creates its "queue" and returns a handle that is to be given to the
2652  * target whenever it posts something to the corresponding SQ.  When an
2653  * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2654  * command contained within the SQE, an io, and assigns a FC exchange
2655  * to it. The SQE and the associated SQ handle are sent in the initial
2656  * CMD IU sents on the exchange. All transfers relative to the io occur
2657  * as part of the exchange.  The CQE is the last thing for the io,
2658  * which is transferred (explicitly or implicitly) with the RSP IU
2659  * sent on the exchange. After the CQE is received, the FC exchange is
2660  * terminaed and the Exchange may be used on a different io.
2661  *
2662  * The transport to LLDD api has the transport making a request for a
2663  * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2664  * resource and transfers the command. The LLDD will then process all
2665  * steps to complete the io. Upon completion, the transport done routine
2666  * is called.
2667  *
2668  * So - while the operation is outstanding to the LLDD, there is a link
2669  * level FC exchange resource that is also outstanding. This must be
2670  * considered in all cleanup operations.
2671  */
2672 static blk_status_t
2673 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2674         struct nvme_fc_fcp_op *op, u32 data_len,
2675         enum nvmefc_fcp_datadir io_dir)
2676 {
2677         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2678         struct nvme_command *sqe = &cmdiu->sqe;
2679         int ret, opstate;
2680
2681         /*
2682          * before attempting to send the io, check to see if we believe
2683          * the target device is present
2684          */
2685         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2686                 return BLK_STS_RESOURCE;
2687
2688         if (!nvme_fc_ctrl_get(ctrl))
2689                 return BLK_STS_IOERR;
2690
2691         /* format the FC-NVME CMD IU and fcp_req */
2692         cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2693         cmdiu->data_len = cpu_to_be32(data_len);
2694         switch (io_dir) {
2695         case NVMEFC_FCP_WRITE:
2696                 cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2697                 break;
2698         case NVMEFC_FCP_READ:
2699                 cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2700                 break;
2701         case NVMEFC_FCP_NODATA:
2702                 cmdiu->flags = 0;
2703                 break;
2704         }
2705         op->fcp_req.payload_length = data_len;
2706         op->fcp_req.io_dir = io_dir;
2707         op->fcp_req.transferred_length = 0;
2708         op->fcp_req.rcv_rsplen = 0;
2709         op->fcp_req.status = NVME_SC_SUCCESS;
2710         op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2711
2712         /*
2713          * validate per fabric rules, set fields mandated by fabric spec
2714          * as well as those by FC-NVME spec.
2715          */
2716         WARN_ON_ONCE(sqe->common.metadata);
2717         sqe->common.flags |= NVME_CMD_SGL_METABUF;
2718
2719         /*
2720          * format SQE DPTR field per FC-NVME rules:
2721          *    type=0x5     Transport SGL Data Block Descriptor
2722          *    subtype=0xA  Transport-specific value
2723          *    address=0
2724          *    length=length of the data series
2725          */
2726         sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2727                                         NVME_SGL_FMT_TRANSPORT_A;
2728         sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2729         sqe->rw.dptr.sgl.addr = 0;
2730
2731         if (!(op->flags & FCOP_FLAGS_AEN)) {
2732                 ret = nvme_fc_map_data(ctrl, op->rq, op);
2733                 if (ret < 0) {
2734                         nvme_cleanup_cmd(op->rq);
2735                         nvme_fc_ctrl_put(ctrl);
2736                         if (ret == -ENOMEM || ret == -EAGAIN)
2737                                 return BLK_STS_RESOURCE;
2738                         return BLK_STS_IOERR;
2739                 }
2740         }
2741
2742         fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2743                                   sizeof(op->cmd_iu), DMA_TO_DEVICE);
2744
2745         atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2746
2747         if (!(op->flags & FCOP_FLAGS_AEN))
2748                 nvme_start_request(op->rq);
2749
2750         cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2751         ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2752                                         &ctrl->rport->remoteport,
2753                                         queue->lldd_handle, &op->fcp_req);
2754
2755         if (ret) {
2756                 /*
2757                  * If the lld fails to send the command is there an issue with
2758                  * the csn value?  If the command that fails is the Connect,
2759                  * no - as the connection won't be live.  If it is a command
2760                  * post-connect, it's possible a gap in csn may be created.
2761                  * Does this matter?  As Linux initiators don't send fused
2762                  * commands, no.  The gap would exist, but as there's nothing
2763                  * that depends on csn order to be delivered on the target
2764                  * side, it shouldn't hurt.  It would be difficult for a
2765                  * target to even detect the csn gap as it has no idea when the
2766                  * cmd with the csn was supposed to arrive.
2767                  */
2768                 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2769                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2770
2771                 if (!(op->flags & FCOP_FLAGS_AEN)) {
2772                         nvme_fc_unmap_data(ctrl, op->rq, op);
2773                         nvme_cleanup_cmd(op->rq);
2774                 }
2775
2776                 nvme_fc_ctrl_put(ctrl);
2777
2778                 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2779                                 ret != -EBUSY)
2780                         return BLK_STS_IOERR;
2781
2782                 return BLK_STS_RESOURCE;
2783         }
2784
2785         return BLK_STS_OK;
2786 }
2787
2788 static blk_status_t
2789 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2790                         const struct blk_mq_queue_data *bd)
2791 {
2792         struct nvme_ns *ns = hctx->queue->queuedata;
2793         struct nvme_fc_queue *queue = hctx->driver_data;
2794         struct nvme_fc_ctrl *ctrl = queue->ctrl;
2795         struct request *rq = bd->rq;
2796         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2797         enum nvmefc_fcp_datadir io_dir;
2798         bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2799         u32 data_len;
2800         blk_status_t ret;
2801
2802         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2803             !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2804                 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2805
2806         ret = nvme_setup_cmd(ns, rq);
2807         if (ret)
2808                 return ret;
2809
2810         /*
2811          * nvme core doesn't quite treat the rq opaquely. Commands such
2812          * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2813          * there is no actual payload to be transferred.
2814          * To get it right, key data transmission on there being 1 or
2815          * more physical segments in the sg list. If there is no
2816          * physical segments, there is no payload.
2817          */
2818         if (blk_rq_nr_phys_segments(rq)) {
2819                 data_len = blk_rq_payload_bytes(rq);
2820                 io_dir = ((rq_data_dir(rq) == WRITE) ?
2821                                         NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2822         } else {
2823                 data_len = 0;
2824                 io_dir = NVMEFC_FCP_NODATA;
2825         }
2826
2827
2828         return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2829 }
2830
2831 static void
2832 nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2833 {
2834         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2835         struct nvme_fc_fcp_op *aen_op;
2836         blk_status_t ret;
2837
2838         if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2839                 return;
2840
2841         aen_op = &ctrl->aen_ops[0];
2842
2843         ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2844                                         NVMEFC_FCP_NODATA);
2845         if (ret)
2846                 dev_err(ctrl->ctrl.device,
2847                         "failed async event work\n");
2848 }
2849
2850 static void
2851 nvme_fc_complete_rq(struct request *rq)
2852 {
2853         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2854         struct nvme_fc_ctrl *ctrl = op->ctrl;
2855
2856         atomic_set(&op->state, FCPOP_STATE_IDLE);
2857         op->flags &= ~FCOP_FLAGS_TERMIO;
2858
2859         nvme_fc_unmap_data(ctrl, rq, op);
2860         nvme_complete_rq(rq);
2861         nvme_fc_ctrl_put(ctrl);
2862 }
2863
2864 static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2865 {
2866         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2867         int i;
2868
2869         for (i = 0; i < set->nr_maps; i++) {
2870                 struct blk_mq_queue_map *map = &set->map[i];
2871
2872                 if (!map->nr_queues) {
2873                         WARN_ON(i == HCTX_TYPE_DEFAULT);
2874                         continue;
2875                 }
2876
2877                 /* Call LLDD map queue functionality if defined */
2878                 if (ctrl->lport->ops->map_queues)
2879                         ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2880                                                      map);
2881                 else
2882                         blk_mq_map_queues(map);
2883         }
2884 }
2885
2886 static const struct blk_mq_ops nvme_fc_mq_ops = {
2887         .queue_rq       = nvme_fc_queue_rq,
2888         .complete       = nvme_fc_complete_rq,
2889         .init_request   = nvme_fc_init_request,
2890         .exit_request   = nvme_fc_exit_request,
2891         .init_hctx      = nvme_fc_init_hctx,
2892         .timeout        = nvme_fc_timeout,
2893         .map_queues     = nvme_fc_map_queues,
2894 };
2895
2896 static int
2897 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2898 {
2899         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2900         unsigned int nr_io_queues;
2901         int ret;
2902
2903         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2904                                 ctrl->lport->ops->max_hw_queues);
2905         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2906         if (ret) {
2907                 dev_info(ctrl->ctrl.device,
2908                         "set_queue_count failed: %d\n", ret);
2909                 return ret;
2910         }
2911
2912         ctrl->ctrl.queue_count = nr_io_queues + 1;
2913         if (!nr_io_queues)
2914                 return 0;
2915
2916         nvme_fc_init_io_queues(ctrl);
2917
2918         ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
2919                         &nvme_fc_mq_ops, 1,
2920                         struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2921                                       ctrl->lport->ops->fcprqst_priv_sz));
2922         if (ret)
2923                 return ret;
2924
2925         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2926         if (ret)
2927                 goto out_cleanup_tagset;
2928
2929         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2930         if (ret)
2931                 goto out_delete_hw_queues;
2932
2933         ctrl->ioq_live = true;
2934
2935         return 0;
2936
2937 out_delete_hw_queues:
2938         nvme_fc_delete_hw_io_queues(ctrl);
2939 out_cleanup_tagset:
2940         nvme_remove_io_tag_set(&ctrl->ctrl);
2941         nvme_fc_free_io_queues(ctrl);
2942
2943         /* force put free routine to ignore io queues */
2944         ctrl->ctrl.tagset = NULL;
2945
2946         return ret;
2947 }
2948
2949 static int
2950 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2951 {
2952         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2953         u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2954         unsigned int nr_io_queues;
2955         int ret;
2956
2957         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2958                                 ctrl->lport->ops->max_hw_queues);
2959         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2960         if (ret) {
2961                 dev_info(ctrl->ctrl.device,
2962                         "set_queue_count failed: %d\n", ret);
2963                 return ret;
2964         }
2965
2966         if (!nr_io_queues && prior_ioq_cnt) {
2967                 dev_info(ctrl->ctrl.device,
2968                         "Fail Reconnect: At least 1 io queue "
2969                         "required (was %d)\n", prior_ioq_cnt);
2970                 return -ENOSPC;
2971         }
2972
2973         ctrl->ctrl.queue_count = nr_io_queues + 1;
2974         /* check for io queues existing */
2975         if (ctrl->ctrl.queue_count == 1)
2976                 return 0;
2977
2978         if (prior_ioq_cnt != nr_io_queues) {
2979                 dev_info(ctrl->ctrl.device,
2980                         "reconnect: revising io queue count from %d to %d\n",
2981                         prior_ioq_cnt, nr_io_queues);
2982                 blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2983         }
2984
2985         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2986         if (ret)
2987                 goto out_free_io_queues;
2988
2989         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2990         if (ret)
2991                 goto out_delete_hw_queues;
2992
2993         return 0;
2994
2995 out_delete_hw_queues:
2996         nvme_fc_delete_hw_io_queues(ctrl);
2997 out_free_io_queues:
2998         nvme_fc_free_io_queues(ctrl);
2999         return ret;
3000 }
3001
3002 static void
3003 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3004 {
3005         struct nvme_fc_lport *lport = rport->lport;
3006
3007         atomic_inc(&lport->act_rport_cnt);
3008 }
3009
3010 static void
3011 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3012 {
3013         struct nvme_fc_lport *lport = rport->lport;
3014         u32 cnt;
3015
3016         cnt = atomic_dec_return(&lport->act_rport_cnt);
3017         if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3018                 lport->ops->localport_delete(&lport->localport);
3019 }
3020
3021 static int
3022 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3023 {
3024         struct nvme_fc_rport *rport = ctrl->rport;
3025         u32 cnt;
3026
3027         if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
3028                 return 1;
3029
3030         cnt = atomic_inc_return(&rport->act_ctrl_cnt);
3031         if (cnt == 1)
3032                 nvme_fc_rport_active_on_lport(rport);
3033
3034         return 0;
3035 }
3036
3037 static int
3038 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3039 {
3040         struct nvme_fc_rport *rport = ctrl->rport;
3041         struct nvme_fc_lport *lport = rport->lport;
3042         u32 cnt;
3043
3044         /* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3045
3046         cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3047         if (cnt == 0) {
3048                 if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3049                         lport->ops->remoteport_delete(&rport->remoteport);
3050                 nvme_fc_rport_inactive_on_lport(rport);
3051         }
3052
3053         return 0;
3054 }
3055
3056 /*
3057  * This routine restarts the controller on the host side, and
3058  * on the link side, recreates the controller association.
3059  */
3060 static int
3061 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3062 {
3063         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3064         struct nvmefc_ls_rcv_op *disls = NULL;
3065         unsigned long flags;
3066         int ret;
3067         bool changed;
3068
3069         ++ctrl->ctrl.nr_reconnects;
3070
3071         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3072                 return -ENODEV;
3073
3074         if (nvme_fc_ctlr_active_on_rport(ctrl))
3075                 return -ENOTUNIQ;
3076
3077         dev_info(ctrl->ctrl.device,
3078                 "NVME-FC{%d}: create association : host wwpn 0x%016llx "
3079                 " rport wwpn 0x%016llx: NQN \"%s\"\n",
3080                 ctrl->cnum, ctrl->lport->localport.port_name,
3081                 ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3082
3083         clear_bit(ASSOC_FAILED, &ctrl->flags);
3084
3085         /*
3086          * Create the admin queue
3087          */
3088
3089         ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3090                                 NVME_AQ_DEPTH);
3091         if (ret)
3092                 goto out_free_queue;
3093
3094         ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3095                                 NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3096         if (ret)
3097                 goto out_delete_hw_queue;
3098
3099         ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3100         if (ret)
3101                 goto out_disconnect_admin_queue;
3102
3103         set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3104
3105         /*
3106          * Check controller capabilities
3107          *
3108          * todo:- add code to check if ctrl attributes changed from
3109          * prior connection values
3110          */
3111
3112         ret = nvme_enable_ctrl(&ctrl->ctrl);
3113         if (ret || test_bit(ASSOC_FAILED, &ctrl->flags))
3114                 goto out_disconnect_admin_queue;
3115
3116         ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3117         ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3118                                                 (ilog2(SZ_4K) - 9);
3119
3120         nvme_unquiesce_admin_queue(&ctrl->ctrl);
3121
3122         ret = nvme_init_ctrl_finish(&ctrl->ctrl, false);
3123         if (ret || test_bit(ASSOC_FAILED, &ctrl->flags))
3124                 goto out_disconnect_admin_queue;
3125
3126         /* sanity checks */
3127
3128         /* FC-NVME does not have other data in the capsule */
3129         if (ctrl->ctrl.icdoff) {
3130                 dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3131                                 ctrl->ctrl.icdoff);
3132                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3133                 goto out_disconnect_admin_queue;
3134         }
3135
3136         /* FC-NVME supports normal SGL Data Block Descriptors */
3137         if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3138                 dev_err(ctrl->ctrl.device,
3139                         "Mandatory sgls are not supported!\n");
3140                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3141                 goto out_disconnect_admin_queue;
3142         }
3143
3144         if (opts->queue_size > ctrl->ctrl.maxcmd) {
3145                 /* warn if maxcmd is lower than queue_size */
3146                 dev_warn(ctrl->ctrl.device,
3147                         "queue_size %zu > ctrl maxcmd %u, reducing "
3148                         "to maxcmd\n",
3149                         opts->queue_size, ctrl->ctrl.maxcmd);
3150                 opts->queue_size = ctrl->ctrl.maxcmd;
3151                 ctrl->ctrl.sqsize = opts->queue_size - 1;
3152         }
3153
3154         ret = nvme_fc_init_aen_ops(ctrl);
3155         if (ret)
3156                 goto out_term_aen_ops;
3157
3158         /*
3159          * Create the io queues
3160          */
3161
3162         if (ctrl->ctrl.queue_count > 1) {
3163                 if (!ctrl->ioq_live)
3164                         ret = nvme_fc_create_io_queues(ctrl);
3165                 else
3166                         ret = nvme_fc_recreate_io_queues(ctrl);
3167         }
3168         if (ret || test_bit(ASSOC_FAILED, &ctrl->flags))
3169                 goto out_term_aen_ops;
3170
3171         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
3172
3173         ctrl->ctrl.nr_reconnects = 0;
3174
3175         if (changed)
3176                 nvme_start_ctrl(&ctrl->ctrl);
3177
3178         return 0;       /* Success */
3179
3180 out_term_aen_ops:
3181         nvme_fc_term_aen_ops(ctrl);
3182 out_disconnect_admin_queue:
3183         /* send a Disconnect(association) LS to fc-nvme target */
3184         nvme_fc_xmt_disconnect_assoc(ctrl);
3185         spin_lock_irqsave(&ctrl->lock, flags);
3186         ctrl->association_id = 0;
3187         disls = ctrl->rcv_disconn;
3188         ctrl->rcv_disconn = NULL;
3189         spin_unlock_irqrestore(&ctrl->lock, flags);
3190         if (disls)
3191                 nvme_fc_xmt_ls_rsp(disls);
3192 out_delete_hw_queue:
3193         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3194 out_free_queue:
3195         nvme_fc_free_queue(&ctrl->queues[0]);
3196         clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3197         nvme_fc_ctlr_inactive_on_rport(ctrl);
3198
3199         return ret;
3200 }
3201
3202
3203 /*
3204  * This routine stops operation of the controller on the host side.
3205  * On the host os stack side: Admin and IO queues are stopped,
3206  *   outstanding ios on them terminated via FC ABTS.
3207  * On the link side: the association is terminated.
3208  */
3209 static void
3210 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3211 {
3212         struct nvmefc_ls_rcv_op *disls = NULL;
3213         unsigned long flags;
3214
3215         if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3216                 return;
3217
3218         spin_lock_irqsave(&ctrl->lock, flags);
3219         set_bit(FCCTRL_TERMIO, &ctrl->flags);
3220         ctrl->iocnt = 0;
3221         spin_unlock_irqrestore(&ctrl->lock, flags);
3222
3223         __nvme_fc_abort_outstanding_ios(ctrl, false);
3224
3225         /* kill the aens as they are a separate path */
3226         nvme_fc_abort_aen_ops(ctrl);
3227
3228         /* wait for all io that had to be aborted */
3229         spin_lock_irq(&ctrl->lock);
3230         wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3231         clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3232         spin_unlock_irq(&ctrl->lock);
3233
3234         nvme_fc_term_aen_ops(ctrl);
3235
3236         /*
3237          * send a Disconnect(association) LS to fc-nvme target
3238          * Note: could have been sent at top of process, but
3239          * cleaner on link traffic if after the aborts complete.
3240          * Note: if association doesn't exist, association_id will be 0
3241          */
3242         if (ctrl->association_id)
3243                 nvme_fc_xmt_disconnect_assoc(ctrl);
3244
3245         spin_lock_irqsave(&ctrl->lock, flags);
3246         ctrl->association_id = 0;
3247         disls = ctrl->rcv_disconn;
3248         ctrl->rcv_disconn = NULL;
3249         spin_unlock_irqrestore(&ctrl->lock, flags);
3250         if (disls)
3251                 /*
3252                  * if a Disconnect Request was waiting for a response, send
3253                  * now that all ABTS's have been issued (and are complete).
3254                  */
3255                 nvme_fc_xmt_ls_rsp(disls);
3256
3257         if (ctrl->ctrl.tagset) {
3258                 nvme_fc_delete_hw_io_queues(ctrl);
3259                 nvme_fc_free_io_queues(ctrl);
3260         }
3261
3262         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3263         nvme_fc_free_queue(&ctrl->queues[0]);
3264
3265         /* re-enable the admin_q so anything new can fast fail */
3266         nvme_unquiesce_admin_queue(&ctrl->ctrl);
3267
3268         /* resume the io queues so that things will fast fail */
3269         nvme_unquiesce_io_queues(&ctrl->ctrl);
3270
3271         nvme_fc_ctlr_inactive_on_rport(ctrl);
3272 }
3273
3274 static void
3275 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3276 {
3277         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3278
3279         cancel_work_sync(&ctrl->ioerr_work);
3280         cancel_delayed_work_sync(&ctrl->connect_work);
3281         /*
3282          * kill the association on the link side.  this will block
3283          * waiting for io to terminate
3284          */
3285         nvme_fc_delete_association(ctrl);
3286 }
3287
3288 static void
3289 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3290 {
3291         struct nvme_fc_rport *rport = ctrl->rport;
3292         struct nvme_fc_remote_port *portptr = &rport->remoteport;
3293         unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3294         bool recon = true;
3295
3296         if (ctrl->ctrl.state != NVME_CTRL_CONNECTING)
3297                 return;
3298
3299         if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3300                 dev_info(ctrl->ctrl.device,
3301                         "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3302                         ctrl->cnum, status);
3303                 if (status > 0 && (status & NVME_SC_DNR))
3304                         recon = false;
3305         } else if (time_after_eq(jiffies, rport->dev_loss_end))
3306                 recon = false;
3307
3308         if (recon && nvmf_should_reconnect(&ctrl->ctrl)) {
3309                 if (portptr->port_state == FC_OBJSTATE_ONLINE)
3310                         dev_info(ctrl->ctrl.device,
3311                                 "NVME-FC{%d}: Reconnect attempt in %ld "
3312                                 "seconds\n",
3313                                 ctrl->cnum, recon_delay / HZ);
3314                 else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3315                         recon_delay = rport->dev_loss_end - jiffies;
3316
3317                 queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3318         } else {
3319                 if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3320                         if (status > 0 && (status & NVME_SC_DNR))
3321                                 dev_warn(ctrl->ctrl.device,
3322                                          "NVME-FC{%d}: reconnect failure\n",
3323                                          ctrl->cnum);
3324                         else
3325                                 dev_warn(ctrl->ctrl.device,
3326                                          "NVME-FC{%d}: Max reconnect attempts "
3327                                          "(%d) reached.\n",
3328                                          ctrl->cnum, ctrl->ctrl.nr_reconnects);
3329                 } else
3330                         dev_warn(ctrl->ctrl.device,
3331                                 "NVME-FC{%d}: dev_loss_tmo (%d) expired "
3332                                 "while waiting for remoteport connectivity.\n",
3333                                 ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3334                                         (ctrl->ctrl.opts->max_reconnects *
3335                                          ctrl->ctrl.opts->reconnect_delay)));
3336                 WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3337         }
3338 }
3339
3340 static void
3341 nvme_fc_reset_ctrl_work(struct work_struct *work)
3342 {
3343         struct nvme_fc_ctrl *ctrl =
3344                 container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3345
3346         nvme_stop_ctrl(&ctrl->ctrl);
3347
3348         /* will block will waiting for io to terminate */
3349         nvme_fc_delete_association(ctrl);
3350
3351         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3352                 dev_err(ctrl->ctrl.device,
3353                         "NVME-FC{%d}: error_recovery: Couldn't change state "
3354                         "to CONNECTING\n", ctrl->cnum);
3355
3356         if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3357                 if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3358                         dev_err(ctrl->ctrl.device,
3359                                 "NVME-FC{%d}: failed to schedule connect "
3360                                 "after reset\n", ctrl->cnum);
3361                 } else {
3362                         flush_delayed_work(&ctrl->connect_work);
3363                 }
3364         } else {
3365                 nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3366         }
3367 }
3368
3369
3370 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3371         .name                   = "fc",
3372         .module                 = THIS_MODULE,
3373         .flags                  = NVME_F_FABRICS,
3374         .reg_read32             = nvmf_reg_read32,
3375         .reg_read64             = nvmf_reg_read64,
3376         .reg_write32            = nvmf_reg_write32,
3377         .free_ctrl              = nvme_fc_nvme_ctrl_freed,
3378         .submit_async_event     = nvme_fc_submit_async_event,
3379         .delete_ctrl            = nvme_fc_delete_ctrl,
3380         .get_address            = nvmf_get_address,
3381 };
3382
3383 static void
3384 nvme_fc_connect_ctrl_work(struct work_struct *work)
3385 {
3386         int ret;
3387
3388         struct nvme_fc_ctrl *ctrl =
3389                         container_of(to_delayed_work(work),
3390                                 struct nvme_fc_ctrl, connect_work);
3391
3392         ret = nvme_fc_create_association(ctrl);
3393         if (ret)
3394                 nvme_fc_reconnect_or_delete(ctrl, ret);
3395         else
3396                 dev_info(ctrl->ctrl.device,
3397                         "NVME-FC{%d}: controller connect complete\n",
3398                         ctrl->cnum);
3399 }
3400
3401
3402 static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3403         .queue_rq       = nvme_fc_queue_rq,
3404         .complete       = nvme_fc_complete_rq,
3405         .init_request   = nvme_fc_init_request,
3406         .exit_request   = nvme_fc_exit_request,
3407         .init_hctx      = nvme_fc_init_admin_hctx,
3408         .timeout        = nvme_fc_timeout,
3409 };
3410
3411
3412 /*
3413  * Fails a controller request if it matches an existing controller
3414  * (association) with the same tuple:
3415  * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3416  *
3417  * The ports don't need to be compared as they are intrinsically
3418  * already matched by the port pointers supplied.
3419  */
3420 static bool
3421 nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3422                 struct nvmf_ctrl_options *opts)
3423 {
3424         struct nvme_fc_ctrl *ctrl;
3425         unsigned long flags;
3426         bool found = false;
3427
3428         spin_lock_irqsave(&rport->lock, flags);
3429         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3430                 found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3431                 if (found)
3432                         break;
3433         }
3434         spin_unlock_irqrestore(&rport->lock, flags);
3435
3436         return found;
3437 }
3438
3439 static struct nvme_ctrl *
3440 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3441         struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3442 {
3443         struct nvme_fc_ctrl *ctrl;
3444         unsigned long flags;
3445         int ret, idx, ctrl_loss_tmo;
3446
3447         if (!(rport->remoteport.port_role &
3448             (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3449                 ret = -EBADR;
3450                 goto out_fail;
3451         }
3452
3453         if (!opts->duplicate_connect &&
3454             nvme_fc_existing_controller(rport, opts)) {
3455                 ret = -EALREADY;
3456                 goto out_fail;
3457         }
3458
3459         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3460         if (!ctrl) {
3461                 ret = -ENOMEM;
3462                 goto out_fail;
3463         }
3464
3465         idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3466         if (idx < 0) {
3467                 ret = -ENOSPC;
3468                 goto out_free_ctrl;
3469         }
3470
3471         /*
3472          * if ctrl_loss_tmo is being enforced and the default reconnect delay
3473          * is being used, change to a shorter reconnect delay for FC.
3474          */
3475         if (opts->max_reconnects != -1 &&
3476             opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3477             opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3478                 ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3479                 opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3480                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3481                                                 opts->reconnect_delay);
3482         }
3483
3484         ctrl->ctrl.opts = opts;
3485         ctrl->ctrl.nr_reconnects = 0;
3486         if (lport->dev)
3487                 ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3488         else
3489                 ctrl->ctrl.numa_node = NUMA_NO_NODE;
3490         INIT_LIST_HEAD(&ctrl->ctrl_list);
3491         ctrl->lport = lport;
3492         ctrl->rport = rport;
3493         ctrl->dev = lport->dev;
3494         ctrl->cnum = idx;
3495         ctrl->ioq_live = false;
3496         init_waitqueue_head(&ctrl->ioabort_wait);
3497
3498         get_device(ctrl->dev);
3499         kref_init(&ctrl->ref);
3500
3501         INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3502         INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3503         INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3504         spin_lock_init(&ctrl->lock);
3505
3506         /* io queue count */
3507         ctrl->ctrl.queue_count = min_t(unsigned int,
3508                                 opts->nr_io_queues,
3509                                 lport->ops->max_hw_queues);
3510         ctrl->ctrl.queue_count++;       /* +1 for admin queue */
3511
3512         ctrl->ctrl.sqsize = opts->queue_size - 1;
3513         ctrl->ctrl.kato = opts->kato;
3514         ctrl->ctrl.cntlid = 0xffff;
3515
3516         ret = -ENOMEM;
3517         ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3518                                 sizeof(struct nvme_fc_queue), GFP_KERNEL);
3519         if (!ctrl->queues)
3520                 goto out_free_ida;
3521
3522         nvme_fc_init_queue(ctrl, 0);
3523
3524         /*
3525          * Would have been nice to init io queues tag set as well.
3526          * However, we require interaction from the controller
3527          * for max io queue count before we can do so.
3528          * Defer this to the connect path.
3529          */
3530
3531         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3532         if (ret)
3533                 goto out_free_queues;
3534
3535         /* at this point, teardown path changes to ref counting on nvme ctrl */
3536
3537         ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
3538                         &nvme_fc_admin_mq_ops,
3539                         struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3540                                       ctrl->lport->ops->fcprqst_priv_sz));
3541         if (ret)
3542                 goto fail_ctrl;
3543
3544         spin_lock_irqsave(&rport->lock, flags);
3545         list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3546         spin_unlock_irqrestore(&rport->lock, flags);
3547
3548         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING) ||
3549             !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3550                 dev_err(ctrl->ctrl.device,
3551                         "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3552                 goto fail_ctrl;
3553         }
3554
3555         if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3556                 dev_err(ctrl->ctrl.device,
3557                         "NVME-FC{%d}: failed to schedule initial connect\n",
3558                         ctrl->cnum);
3559                 goto fail_ctrl;
3560         }
3561
3562         flush_delayed_work(&ctrl->connect_work);
3563
3564         dev_info(ctrl->ctrl.device,
3565                 "NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
3566                 ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl));
3567
3568         return &ctrl->ctrl;
3569
3570 fail_ctrl:
3571         nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3572         cancel_work_sync(&ctrl->ioerr_work);
3573         cancel_work_sync(&ctrl->ctrl.reset_work);
3574         cancel_delayed_work_sync(&ctrl->connect_work);
3575
3576         ctrl->ctrl.opts = NULL;
3577
3578         /* initiate nvme ctrl ref counting teardown */
3579         nvme_uninit_ctrl(&ctrl->ctrl);
3580
3581         /* Remove core ctrl ref. */
3582         nvme_put_ctrl(&ctrl->ctrl);
3583
3584         /* as we're past the point where we transition to the ref
3585          * counting teardown path, if we return a bad pointer here,
3586          * the calling routine, thinking it's prior to the
3587          * transition, will do an rport put. Since the teardown
3588          * path also does a rport put, we do an extra get here to
3589          * so proper order/teardown happens.
3590          */
3591         nvme_fc_rport_get(rport);
3592
3593         return ERR_PTR(-EIO);
3594
3595 out_free_queues:
3596         kfree(ctrl->queues);
3597 out_free_ida:
3598         put_device(ctrl->dev);
3599         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3600 out_free_ctrl:
3601         kfree(ctrl);
3602 out_fail:
3603         /* exit via here doesn't follow ctlr ref points */
3604         return ERR_PTR(ret);
3605 }
3606
3607
3608 struct nvmet_fc_traddr {
3609         u64     nn;
3610         u64     pn;
3611 };
3612
3613 static int
3614 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3615 {
3616         u64 token64;
3617
3618         if (match_u64(sstr, &token64))
3619                 return -EINVAL;
3620         *val = token64;
3621
3622         return 0;
3623 }
3624
3625 /*
3626  * This routine validates and extracts the WWN's from the TRADDR string.
3627  * As kernel parsers need the 0x to determine number base, universally
3628  * build string to parse with 0x prefix before parsing name strings.
3629  */
3630 static int
3631 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3632 {
3633         char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3634         substring_t wwn = { name, &name[sizeof(name)-1] };
3635         int nnoffset, pnoffset;
3636
3637         /* validate if string is one of the 2 allowed formats */
3638         if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3639                         !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3640                         !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3641                                 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3642                 nnoffset = NVME_FC_TRADDR_OXNNLEN;
3643                 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3644                                                 NVME_FC_TRADDR_OXNNLEN;
3645         } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3646                         !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3647                         !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3648                                 "pn-", NVME_FC_TRADDR_NNLEN))) {
3649                 nnoffset = NVME_FC_TRADDR_NNLEN;
3650                 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3651         } else
3652                 goto out_einval;
3653
3654         name[0] = '0';
3655         name[1] = 'x';
3656         name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3657
3658         memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3659         if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3660                 goto out_einval;
3661
3662         memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3663         if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3664                 goto out_einval;
3665
3666         return 0;
3667
3668 out_einval:
3669         pr_warn("%s: bad traddr string\n", __func__);
3670         return -EINVAL;
3671 }
3672
3673 static struct nvme_ctrl *
3674 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3675 {
3676         struct nvme_fc_lport *lport;
3677         struct nvme_fc_rport *rport;
3678         struct nvme_ctrl *ctrl;
3679         struct nvmet_fc_traddr laddr = { 0L, 0L };
3680         struct nvmet_fc_traddr raddr = { 0L, 0L };
3681         unsigned long flags;
3682         int ret;
3683
3684         ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3685         if (ret || !raddr.nn || !raddr.pn)
3686                 return ERR_PTR(-EINVAL);
3687
3688         ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3689         if (ret || !laddr.nn || !laddr.pn)
3690                 return ERR_PTR(-EINVAL);
3691
3692         /* find the host and remote ports to connect together */
3693         spin_lock_irqsave(&nvme_fc_lock, flags);
3694         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3695                 if (lport->localport.node_name != laddr.nn ||
3696                     lport->localport.port_name != laddr.pn ||
3697                     lport->localport.port_state != FC_OBJSTATE_ONLINE)
3698                         continue;
3699
3700                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3701                         if (rport->remoteport.node_name != raddr.nn ||
3702                             rport->remoteport.port_name != raddr.pn ||
3703                             rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3704                                 continue;
3705
3706                         /* if fail to get reference fall through. Will error */
3707                         if (!nvme_fc_rport_get(rport))
3708                                 break;
3709
3710                         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3711
3712                         ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3713                         if (IS_ERR(ctrl))
3714                                 nvme_fc_rport_put(rport);
3715                         return ctrl;
3716                 }
3717         }
3718         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3719
3720         pr_warn("%s: %s - %s combination not found\n",
3721                 __func__, opts->traddr, opts->host_traddr);
3722         return ERR_PTR(-ENOENT);
3723 }
3724
3725
3726 static struct nvmf_transport_ops nvme_fc_transport = {
3727         .name           = "fc",
3728         .module         = THIS_MODULE,
3729         .required_opts  = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3730         .allowed_opts   = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3731         .create_ctrl    = nvme_fc_create_ctrl,
3732 };
3733
3734 /* Arbitrary successive failures max. With lots of subsystems could be high */
3735 #define DISCOVERY_MAX_FAIL      20
3736
3737 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3738                 struct device_attribute *attr, const char *buf, size_t count)
3739 {
3740         unsigned long flags;
3741         LIST_HEAD(local_disc_list);
3742         struct nvme_fc_lport *lport;
3743         struct nvme_fc_rport *rport;
3744         int failcnt = 0;
3745
3746         spin_lock_irqsave(&nvme_fc_lock, flags);
3747 restart:
3748         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3749                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3750                         if (!nvme_fc_lport_get(lport))
3751                                 continue;
3752                         if (!nvme_fc_rport_get(rport)) {
3753                                 /*
3754                                  * This is a temporary condition. Upon restart
3755                                  * this rport will be gone from the list.
3756                                  *
3757                                  * Revert the lport put and retry.  Anything
3758                                  * added to the list already will be skipped (as
3759                                  * they are no longer list_empty).  Loops should
3760                                  * resume at rports that were not yet seen.
3761                                  */
3762                                 nvme_fc_lport_put(lport);
3763
3764                                 if (failcnt++ < DISCOVERY_MAX_FAIL)
3765                                         goto restart;
3766
3767                                 pr_err("nvme_discovery: too many reference "
3768                                        "failures\n");
3769                                 goto process_local_list;
3770                         }
3771                         if (list_empty(&rport->disc_list))
3772                                 list_add_tail(&rport->disc_list,
3773                                               &local_disc_list);
3774                 }
3775         }
3776
3777 process_local_list:
3778         while (!list_empty(&local_disc_list)) {
3779                 rport = list_first_entry(&local_disc_list,
3780                                          struct nvme_fc_rport, disc_list);
3781                 list_del_init(&rport->disc_list);
3782                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
3783
3784                 lport = rport->lport;
3785                 /* signal discovery. Won't hurt if it repeats */
3786                 nvme_fc_signal_discovery_scan(lport, rport);
3787                 nvme_fc_rport_put(rport);
3788                 nvme_fc_lport_put(lport);
3789
3790                 spin_lock_irqsave(&nvme_fc_lock, flags);
3791         }
3792         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3793
3794         return count;
3795 }
3796
3797 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3798
3799 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3800 /* Parse the cgroup id from a buf and return the length of cgrpid */
3801 static int fc_parse_cgrpid(const char *buf, u64 *id)
3802 {
3803         char cgrp_id[16+1];
3804         int cgrpid_len, j;
3805
3806         memset(cgrp_id, 0x0, sizeof(cgrp_id));
3807         for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3808                 if (buf[cgrpid_len] != ':')
3809                         cgrp_id[cgrpid_len] = buf[cgrpid_len];
3810                 else {
3811                         j = 1;
3812                         break;
3813                 }
3814         }
3815         if (!j)
3816                 return -EINVAL;
3817         if (kstrtou64(cgrp_id, 16, id) < 0)
3818                 return -EINVAL;
3819         return cgrpid_len;
3820 }
3821
3822 /*
3823  * Parse and update the appid in the blkcg associated with the cgroupid.
3824  */
3825 static ssize_t fc_appid_store(struct device *dev,
3826                 struct device_attribute *attr, const char *buf, size_t count)
3827 {
3828         size_t orig_count = count;
3829         u64 cgrp_id;
3830         int appid_len = 0;
3831         int cgrpid_len = 0;
3832         char app_id[FC_APPID_LEN];
3833         int ret = 0;
3834
3835         if (buf[count-1] == '\n')
3836                 count--;
3837
3838         if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3839                 return -EINVAL;
3840
3841         cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3842         if (cgrpid_len < 0)
3843                 return -EINVAL;
3844         appid_len = count - cgrpid_len - 1;
3845         if (appid_len > FC_APPID_LEN)
3846                 return -EINVAL;
3847
3848         memset(app_id, 0x0, sizeof(app_id));
3849         memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3850         ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3851         if (ret < 0)
3852                 return ret;
3853         return orig_count;
3854 }
3855 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3856 #endif /* CONFIG_BLK_CGROUP_FC_APPID */
3857
3858 static struct attribute *nvme_fc_attrs[] = {
3859         &dev_attr_nvme_discovery.attr,
3860 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3861         &dev_attr_appid_store.attr,
3862 #endif
3863         NULL
3864 };
3865
3866 static const struct attribute_group nvme_fc_attr_group = {
3867         .attrs = nvme_fc_attrs,
3868 };
3869
3870 static const struct attribute_group *nvme_fc_attr_groups[] = {
3871         &nvme_fc_attr_group,
3872         NULL
3873 };
3874
3875 static struct class fc_class = {
3876         .name = "fc",
3877         .dev_groups = nvme_fc_attr_groups,
3878 };
3879
3880 static int __init nvme_fc_init_module(void)
3881 {
3882         int ret;
3883
3884         nvme_fc_wq = alloc_workqueue("nvme_fc_wq", WQ_MEM_RECLAIM, 0);
3885         if (!nvme_fc_wq)
3886                 return -ENOMEM;
3887
3888         /*
3889          * NOTE:
3890          * It is expected that in the future the kernel will combine
3891          * the FC-isms that are currently under scsi and now being
3892          * added to by NVME into a new standalone FC class. The SCSI
3893          * and NVME protocols and their devices would be under this
3894          * new FC class.
3895          *
3896          * As we need something to post FC-specific udev events to,
3897          * specifically for nvme probe events, start by creating the
3898          * new device class.  When the new standalone FC class is
3899          * put in place, this code will move to a more generic
3900          * location for the class.
3901          */
3902         ret = class_register(&fc_class);
3903         if (ret) {
3904                 pr_err("couldn't register class fc\n");
3905                 goto out_destroy_wq;
3906         }
3907
3908         /*
3909          * Create a device for the FC-centric udev events
3910          */
3911         fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3912                                 "fc_udev_device");
3913         if (IS_ERR(fc_udev_device)) {
3914                 pr_err("couldn't create fc_udev device!\n");
3915                 ret = PTR_ERR(fc_udev_device);
3916                 goto out_destroy_class;
3917         }
3918
3919         ret = nvmf_register_transport(&nvme_fc_transport);
3920         if (ret)
3921                 goto out_destroy_device;
3922
3923         return 0;
3924
3925 out_destroy_device:
3926         device_destroy(&fc_class, MKDEV(0, 0));
3927 out_destroy_class:
3928         class_unregister(&fc_class);
3929 out_destroy_wq:
3930         destroy_workqueue(nvme_fc_wq);
3931
3932         return ret;
3933 }
3934
3935 static void
3936 nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3937 {
3938         struct nvme_fc_ctrl *ctrl;
3939
3940         spin_lock(&rport->lock);
3941         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3942                 dev_warn(ctrl->ctrl.device,
3943                         "NVME-FC{%d}: transport unloading: deleting ctrl\n",
3944                         ctrl->cnum);
3945                 nvme_delete_ctrl(&ctrl->ctrl);
3946         }
3947         spin_unlock(&rport->lock);
3948 }
3949
3950 static void
3951 nvme_fc_cleanup_for_unload(void)
3952 {
3953         struct nvme_fc_lport *lport;
3954         struct nvme_fc_rport *rport;
3955
3956         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3957                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3958                         nvme_fc_delete_controllers(rport);
3959                 }
3960         }
3961 }
3962
3963 static void __exit nvme_fc_exit_module(void)
3964 {
3965         unsigned long flags;
3966         bool need_cleanup = false;
3967
3968         spin_lock_irqsave(&nvme_fc_lock, flags);
3969         nvme_fc_waiting_to_unload = true;
3970         if (!list_empty(&nvme_fc_lport_list)) {
3971                 need_cleanup = true;
3972                 nvme_fc_cleanup_for_unload();
3973         }
3974         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3975         if (need_cleanup) {
3976                 pr_info("%s: waiting for ctlr deletes\n", __func__);
3977                 wait_for_completion(&nvme_fc_unload_proceed);
3978                 pr_info("%s: ctrl deletes complete\n", __func__);
3979         }
3980
3981         nvmf_unregister_transport(&nvme_fc_transport);
3982
3983         ida_destroy(&nvme_fc_local_port_cnt);
3984         ida_destroy(&nvme_fc_ctrl_cnt);
3985
3986         device_destroy(&fc_class, MKDEV(0, 0));
3987         class_unregister(&fc_class);
3988         destroy_workqueue(nvme_fc_wq);
3989 }
3990
3991 module_init(nvme_fc_init_module);
3992 module_exit(nvme_fc_exit_module);
3993
3994 MODULE_LICENSE("GPL v2");