drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) PFX fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43
  44 #include <linux/atomic.h>
  45
  46 #include <scsi/scsi.h>
  47 #include <scsi/scsi_device.h>
  48 #include <scsi/scsi_dbg.h>
  49 #include <scsi/srp.h>
  50 #include <scsi/scsi_transport_srp.h>
  51
  52 #include "ib_srp.h"
  53
  54 #define DRV_NAME        "ib_srp"
  55 #define PFX             DRV_NAME ": "
  56 #define DRV_VERSION     "1.0"
  57 #define DRV_RELDATE     "July 1, 2013"
  58
  59 MODULE_AUTHOR("Roland Dreier");
  60 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator "
  61                    "v" DRV_VERSION " (" DRV_RELDATE ")");
  62 MODULE_LICENSE("Dual BSD/GPL");
  63
  64 static unsigned int srp_sg_tablesize;
  65 static unsigned int cmd_sg_entries;
  66 static unsigned int indirect_sg_entries;
  67 static bool allow_ext_sg;
  68 static int topspin_workarounds = 1;
  69
  70 module_param(srp_sg_tablesize, uint, 0444);
  71 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  72
  73 module_param(cmd_sg_entries, uint, 0444);
  74 MODULE_PARM_DESC(cmd_sg_entries,
  75                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  76
  77 module_param(indirect_sg_entries, uint, 0444);
  78 MODULE_PARM_DESC(indirect_sg_entries,
  79                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
  80
  81 module_param(allow_ext_sg, bool, 0444);
  82 MODULE_PARM_DESC(allow_ext_sg,
  83                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  84
  85 module_param(topspin_workarounds, int, 0444);
  86 MODULE_PARM_DESC(topspin_workarounds,
  87                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  88
  89 static struct kernel_param_ops srp_tmo_ops;
  90
  91 static int srp_reconnect_delay = 10;
  92 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
  93                 S_IRUGO | S_IWUSR);
  94 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
  95
  96 static int srp_fast_io_fail_tmo = 15;
  97 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
  98                 S_IRUGO | S_IWUSR);
  99 MODULE_PARM_DESC(fast_io_fail_tmo,
 100                  "Number of seconds between the observation of a transport"
 101                  " layer error and failing all I/O. \"off\" means that this"
 102                  " functionality is disabled.");
 103
 104 static int srp_dev_loss_tmo = 600;
 105 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 106                 S_IRUGO | S_IWUSR);
 107 MODULE_PARM_DESC(dev_loss_tmo,
 108                  "Maximum number of seconds that the SRP transport should"
 109                  " insulate transport layer errors. After this time has been"
 110                  " exceeded the SCSI host is removed. Should be"
 111                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 112                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 113                  " this functionality is disabled.");
 114
 115 static void srp_add_one(struct ib_device *device);
 116 static void srp_remove_one(struct ib_device *device);
 117 static void srp_recv_completion(struct ib_cq *cq, void *target_ptr);
 118 static void srp_send_completion(struct ib_cq *cq, void *target_ptr);
 119 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 120
 121 static struct scsi_transport_template *ib_srp_transport_template;
 122
 123 static struct ib_client srp_client = {
 124         .name   = "srp",
 125         .add    = srp_add_one,
 126         .remove = srp_remove_one
 127 };
 128
 129 static struct ib_sa_client srp_sa_client;
 130
 131 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 132 {
 133         int tmo = *(int *)kp->arg;
 134
 135         if (tmo >= 0)
 136                 return sprintf(buffer, "%d", tmo);
 137         else
 138                 return sprintf(buffer, "off");
 139 }
 140
 141 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 142 {
 143         int tmo, res;
 144
 145         if (strncmp(val, "off", 3) != 0) {
 146                 res = kstrtoint(val, 0, &tmo);
 147                 if (res)
 148                         goto out;
 149         } else {
 150                 tmo = -1;
 151         }
 152         if (kp->arg == &srp_reconnect_delay)
 153                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 154                                     srp_dev_loss_tmo);
 155         else if (kp->arg == &srp_fast_io_fail_tmo)
 156                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 157         else
 158                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 159                                     tmo);
 160         if (res)
 161                 goto out;
 162         *(int *)kp->arg = tmo;
 163
 164 out:
 165         return res;
 166 }
 167
 168 static struct kernel_param_ops srp_tmo_ops = {
 169         .get = srp_tmo_get,
 170         .set = srp_tmo_set,
 171 };
 172
 173 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 174 {
 175         return (struct srp_target_port *) host->hostdata;
 176 }
 177
 178 static const char *srp_target_info(struct Scsi_Host *host)
 179 {
 180         return host_to_target(host)->target_name;
 181 }
 182
 183 static int srp_target_is_topspin(struct srp_target_port *target)
 184 {
 185         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 186         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 187
 188         return topspin_workarounds &&
 189                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 190                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 191 }
 192
 193 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 194                                    gfp_t gfp_mask,
 195                                    enum dma_data_direction direction)
 196 {
 197         struct srp_iu *iu;
 198
 199         iu = kmalloc(sizeof *iu, gfp_mask);
 200         if (!iu)
 201                 goto out;
 202
 203         iu->buf = kzalloc(size, gfp_mask);
 204         if (!iu->buf)
 205                 goto out_free_iu;
 206
 207         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 208                                     direction);
 209         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 210                 goto out_free_buf;
 211
 212         iu->size      = size;
 213         iu->direction = direction;
 214
 215         return iu;
 216
 217 out_free_buf:
 218         kfree(iu->buf);
 219 out_free_iu:
 220         kfree(iu);
 221 out:
 222         return NULL;
 223 }
 224
 225 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 226 {
 227         if (!iu)
 228                 return;
 229
 230         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 231                             iu->direction);
 232         kfree(iu->buf);
 233         kfree(iu);
 234 }
 235
 236 static void srp_qp_event(struct ib_event *event, void *context)
 237 {
 238         pr_debug("QP event %d\n", event->event);
 239 }
 240
 241 static int srp_init_qp(struct srp_target_port *target,
 242                        struct ib_qp *qp)
 243 {
 244         struct ib_qp_attr *attr;
 245         int ret;
 246
 247         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 248         if (!attr)
 249                 return -ENOMEM;
 250
 251         ret = ib_find_pkey(target->srp_host->srp_dev->dev,
 252                            target->srp_host->port,
 253                            be16_to_cpu(target->path.pkey),
 254                            &attr->pkey_index);
 255         if (ret)
 256                 goto out;
 257
 258         attr->qp_state        = IB_QPS_INIT;
 259         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 260                                     IB_ACCESS_REMOTE_WRITE);
 261         attr->port_num        = target->srp_host->port;
 262
 263         ret = ib_modify_qp(qp, attr,
 264                            IB_QP_STATE          |
 265                            IB_QP_PKEY_INDEX     |
 266                            IB_QP_ACCESS_FLAGS   |
 267                            IB_QP_PORT);
 268
 269 out:
 270         kfree(attr);
 271         return ret;
 272 }
 273
 274 static int srp_new_cm_id(struct srp_target_port *target)
 275 {
 276         struct ib_cm_id *new_cm_id;
 277
 278         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 279                                     srp_cm_handler, target);
 280         if (IS_ERR(new_cm_id))
 281                 return PTR_ERR(new_cm_id);
 282
 283         if (target->cm_id)
 284                 ib_destroy_cm_id(target->cm_id);
 285         target->cm_id = new_cm_id;
 286
 287         return 0;
 288 }
 289
 290 static int srp_create_target_ib(struct srp_target_port *target)
 291 {
 292         struct ib_qp_init_attr *init_attr;
 293         struct ib_cq *recv_cq, *send_cq;
 294         struct ib_qp *qp;
 295         int ret;
 296
 297         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 298         if (!init_attr)
 299                 return -ENOMEM;
 300
 301         recv_cq = ib_create_cq(target->srp_host->srp_dev->dev,
 302                                srp_recv_completion, NULL, target,
 303                                target->queue_size, target->comp_vector);
 304         if (IS_ERR(recv_cq)) {
 305                 ret = PTR_ERR(recv_cq);
 306                 goto err;
 307         }
 308
 309         send_cq = ib_create_cq(target->srp_host->srp_dev->dev,
 310                                srp_send_completion, NULL, target,
 311                                target->queue_size, target->comp_vector);
 312         if (IS_ERR(send_cq)) {
 313                 ret = PTR_ERR(send_cq);
 314                 goto err_recv_cq;
 315         }
 316
 317         ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
 318
 319         init_attr->event_handler       = srp_qp_event;
 320         init_attr->cap.max_send_wr     = target->queue_size;
 321         init_attr->cap.max_recv_wr     = target->queue_size;
 322         init_attr->cap.max_recv_sge    = 1;
 323         init_attr->cap.max_send_sge    = 1;
 324         init_attr->sq_sig_type         = IB_SIGNAL_ALL_WR;
 325         init_attr->qp_type             = IB_QPT_RC;
 326         init_attr->send_cq             = send_cq;
 327         init_attr->recv_cq             = recv_cq;
 328
 329         qp = ib_create_qp(target->srp_host->srp_dev->pd, init_attr);
 330         if (IS_ERR(qp)) {
 331                 ret = PTR_ERR(qp);
 332                 goto err_send_cq;
 333         }
 334
 335         ret = srp_init_qp(target, qp);
 336         if (ret)
 337                 goto err_qp;
 338
 339         if (target->qp)
 340                 ib_destroy_qp(target->qp);
 341         if (target->recv_cq)
 342                 ib_destroy_cq(target->recv_cq);
 343         if (target->send_cq)
 344                 ib_destroy_cq(target->send_cq);
 345
 346         target->qp = qp;
 347         target->recv_cq = recv_cq;
 348         target->send_cq = send_cq;
 349
 350         kfree(init_attr);
 351         return 0;
 352
 353 err_qp:
 354         ib_destroy_qp(qp);
 355
 356 err_send_cq:
 357         ib_destroy_cq(send_cq);
 358
 359 err_recv_cq:
 360         ib_destroy_cq(recv_cq);
 361
 362 err:
 363         kfree(init_attr);
 364         return ret;
 365 }
 366
 367 /*
 368  * Note: this function may be called without srp_alloc_iu_bufs() having been
 369  * invoked. Hence the target->[rt]x_ring checks.
 370  */
 371 static void srp_free_target_ib(struct srp_target_port *target)
 372 {
 373         int i;
 374
 375         ib_destroy_qp(target->qp);
 376         ib_destroy_cq(target->send_cq);
 377         ib_destroy_cq(target->recv_cq);
 378
 379         target->qp = NULL;
 380         target->send_cq = target->recv_cq = NULL;
 381
 382         if (target->rx_ring) {
 383                 for (i = 0; i < target->queue_size; ++i)
 384                         srp_free_iu(target->srp_host, target->rx_ring[i]);
 385                 kfree(target->rx_ring);
 386                 target->rx_ring = NULL;
 387         }
 388         if (target->tx_ring) {
 389                 for (i = 0; i < target->queue_size; ++i)
 390                         srp_free_iu(target->srp_host, target->tx_ring[i]);
 391                 kfree(target->tx_ring);
 392                 target->tx_ring = NULL;
 393         }
 394 }
 395
 396 static void srp_path_rec_completion(int status,
 397                                     struct ib_sa_path_rec *pathrec,
 398                                     void *target_ptr)
 399 {
 400         struct srp_target_port *target = target_ptr;
 401
 402         target->status = status;
 403         if (status)
 404                 shost_printk(KERN_ERR, target->scsi_host,
 405                              PFX "Got failed path rec status %d\n", status);
 406         else
 407                 target->path = *pathrec;
 408         complete(&target->done);
 409 }
 410
 411 static int srp_lookup_path(struct srp_target_port *target)
 412 {
 413         target->path.numb_path = 1;
 414
 415         init_completion(&target->done);
 416
 417         target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 418                                                    target->srp_host->srp_dev->dev,
 419                                                    target->srp_host->port,
 420                                                    &target->path,
 421                                                    IB_SA_PATH_REC_SERVICE_ID    |
 422                                                    IB_SA_PATH_REC_DGID          |
 423                                                    IB_SA_PATH_REC_SGID          |
 424                                                    IB_SA_PATH_REC_NUMB_PATH     |
 425                                                    IB_SA_PATH_REC_PKEY,
 426                                                    SRP_PATH_REC_TIMEOUT_MS,
 427                                                    GFP_KERNEL,
 428                                                    srp_path_rec_completion,
 429                                                    target, &target->path_query);
 430         if (target->path_query_id < 0)
 431                 return target->path_query_id;
 432
 433         wait_for_completion(&target->done);
 434
 435         if (target->status < 0)
 436                 shost_printk(KERN_WARNING, target->scsi_host,
 437                              PFX "Path record query failed\n");
 438
 439         return target->status;
 440 }
 441
 442 static int srp_send_req(struct srp_target_port *target)
 443 {
 444         struct {
 445                 struct ib_cm_req_param param;
 446                 struct srp_login_req   priv;
 447         } *req = NULL;
 448         int status;
 449
 450         req = kzalloc(sizeof *req, GFP_KERNEL);
 451         if (!req)
 452                 return -ENOMEM;
 453
 454         req->param.primary_path               = &target->path;
 455         req->param.alternate_path             = NULL;
 456         req->param.service_id                 = target->service_id;
 457         req->param.qp_num                     = target->qp->qp_num;
 458         req->param.qp_type                    = target->qp->qp_type;
 459         req->param.private_data               = &req->priv;
 460         req->param.private_data_len           = sizeof req->priv;
 461         req->param.flow_control               = 1;
 462
 463         get_random_bytes(&req->param.starting_psn, 4);
 464         req->param.starting_psn              &= 0xffffff;
 465
 466         /*
 467          * Pick some arbitrary defaults here; we could make these
 468          * module parameters if anyone cared about setting them.
 469          */
 470         req->param.responder_resources        = 4;
 471         req->param.remote_cm_response_timeout = 20;
 472         req->param.local_cm_response_timeout  = 20;
 473         req->param.retry_count                = target->tl_retry_count;
 474         req->param.rnr_retry_count            = 7;
 475         req->param.max_cm_retries             = 15;
 476
 477         req->priv.opcode        = SRP_LOGIN_REQ;
 478         req->priv.tag           = 0;
 479         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 480         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 481                                               SRP_BUF_FORMAT_INDIRECT);
 482         /*
 483          * In the published SRP specification (draft rev. 16a), the
 484          * port identifier format is 8 bytes of ID extension followed
 485          * by 8 bytes of GUID.  Older drafts put the two halves in the
 486          * opposite order, so that the GUID comes first.
 487          *
 488          * Targets conforming to these obsolete drafts can be
 489          * recognized by the I/O Class they report.
 490          */
 491         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 492                 memcpy(req->priv.initiator_port_id,
 493                        &target->path.sgid.global.interface_id, 8);
 494                 memcpy(req->priv.initiator_port_id + 8,
 495                        &target->initiator_ext, 8);
 496                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 497                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 498         } else {
 499                 memcpy(req->priv.initiator_port_id,
 500                        &target->initiator_ext, 8);
 501                 memcpy(req->priv.initiator_port_id + 8,
 502                        &target->path.sgid.global.interface_id, 8);
 503                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 504                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 505         }
 506
 507         /*
 508          * Topspin/Cisco SRP targets will reject our login unless we
 509          * zero out the first 8 bytes of our initiator port ID and set
 510          * the second 8 bytes to the local node GUID.
 511          */
 512         if (srp_target_is_topspin(target)) {
 513                 shost_printk(KERN_DEBUG, target->scsi_host,
 514                              PFX "Topspin/Cisco initiator port ID workaround "
 515                              "activated for target GUID %016llx\n",
 516                              (unsigned long long) be64_to_cpu(target->ioc_guid));
 517                 memset(req->priv.initiator_port_id, 0, 8);
 518                 memcpy(req->priv.initiator_port_id + 8,
 519                        &target->srp_host->srp_dev->dev->node_guid, 8);
 520         }
 521
 522         status = ib_send_cm_req(target->cm_id, &req->param);
 523
 524         kfree(req);
 525
 526         return status;
 527 }
 528
 529 static bool srp_queue_remove_work(struct srp_target_port *target)
 530 {
 531         bool changed = false;
 532
 533         spin_lock_irq(&target->lock);
 534         if (target->state != SRP_TARGET_REMOVED) {
 535                 target->state = SRP_TARGET_REMOVED;
 536                 changed = true;
 537         }
 538         spin_unlock_irq(&target->lock);
 539
 540         if (changed)
 541                 queue_work(system_long_wq, &target->remove_work);
 542
 543         return changed;
 544 }
 545
 546 static bool srp_change_conn_state(struct srp_target_port *target,
 547                                   bool connected)
 548 {
 549         bool changed = false;
 550
 551         spin_lock_irq(&target->lock);
 552         if (target->connected != connected) {
 553                 target->connected = connected;
 554                 changed = true;
 555         }
 556         spin_unlock_irq(&target->lock);
 557
 558         return changed;
 559 }
 560
 561 static void srp_disconnect_target(struct srp_target_port *target)
 562 {
 563         if (srp_change_conn_state(target, false)) {
 564                 /* XXX should send SRP_I_LOGOUT request */
 565
 566                 if (ib_send_cm_dreq(target->cm_id, NULL, 0)) {
 567                         shost_printk(KERN_DEBUG, target->scsi_host,
 568                                      PFX "Sending CM DREQ failed\n");
 569                 }
 570         }
 571 }
 572
 573 static void srp_free_req_data(struct srp_target_port *target)
 574 {
 575         struct ib_device *ibdev = target->srp_host->srp_dev->dev;
 576         struct srp_request *req;
 577         int i;
 578
 579         if (!target->req_ring)
 580                 return;
 581
 582         for (i = 0; i < target->req_ring_size; ++i) {
 583                 req = &target->req_ring[i];
 584                 kfree(req->fmr_list);
 585                 kfree(req->map_page);
 586                 if (req->indirect_dma_addr) {
 587                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 588                                             target->indirect_size,
 589                                             DMA_TO_DEVICE);
 590                 }
 591                 kfree(req->indirect_desc);
 592         }
 593
 594         kfree(target->req_ring);
 595         target->req_ring = NULL;
 596 }
 597
 598 static int srp_alloc_req_data(struct srp_target_port *target)
 599 {
 600         struct srp_device *srp_dev = target->srp_host->srp_dev;
 601         struct ib_device *ibdev = srp_dev->dev;
 602         struct srp_request *req;
 603         dma_addr_t dma_addr;
 604         int i, ret = -ENOMEM;
 605
 606         INIT_LIST_HEAD(&target->free_reqs);
 607
 608         target->req_ring = kzalloc(target->req_ring_size *
 609                                    sizeof(*target->req_ring), GFP_KERNEL);
 610         if (!target->req_ring)
 611                 goto out;
 612
 613         for (i = 0; i < target->req_ring_size; ++i) {
 614                 req = &target->req_ring[i];
 615                 req->fmr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
 616                                         GFP_KERNEL);
 617                 req->map_page = kmalloc(SRP_FMR_SIZE * sizeof(void *),
 618                                         GFP_KERNEL);
 619                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 620                 if (!req->fmr_list || !req->map_page || !req->indirect_desc)
 621                         goto out;
 622
 623                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 624                                              target->indirect_size,
 625                                              DMA_TO_DEVICE);
 626                 if (ib_dma_mapping_error(ibdev, dma_addr))
 627                         goto out;
 628
 629                 req->indirect_dma_addr = dma_addr;
 630                 req->index = i;
 631                 list_add_tail(&req->list, &target->free_reqs);
 632         }
 633         ret = 0;
 634
 635 out:
 636         return ret;
 637 }
 638
 639 /**
 640  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 641  * @shost: SCSI host whose attributes to remove from sysfs.
 642  *
 643  * Note: Any attributes defined in the host template and that did not exist
 644  * before invocation of this function will be ignored.
 645  */
 646 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 647 {
 648         struct device_attribute **attr;
 649
 650         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 651                 device_remove_file(&shost->shost_dev, *attr);
 652 }
 653
 654 static void srp_remove_target(struct srp_target_port *target)
 655 {
 656         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 657
 658         srp_del_scsi_host_attr(target->scsi_host);
 659         srp_rport_get(target->rport);
 660         srp_remove_host(target->scsi_host);
 661         scsi_remove_host(target->scsi_host);
 662         srp_disconnect_target(target);
 663         ib_destroy_cm_id(target->cm_id);
 664         srp_free_target_ib(target);
 665         cancel_work_sync(&target->tl_err_work);
 666         srp_rport_put(target->rport);
 667         srp_free_req_data(target);
 668         scsi_host_put(target->scsi_host);
 669 }
 670
 671 static void srp_remove_work(struct work_struct *work)
 672 {
 673         struct srp_target_port *target =
 674                 container_of(work, struct srp_target_port, remove_work);
 675
 676         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 677
 678         srp_remove_target(target);
 679
 680         spin_lock(&target->srp_host->target_lock);
 681         list_del(&target->list);
 682         spin_unlock(&target->srp_host->target_lock);
 683 }
 684
 685 static void srp_rport_delete(struct srp_rport *rport)
 686 {
 687         struct srp_target_port *target = rport->lld_data;
 688
 689         srp_queue_remove_work(target);
 690 }
 691
 692 static int srp_connect_target(struct srp_target_port *target)
 693 {
 694         int retries = 3;
 695         int ret;
 696
 697         WARN_ON_ONCE(target->connected);
 698
 699         target->qp_in_error = false;
 700
 701         ret = srp_lookup_path(target);
 702         if (ret)
 703                 return ret;
 704
 705         while (1) {
 706                 init_completion(&target->done);
 707                 ret = srp_send_req(target);
 708                 if (ret)
 709                         return ret;
 710                 wait_for_completion(&target->done);
 711
 712                 /*
 713                  * The CM event handling code will set status to
 714                  * SRP_PORT_REDIRECT if we get a port redirect REJ
 715                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
 716                  * redirect REJ back.
 717                  */
 718                 switch (target->status) {
 719                 case 0:
 720                         srp_change_conn_state(target, true);
 721                         return 0;
 722
 723                 case SRP_PORT_REDIRECT:
 724                         ret = srp_lookup_path(target);
 725                         if (ret)
 726                                 return ret;
 727                         break;
 728
 729                 case SRP_DLID_REDIRECT:
 730                         break;
 731
 732                 case SRP_STALE_CONN:
 733                         /* Our current CM id was stale, and is now in timewait.
 734                          * Try to reconnect with a new one.
 735                          */
 736                         if (!retries-- || srp_new_cm_id(target)) {
 737                                 shost_printk(KERN_ERR, target->scsi_host, PFX
 738                                              "giving up on stale connection\n");
 739                                 target->status = -ECONNRESET;
 740                                 return target->status;
 741                         }
 742
 743                         shost_printk(KERN_ERR, target->scsi_host, PFX
 744                                      "retrying stale connection\n");
 745                         break;
 746
 747                 default:
 748                         return target->status;
 749                 }
 750         }
 751 }
 752
 753 static void srp_unmap_data(struct scsi_cmnd *scmnd,
 754                            struct srp_target_port *target,
 755                            struct srp_request *req)
 756 {
 757         struct ib_device *ibdev = target->srp_host->srp_dev->dev;
 758         struct ib_pool_fmr **pfmr;
 759
 760         if (!scsi_sglist(scmnd) ||
 761             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
 762              scmnd->sc_data_direction != DMA_FROM_DEVICE))
 763                 return;
 764
 765         pfmr = req->fmr_list;
 766         while (req->nfmr--)
 767                 ib_fmr_pool_unmap(*pfmr++);
 768
 769         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
 770                         scmnd->sc_data_direction);
 771 }
 772
 773 /**
 774  * srp_claim_req - Take ownership of the scmnd associated with a request.
 775  * @target: SRP target port.
 776  * @req: SRP request.
 777  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
 778  *         ownership of @req->scmnd if it equals @scmnd.
 779  *
 780  * Return value:
 781  * Either NULL or a pointer to the SCSI command the caller became owner of.
 782  */
 783 static struct scsi_cmnd *srp_claim_req(struct srp_target_port *target,
 784                                        struct srp_request *req,
 785                                        struct scsi_cmnd *scmnd)
 786 {
 787         unsigned long flags;
 788
 789         spin_lock_irqsave(&target->lock, flags);
 790         if (!scmnd) {
 791                 scmnd = req->scmnd;
 792                 req->scmnd = NULL;
 793         } else if (req->scmnd == scmnd) {
 794                 req->scmnd = NULL;
 795         } else {
 796                 scmnd = NULL;
 797         }
 798         spin_unlock_irqrestore(&target->lock, flags);
 799
 800         return scmnd;
 801 }
 802
 803 /**
 804  * srp_free_req() - Unmap data and add request to the free request list.
 805  */
 806 static void srp_free_req(struct srp_target_port *target,
 807                          struct srp_request *req, struct scsi_cmnd *scmnd,
 808                          s32 req_lim_delta)
 809 {
 810         unsigned long flags;
 811
 812         srp_unmap_data(scmnd, target, req);
 813
 814         spin_lock_irqsave(&target->lock, flags);
 815         target->req_lim += req_lim_delta;
 816         list_add_tail(&req->list, &target->free_reqs);
 817         spin_unlock_irqrestore(&target->lock, flags);
 818 }
 819
 820 static void srp_finish_req(struct srp_target_port *target,
 821                            struct srp_request *req, int result)
 822 {
 823         struct scsi_cmnd *scmnd = srp_claim_req(target, req, NULL);
 824
 825         if (scmnd) {
 826                 srp_free_req(target, req, scmnd, 0);
 827                 scmnd->result = result;
 828                 scmnd->scsi_done(scmnd);
 829         }
 830 }
 831
 832 static void srp_terminate_io(struct srp_rport *rport)
 833 {
 834         struct srp_target_port *target = rport->lld_data;
 835         int i;
 836
 837         for (i = 0; i < target->req_ring_size; ++i) {
 838                 struct srp_request *req = &target->req_ring[i];
 839                 srp_finish_req(target, req, DID_TRANSPORT_FAILFAST << 16);
 840         }
 841 }
 842
 843 /*
 844  * It is up to the caller to ensure that srp_rport_reconnect() calls are
 845  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
 846  * srp_reset_device() or srp_reset_host() calls will occur while this function
 847  * is in progress. One way to realize that is not to call this function
 848  * directly but to call srp_reconnect_rport() instead since that last function
 849  * serializes calls of this function via rport->mutex and also blocks
 850  * srp_queuecommand() calls before invoking this function.
 851  */
 852 static int srp_rport_reconnect(struct srp_rport *rport)
 853 {
 854         struct srp_target_port *target = rport->lld_data;
 855         int i, ret;
 856
 857         srp_disconnect_target(target);
 858         /*
 859          * Now get a new local CM ID so that we avoid confusing the target in
 860          * case things are really fouled up. Doing so also ensures that all CM
 861          * callbacks will have finished before a new QP is allocated.
 862          */
 863         ret = srp_new_cm_id(target);
 864         /*
 865          * Whether or not creating a new CM ID succeeded, create a new
 866          * QP. This guarantees that all completion callback function
 867          * invocations have finished before request resetting starts.
 868          */
 869         if (ret == 0)
 870                 ret = srp_create_target_ib(target);
 871         else
 872                 srp_create_target_ib(target);
 873
 874         for (i = 0; i < target->req_ring_size; ++i) {
 875                 struct srp_request *req = &target->req_ring[i];
 876                 srp_finish_req(target, req, DID_RESET << 16);
 877         }
 878
 879         INIT_LIST_HEAD(&target->free_tx);
 880         for (i = 0; i < target->queue_size; ++i)
 881                 list_add(&target->tx_ring[i]->list, &target->free_tx);
 882
 883         if (ret == 0)
 884                 ret = srp_connect_target(target);
 885
 886         if (ret == 0)
 887                 shost_printk(KERN_INFO, target->scsi_host,
 888                              PFX "reconnect succeeded\n");
 889
 890         return ret;
 891 }
 892
 893 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
 894                          unsigned int dma_len, u32 rkey)
 895 {
 896         struct srp_direct_buf *desc = state->desc;
 897
 898         desc->va = cpu_to_be64(dma_addr);
 899         desc->key = cpu_to_be32(rkey);
 900         desc->len = cpu_to_be32(dma_len);
 901
 902         state->total_len += dma_len;
 903         state->desc++;
 904         state->ndesc++;
 905 }
 906
 907 static int srp_map_finish_fmr(struct srp_map_state *state,
 908                               struct srp_target_port *target)
 909 {
 910         struct srp_device *dev = target->srp_host->srp_dev;
 911         struct ib_pool_fmr *fmr;
 912         u64 io_addr = 0;
 913
 914         if (!state->npages)
 915                 return 0;
 916
 917         if (state->npages == 1) {
 918                 srp_map_desc(state, state->base_dma_addr, state->fmr_len,
 919                              target->rkey);
 920                 state->npages = state->fmr_len = 0;
 921                 return 0;
 922         }
 923
 924         fmr = ib_fmr_pool_map_phys(dev->fmr_pool, state->pages,
 925                                    state->npages, io_addr);
 926         if (IS_ERR(fmr))
 927                 return PTR_ERR(fmr);
 928
 929         *state->next_fmr++ = fmr;
 930         state->nfmr++;
 931
 932         srp_map_desc(state, 0, state->fmr_len, fmr->fmr->rkey);
 933         state->npages = state->fmr_len = 0;
 934         return 0;
 935 }
 936
 937 static void srp_map_update_start(struct srp_map_state *state,
 938                                  struct scatterlist *sg, int sg_index,
 939                                  dma_addr_t dma_addr)
 940 {
 941         state->unmapped_sg = sg;
 942         state->unmapped_index = sg_index;
 943         state->unmapped_addr = dma_addr;
 944 }
 945
 946 static int srp_map_sg_entry(struct srp_map_state *state,
 947                             struct srp_target_port *target,
 948                             struct scatterlist *sg, int sg_index,
 949                             int use_fmr)
 950 {
 951         struct srp_device *dev = target->srp_host->srp_dev;
 952         struct ib_device *ibdev = dev->dev;
 953         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
 954         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
 955         unsigned int len;
 956         int ret;
 957
 958         if (!dma_len)
 959                 return 0;
 960
 961         if (use_fmr == SRP_MAP_NO_FMR) {
 962                 /* Once we're in direct map mode for a request, we don't
 963                  * go back to FMR mode, so no need to update anything
 964                  * other than the descriptor.
 965                  */
 966                 srp_map_desc(state, dma_addr, dma_len, target->rkey);
 967                 return 0;
 968         }
 969
 970         /* If we start at an offset into the FMR page, don't merge into
 971          * the current FMR. Finish it out, and use the kernel's MR for this
 972          * sg entry. This is to avoid potential bugs on some SRP targets
 973          * that were never quite defined, but went away when the initiator
 974          * avoided using FMR on such page fragments.
 975          */
 976         if (dma_addr & ~dev->fmr_page_mask || dma_len > dev->fmr_max_size) {
 977                 ret = srp_map_finish_fmr(state, target);
 978                 if (ret)
 979                         return ret;
 980
 981                 srp_map_desc(state, dma_addr, dma_len, target->rkey);
 982                 srp_map_update_start(state, NULL, 0, 0);
 983                 return 0;
 984         }
 985
 986         /* If this is the first sg to go into the FMR, save our position.
 987          * We need to know the first unmapped entry, its index, and the
 988          * first unmapped address within that entry to be able to restart
 989          * mapping after an error.
 990          */
 991         if (!state->unmapped_sg)
 992                 srp_map_update_start(state, sg, sg_index, dma_addr);
 993
 994         while (dma_len) {
 995                 if (state->npages == SRP_FMR_SIZE) {
 996                         ret = srp_map_finish_fmr(state, target);
 997                         if (ret)
 998                                 return ret;
 999
1000                         srp_map_update_start(state, sg, sg_index, dma_addr);
1001                 }
1002
1003                 len = min_t(unsigned int, dma_len, dev->fmr_page_size);
1004
1005                 if (!state->npages)
1006                         state->base_dma_addr = dma_addr;
1007                 state->pages[state->npages++] = dma_addr;
1008                 state->fmr_len += len;
1009                 dma_addr += len;
1010                 dma_len -= len;
1011         }
1012
1013         /* If the last entry of the FMR wasn't a full page, then we need to
1014          * close it out and start a new one -- we can only merge at page
1015          * boundries.
1016          */
1017         ret = 0;
1018         if (len != dev->fmr_page_size) {
1019                 ret = srp_map_finish_fmr(state, target);
1020                 if (!ret)
1021                         srp_map_update_start(state, NULL, 0, 0);
1022         }
1023         return ret;
1024 }
1025
1026 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target,
1027                         struct srp_request *req)
1028 {
1029         struct scatterlist *scat, *sg;
1030         struct srp_cmd *cmd = req->cmd->buf;
1031         int i, len, nents, count, use_fmr;
1032         struct srp_device *dev;
1033         struct ib_device *ibdev;
1034         struct srp_map_state state;
1035         struct srp_indirect_buf *indirect_hdr;
1036         u32 table_len;
1037         u8 fmt;
1038
1039         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1040                 return sizeof (struct srp_cmd);
1041
1042         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1043             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1044                 shost_printk(KERN_WARNING, target->scsi_host,
1045                              PFX "Unhandled data direction %d\n",
1046                              scmnd->sc_data_direction);
1047                 return -EINVAL;
1048         }
1049
1050         nents = scsi_sg_count(scmnd);
1051         scat  = scsi_sglist(scmnd);
1052
1053         dev = target->srp_host->srp_dev;
1054         ibdev = dev->dev;
1055
1056         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1057         if (unlikely(count == 0))
1058                 return -EIO;
1059
1060         fmt = SRP_DATA_DESC_DIRECT;
1061         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1062
1063         if (count == 1) {
1064                 /*
1065                  * The midlayer only generated a single gather/scatter
1066                  * entry, or DMA mapping coalesced everything to a
1067                  * single entry.  So a direct descriptor along with
1068                  * the DMA MR suffices.
1069                  */
1070                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1071
1072                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1073                 buf->key = cpu_to_be32(target->rkey);
1074                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1075
1076                 req->nfmr = 0;
1077                 goto map_complete;
1078         }
1079
1080         /* We have more than one scatter/gather entry, so build our indirect
1081          * descriptor table, trying to merge as many entries with FMR as we
1082          * can.
1083          */
1084         indirect_hdr = (void *) cmd->add_data;
1085
1086         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1087                                    target->indirect_size, DMA_TO_DEVICE);
1088
1089         memset(&state, 0, sizeof(state));
1090         state.desc      = req->indirect_desc;
1091         state.pages     = req->map_page;
1092         state.next_fmr  = req->fmr_list;
1093
1094         use_fmr = dev->fmr_pool ? SRP_MAP_ALLOW_FMR : SRP_MAP_NO_FMR;
1095
1096         for_each_sg(scat, sg, count, i) {
1097                 if (srp_map_sg_entry(&state, target, sg, i, use_fmr)) {
1098                         /* FMR mapping failed, so backtrack to the first
1099                          * unmapped entry and continue on without using FMR.
1100                          */
1101                         dma_addr_t dma_addr;
1102                         unsigned int dma_len;
1103
1104 backtrack:
1105                         sg = state.unmapped_sg;
1106                         i = state.unmapped_index;
1107
1108                         dma_addr = ib_sg_dma_address(ibdev, sg);
1109                         dma_len = ib_sg_dma_len(ibdev, sg);
1110                         dma_len -= (state.unmapped_addr - dma_addr);
1111                         dma_addr = state.unmapped_addr;
1112                         use_fmr = SRP_MAP_NO_FMR;
1113                         srp_map_desc(&state, dma_addr, dma_len, target->rkey);
1114                 }
1115         }
1116
1117         if (use_fmr == SRP_MAP_ALLOW_FMR && srp_map_finish_fmr(&state, target))
1118                 goto backtrack;
1119
1120         /* We've mapped the request, now pull as much of the indirect
1121          * descriptor table as we can into the command buffer. If this
1122          * target is not using an external indirect table, we are
1123          * guaranteed to fit into the command, as the SCSI layer won't
1124          * give us more S/G entries than we allow.
1125          */
1126         req->nfmr = state.nfmr;
1127         if (state.ndesc == 1) {
1128                 /* FMR mapping was able to collapse this to one entry,
1129                  * so use a direct descriptor.
1130                  */
1131                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1132
1133                 *buf = req->indirect_desc[0];
1134                 goto map_complete;
1135         }
1136
1137         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1138                                                 !target->allow_ext_sg)) {
1139                 shost_printk(KERN_ERR, target->scsi_host,
1140                              "Could not fit S/G list into SRP_CMD\n");
1141                 return -EIO;
1142         }
1143
1144         count = min(state.ndesc, target->cmd_sg_cnt);
1145         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1146
1147         fmt = SRP_DATA_DESC_INDIRECT;
1148         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1149         len += count * sizeof (struct srp_direct_buf);
1150
1151         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1152                count * sizeof (struct srp_direct_buf));
1153
1154         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1155         indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1156         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1157         indirect_hdr->len = cpu_to_be32(state.total_len);
1158
1159         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1160                 cmd->data_out_desc_cnt = count;
1161         else
1162                 cmd->data_in_desc_cnt = count;
1163
1164         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1165                                       DMA_TO_DEVICE);
1166
1167 map_complete:
1168         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1169                 cmd->buf_fmt = fmt << 4;
1170         else
1171                 cmd->buf_fmt = fmt;
1172
1173         return len;
1174 }
1175
1176 /*
1177  * Return an IU and possible credit to the free pool
1178  */
1179 static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu,
1180                           enum srp_iu_type iu_type)
1181 {
1182         unsigned long flags;
1183
1184         spin_lock_irqsave(&target->lock, flags);
1185         list_add(&iu->list, &target->free_tx);
1186         if (iu_type != SRP_IU_RSP)
1187                 ++target->req_lim;
1188         spin_unlock_irqrestore(&target->lock, flags);
1189 }
1190
1191 /*
1192  * Must be called with target->lock held to protect req_lim and free_tx.
1193  * If IU is not sent, it must be returned using srp_put_tx_iu().
1194  *
1195  * Note:
1196  * An upper limit for the number of allocated information units for each
1197  * request type is:
1198  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1199  *   more than Scsi_Host.can_queue requests.
1200  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1201  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1202  *   one unanswered SRP request to an initiator.
1203  */
1204 static struct srp_iu *__srp_get_tx_iu(struct srp_target_port *target,
1205                                       enum srp_iu_type iu_type)
1206 {
1207         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1208         struct srp_iu *iu;
1209
1210         srp_send_completion(target->send_cq, target);
1211
1212         if (list_empty(&target->free_tx))
1213                 return NULL;
1214
1215         /* Initiator responses to target requests do not consume credits */
1216         if (iu_type != SRP_IU_RSP) {
1217                 if (target->req_lim <= rsv) {
1218                         ++target->zero_req_lim;
1219                         return NULL;
1220                 }
1221
1222                 --target->req_lim;
1223         }
1224
1225         iu = list_first_entry(&target->free_tx, struct srp_iu, list);
1226         list_del(&iu->list);
1227         return iu;
1228 }
1229
1230 static int srp_post_send(struct srp_target_port *target,
1231                          struct srp_iu *iu, int len)
1232 {
1233         struct ib_sge list;
1234         struct ib_send_wr wr, *bad_wr;
1235
1236         list.addr   = iu->dma;
1237         list.length = len;
1238         list.lkey   = target->lkey;
1239
1240         wr.next       = NULL;
1241         wr.wr_id      = (uintptr_t) iu;
1242         wr.sg_list    = &list;
1243         wr.num_sge    = 1;
1244         wr.opcode     = IB_WR_SEND;
1245         wr.send_flags = IB_SEND_SIGNALED;
1246
1247         return ib_post_send(target->qp, &wr, &bad_wr);
1248 }
1249
1250 static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu)
1251 {
1252         struct ib_recv_wr wr, *bad_wr;
1253         struct ib_sge list;
1254
1255         list.addr   = iu->dma;
1256         list.length = iu->size;
1257         list.lkey   = target->lkey;
1258
1259         wr.next     = NULL;
1260         wr.wr_id    = (uintptr_t) iu;
1261         wr.sg_list  = &list;
1262         wr.num_sge  = 1;
1263
1264         return ib_post_recv(target->qp, &wr, &bad_wr);
1265 }
1266
1267 static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp)
1268 {
1269         struct srp_request *req;
1270         struct scsi_cmnd *scmnd;
1271         unsigned long flags;
1272
1273         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1274                 spin_lock_irqsave(&target->lock, flags);
1275                 target->req_lim += be32_to_cpu(rsp->req_lim_delta);
1276                 spin_unlock_irqrestore(&target->lock, flags);
1277
1278                 target->tsk_mgmt_status = -1;
1279                 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1280                         target->tsk_mgmt_status = rsp->data[3];
1281                 complete(&target->tsk_mgmt_done);
1282         } else {
1283                 req = &target->req_ring[rsp->tag];
1284                 scmnd = srp_claim_req(target, req, NULL);
1285                 if (!scmnd) {
1286                         shost_printk(KERN_ERR, target->scsi_host,
1287                                      "Null scmnd for RSP w/tag %016llx\n",
1288                                      (unsigned long long) rsp->tag);
1289
1290                         spin_lock_irqsave(&target->lock, flags);
1291                         target->req_lim += be32_to_cpu(rsp->req_lim_delta);
1292                         spin_unlock_irqrestore(&target->lock, flags);
1293
1294                         return;
1295                 }
1296                 scmnd->result = rsp->status;
1297
1298                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1299                         memcpy(scmnd->sense_buffer, rsp->data +
1300                                be32_to_cpu(rsp->resp_data_len),
1301                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1302                                      SCSI_SENSE_BUFFERSIZE));
1303                 }
1304
1305                 if (rsp->flags & (SRP_RSP_FLAG_DOOVER | SRP_RSP_FLAG_DOUNDER))
1306                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1307                 else if (rsp->flags & (SRP_RSP_FLAG_DIOVER | SRP_RSP_FLAG_DIUNDER))
1308                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1309
1310                 srp_free_req(target, req, scmnd,
1311                              be32_to_cpu(rsp->req_lim_delta));
1312
1313                 scmnd->host_scribble = NULL;
1314                 scmnd->scsi_done(scmnd);
1315         }
1316 }
1317
1318 static int srp_response_common(struct srp_target_port *target, s32 req_delta,
1319                                void *rsp, int len)
1320 {
1321         struct ib_device *dev = target->srp_host->srp_dev->dev;
1322         unsigned long flags;
1323         struct srp_iu *iu;
1324         int err;
1325
1326         spin_lock_irqsave(&target->lock, flags);
1327         target->req_lim += req_delta;
1328         iu = __srp_get_tx_iu(target, SRP_IU_RSP);
1329         spin_unlock_irqrestore(&target->lock, flags);
1330
1331         if (!iu) {
1332                 shost_printk(KERN_ERR, target->scsi_host, PFX
1333                              "no IU available to send response\n");
1334                 return 1;
1335         }
1336
1337         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1338         memcpy(iu->buf, rsp, len);
1339         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1340
1341         err = srp_post_send(target, iu, len);
1342         if (err) {
1343                 shost_printk(KERN_ERR, target->scsi_host, PFX
1344                              "unable to post response: %d\n", err);
1345                 srp_put_tx_iu(target, iu, SRP_IU_RSP);
1346         }
1347
1348         return err;
1349 }
1350
1351 static void srp_process_cred_req(struct srp_target_port *target,
1352                                  struct srp_cred_req *req)
1353 {
1354         struct srp_cred_rsp rsp = {
1355                 .opcode = SRP_CRED_RSP,
1356                 .tag = req->tag,
1357         };
1358         s32 delta = be32_to_cpu(req->req_lim_delta);
1359
1360         if (srp_response_common(target, delta, &rsp, sizeof rsp))
1361                 shost_printk(KERN_ERR, target->scsi_host, PFX
1362                              "problems processing SRP_CRED_REQ\n");
1363 }
1364
1365 static void srp_process_aer_req(struct srp_target_port *target,
1366                                 struct srp_aer_req *req)
1367 {
1368         struct srp_aer_rsp rsp = {
1369                 .opcode = SRP_AER_RSP,
1370                 .tag = req->tag,
1371         };
1372         s32 delta = be32_to_cpu(req->req_lim_delta);
1373
1374         shost_printk(KERN_ERR, target->scsi_host, PFX
1375                      "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
1376
1377         if (srp_response_common(target, delta, &rsp, sizeof rsp))
1378                 shost_printk(KERN_ERR, target->scsi_host, PFX
1379                              "problems processing SRP_AER_REQ\n");
1380 }
1381
1382 static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc)
1383 {
1384         struct ib_device *dev = target->srp_host->srp_dev->dev;
1385         struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
1386         int res;
1387         u8 opcode;
1388
1389         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_ti_iu_len,
1390                                    DMA_FROM_DEVICE);
1391
1392         opcode = *(u8 *) iu->buf;
1393
1394         if (0) {
1395                 shost_printk(KERN_ERR, target->scsi_host,
1396                              PFX "recv completion, opcode 0x%02x\n", opcode);
1397                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1398                                iu->buf, wc->byte_len, true);
1399         }
1400
1401         switch (opcode) {
1402         case SRP_RSP:
1403                 srp_process_rsp(target, iu->buf);
1404                 break;
1405
1406         case SRP_CRED_REQ:
1407                 srp_process_cred_req(target, iu->buf);
1408                 break;
1409
1410         case SRP_AER_REQ:
1411                 srp_process_aer_req(target, iu->buf);
1412                 break;
1413
1414         case SRP_T_LOGOUT:
1415                 /* XXX Handle target logout */
1416                 shost_printk(KERN_WARNING, target->scsi_host,
1417                              PFX "Got target logout request\n");
1418                 break;
1419
1420         default:
1421                 shost_printk(KERN_WARNING, target->scsi_host,
1422                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1423                 break;
1424         }
1425
1426         ib_dma_sync_single_for_device(dev, iu->dma, target->max_ti_iu_len,
1427                                       DMA_FROM_DEVICE);
1428
1429         res = srp_post_recv(target, iu);
1430         if (res != 0)
1431                 shost_printk(KERN_ERR, target->scsi_host,
1432                              PFX "Recv failed with error code %d\n", res);
1433 }
1434
1435 /**
1436  * srp_tl_err_work() - handle a transport layer error
1437  *
1438  * Note: This function may get invoked before the rport has been created,
1439  * hence the target->rport test.
1440  */
1441 static void srp_tl_err_work(struct work_struct *work)
1442 {
1443         struct srp_target_port *target;
1444
1445         target = container_of(work, struct srp_target_port, tl_err_work);
1446         if (target->rport)
1447                 srp_start_tl_fail_timers(target->rport);
1448 }
1449
1450 static void srp_handle_qp_err(enum ib_wc_status wc_status,
1451                               enum ib_wc_opcode wc_opcode,
1452                               struct srp_target_port *target)
1453 {
1454         if (target->connected && !target->qp_in_error) {
1455                 shost_printk(KERN_ERR, target->scsi_host,
1456                              PFX "failed %s status %d\n",
1457                              wc_opcode & IB_WC_RECV ? "receive" : "send",
1458                              wc_status);
1459                 queue_work(system_long_wq, &target->tl_err_work);
1460         }
1461         target->qp_in_error = true;
1462 }
1463
1464 static void srp_recv_completion(struct ib_cq *cq, void *target_ptr)
1465 {
1466         struct srp_target_port *target = target_ptr;
1467         struct ib_wc wc;
1468
1469         ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1470         while (ib_poll_cq(cq, 1, &wc) > 0) {
1471                 if (likely(wc.status == IB_WC_SUCCESS)) {
1472                         srp_handle_recv(target, &wc);
1473                 } else {
1474                         srp_handle_qp_err(wc.status, wc.opcode, target);
1475                 }
1476         }
1477 }
1478
1479 static void srp_send_completion(struct ib_cq *cq, void *target_ptr)
1480 {
1481         struct srp_target_port *target = target_ptr;
1482         struct ib_wc wc;
1483         struct srp_iu *iu;
1484
1485         while (ib_poll_cq(cq, 1, &wc) > 0) {
1486                 if (likely(wc.status == IB_WC_SUCCESS)) {
1487                         iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
1488                         list_add(&iu->list, &target->free_tx);
1489                 } else {
1490                         srp_handle_qp_err(wc.status, wc.opcode, target);
1491                 }
1492         }
1493 }
1494
1495 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
1496 {
1497         struct srp_target_port *target = host_to_target(shost);
1498         struct srp_rport *rport = target->rport;
1499         struct srp_request *req;
1500         struct srp_iu *iu;
1501         struct srp_cmd *cmd;
1502         struct ib_device *dev;
1503         unsigned long flags;
1504         int len, result;
1505         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1506
1507         /*
1508          * The SCSI EH thread is the only context from which srp_queuecommand()
1509          * can get invoked for blocked devices (SDEV_BLOCK /
1510          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
1511          * locking the rport mutex if invoked from inside the SCSI EH.
1512          */
1513         if (in_scsi_eh)
1514                 mutex_lock(&rport->mutex);
1515
1516         result = srp_chkready(target->rport);
1517         if (unlikely(result)) {
1518                 scmnd->result = result;
1519                 scmnd->scsi_done(scmnd);
1520                 goto unlock_rport;
1521         }
1522
1523         spin_lock_irqsave(&target->lock, flags);
1524         iu = __srp_get_tx_iu(target, SRP_IU_CMD);
1525         if (!iu)
1526                 goto err_unlock;
1527
1528         req = list_first_entry(&target->free_reqs, struct srp_request, list);
1529         list_del(&req->list);
1530         spin_unlock_irqrestore(&target->lock, flags);
1531
1532         dev = target->srp_host->srp_dev->dev;
1533         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
1534                                    DMA_TO_DEVICE);
1535
1536         scmnd->result        = 0;
1537         scmnd->host_scribble = (void *) req;
1538
1539         cmd = iu->buf;
1540         memset(cmd, 0, sizeof *cmd);
1541
1542         cmd->opcode = SRP_CMD;
1543         cmd->lun    = cpu_to_be64((u64) scmnd->device->lun << 48);
1544         cmd->tag    = req->index;
1545         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
1546
1547         req->scmnd    = scmnd;
1548         req->cmd      = iu;
1549
1550         len = srp_map_data(scmnd, target, req);
1551         if (len < 0) {
1552                 shost_printk(KERN_ERR, target->scsi_host,
1553                              PFX "Failed to map data\n");
1554                 goto err_iu;
1555         }
1556
1557         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
1558                                       DMA_TO_DEVICE);
1559
1560         if (srp_post_send(target, iu, len)) {
1561                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
1562                 goto err_unmap;
1563         }
1564
1565 unlock_rport:
1566         if (in_scsi_eh)
1567                 mutex_unlock(&rport->mutex);
1568
1569         return 0;
1570
1571 err_unmap:
1572         srp_unmap_data(scmnd, target, req);
1573
1574 err_iu:
1575         srp_put_tx_iu(target, iu, SRP_IU_CMD);
1576
1577         spin_lock_irqsave(&target->lock, flags);
1578         list_add(&req->list, &target->free_reqs);
1579
1580 err_unlock:
1581         spin_unlock_irqrestore(&target->lock, flags);
1582
1583         if (in_scsi_eh)
1584                 mutex_unlock(&rport->mutex);
1585
1586         return SCSI_MLQUEUE_HOST_BUSY;
1587 }
1588
1589 /*
1590  * Note: the resources allocated in this function are freed in
1591  * srp_free_target_ib().
1592  */
1593 static int srp_alloc_iu_bufs(struct srp_target_port *target)
1594 {
1595         int i;
1596
1597         target->rx_ring = kzalloc(target->queue_size * sizeof(*target->rx_ring),
1598                                   GFP_KERNEL);
1599         if (!target->rx_ring)
1600                 goto err_no_ring;
1601         target->tx_ring = kzalloc(target->queue_size * sizeof(*target->tx_ring),
1602                                   GFP_KERNEL);
1603         if (!target->tx_ring)
1604                 goto err_no_ring;
1605
1606         for (i = 0; i < target->queue_size; ++i) {
1607                 target->rx_ring[i] = srp_alloc_iu(target->srp_host,
1608                                                   target->max_ti_iu_len,
1609                                                   GFP_KERNEL, DMA_FROM_DEVICE);
1610                 if (!target->rx_ring[i])
1611                         goto err;
1612         }
1613
1614         for (i = 0; i < target->queue_size; ++i) {
1615                 target->tx_ring[i] = srp_alloc_iu(target->srp_host,
1616                                                   target->max_iu_len,
1617                                                   GFP_KERNEL, DMA_TO_DEVICE);
1618                 if (!target->tx_ring[i])
1619                         goto err;
1620
1621                 list_add(&target->tx_ring[i]->list, &target->free_tx);
1622         }
1623
1624         return 0;
1625
1626 err:
1627         for (i = 0; i < target->queue_size; ++i) {
1628                 srp_free_iu(target->srp_host, target->rx_ring[i]);
1629                 srp_free_iu(target->srp_host, target->tx_ring[i]);
1630         }
1631
1632
1633 err_no_ring:
1634         kfree(target->tx_ring);
1635         target->tx_ring = NULL;
1636         kfree(target->rx_ring);
1637         target->rx_ring = NULL;
1638
1639         return -ENOMEM;
1640 }
1641
1642 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
1643 {
1644         uint64_t T_tr_ns, max_compl_time_ms;
1645         uint32_t rq_tmo_jiffies;
1646
1647         /*
1648          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
1649          * table 91), both the QP timeout and the retry count have to be set
1650          * for RC QP's during the RTR to RTS transition.
1651          */
1652         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
1653                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
1654
1655         /*
1656          * Set target->rq_tmo_jiffies to one second more than the largest time
1657          * it can take before an error completion is generated. See also
1658          * C9-140..142 in the IBTA spec for more information about how to
1659          * convert the QP Local ACK Timeout value to nanoseconds.
1660          */
1661         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
1662         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
1663         do_div(max_compl_time_ms, NSEC_PER_MSEC);
1664         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
1665
1666         return rq_tmo_jiffies;
1667 }
1668
1669 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
1670                                struct srp_login_rsp *lrsp,
1671                                struct srp_target_port *target)
1672 {
1673         struct ib_qp_attr *qp_attr = NULL;
1674         int attr_mask = 0;
1675         int ret;
1676         int i;
1677
1678         if (lrsp->opcode == SRP_LOGIN_RSP) {
1679                 target->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
1680                 target->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
1681
1682                 /*
1683                  * Reserve credits for task management so we don't
1684                  * bounce requests back to the SCSI mid-layer.
1685                  */
1686                 target->scsi_host->can_queue
1687                         = min(target->req_lim - SRP_TSK_MGMT_SQ_SIZE,
1688                               target->scsi_host->can_queue);
1689                 target->scsi_host->cmd_per_lun
1690                         = min_t(int, target->scsi_host->can_queue,
1691                                 target->scsi_host->cmd_per_lun);
1692         } else {
1693                 shost_printk(KERN_WARNING, target->scsi_host,
1694                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
1695                 ret = -ECONNRESET;
1696                 goto error;
1697         }
1698
1699         if (!target->rx_ring) {
1700                 ret = srp_alloc_iu_bufs(target);
1701                 if (ret)
1702                         goto error;
1703         }
1704
1705         ret = -ENOMEM;
1706         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
1707         if (!qp_attr)
1708                 goto error;
1709
1710         qp_attr->qp_state = IB_QPS_RTR;
1711         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
1712         if (ret)
1713                 goto error_free;
1714
1715         ret = ib_modify_qp(target->qp, qp_attr, attr_mask);
1716         if (ret)
1717                 goto error_free;
1718
1719         for (i = 0; i < target->queue_size; i++) {
1720                 struct srp_iu *iu = target->rx_ring[i];
1721                 ret = srp_post_recv(target, iu);
1722                 if (ret)
1723                         goto error_free;
1724         }
1725
1726         qp_attr->qp_state = IB_QPS_RTS;
1727         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
1728         if (ret)
1729                 goto error_free;
1730
1731         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
1732
1733         ret = ib_modify_qp(target->qp, qp_attr, attr_mask);
1734         if (ret)
1735                 goto error_free;
1736
1737         ret = ib_send_cm_rtu(cm_id, NULL, 0);
1738
1739 error_free:
1740         kfree(qp_attr);
1741
1742 error:
1743         target->status = ret;
1744 }
1745
1746 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
1747                                struct ib_cm_event *event,
1748                                struct srp_target_port *target)
1749 {
1750         struct Scsi_Host *shost = target->scsi_host;
1751         struct ib_class_port_info *cpi;
1752         int opcode;
1753
1754         switch (event->param.rej_rcvd.reason) {
1755         case IB_CM_REJ_PORT_CM_REDIRECT:
1756                 cpi = event->param.rej_rcvd.ari;
1757                 target->path.dlid = cpi->redirect_lid;
1758                 target->path.pkey = cpi->redirect_pkey;
1759                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
1760                 memcpy(target->path.dgid.raw, cpi->redirect_gid, 16);
1761
1762                 target->status = target->path.dlid ?
1763                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
1764                 break;
1765
1766         case IB_CM_REJ_PORT_REDIRECT:
1767                 if (srp_target_is_topspin(target)) {
1768                         /*
1769                          * Topspin/Cisco SRP gateways incorrectly send
1770                          * reject reason code 25 when they mean 24
1771                          * (port redirect).
1772                          */
1773                         memcpy(target->path.dgid.raw,
1774                                event->param.rej_rcvd.ari, 16);
1775
1776                         shost_printk(KERN_DEBUG, shost,
1777                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
1778                                      (unsigned long long) be64_to_cpu(target->path.dgid.global.subnet_prefix),
1779                                      (unsigned long long) be64_to_cpu(target->path.dgid.global.interface_id));
1780
1781                         target->status = SRP_PORT_REDIRECT;
1782                 } else {
1783                         shost_printk(KERN_WARNING, shost,
1784                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
1785                         target->status = -ECONNRESET;
1786                 }
1787                 break;
1788
1789         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
1790                 shost_printk(KERN_WARNING, shost,
1791                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
1792                 target->status = -ECONNRESET;
1793                 break;
1794
1795         case IB_CM_REJ_CONSUMER_DEFINED:
1796                 opcode = *(u8 *) event->private_data;
1797                 if (opcode == SRP_LOGIN_REJ) {
1798                         struct srp_login_rej *rej = event->private_data;
1799                         u32 reason = be32_to_cpu(rej->reason);
1800
1801                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
1802                                 shost_printk(KERN_WARNING, shost,
1803                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
1804                         else
1805                                 shost_printk(KERN_WARNING, shost,
1806                                             PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason);
1807                 } else
1808                         shost_printk(KERN_WARNING, shost,
1809                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
1810                                      " opcode 0x%02x\n", opcode);
1811                 target->status = -ECONNRESET;
1812                 break;
1813
1814         case IB_CM_REJ_STALE_CONN:
1815                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
1816                 target->status = SRP_STALE_CONN;
1817                 break;
1818
1819         default:
1820                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
1821                              event->param.rej_rcvd.reason);
1822                 target->status = -ECONNRESET;
1823         }
1824 }
1825
1826 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1827 {
1828         struct srp_target_port *target = cm_id->context;
1829         int comp = 0;
1830
1831         switch (event->event) {
1832         case IB_CM_REQ_ERROR:
1833                 shost_printk(KERN_DEBUG, target->scsi_host,
1834                              PFX "Sending CM REQ failed\n");
1835                 comp = 1;
1836                 target->status = -ECONNRESET;
1837                 break;
1838
1839         case IB_CM_REP_RECEIVED:
1840                 comp = 1;
1841                 srp_cm_rep_handler(cm_id, event->private_data, target);
1842                 break;
1843
1844         case IB_CM_REJ_RECEIVED:
1845                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
1846                 comp = 1;
1847
1848                 srp_cm_rej_handler(cm_id, event, target);
1849                 break;
1850
1851         case IB_CM_DREQ_RECEIVED:
1852                 shost_printk(KERN_WARNING, target->scsi_host,
1853                              PFX "DREQ received - connection closed\n");
1854                 srp_change_conn_state(target, false);
1855                 if (ib_send_cm_drep(cm_id, NULL, 0))
1856                         shost_printk(KERN_ERR, target->scsi_host,
1857                                      PFX "Sending CM DREP failed\n");
1858                 queue_work(system_long_wq, &target->tl_err_work);
1859                 break;
1860
1861         case IB_CM_TIMEWAIT_EXIT:
1862                 shost_printk(KERN_ERR, target->scsi_host,
1863                              PFX "connection closed\n");
1864
1865                 target->status = 0;
1866                 break;
1867
1868         case IB_CM_MRA_RECEIVED:
1869         case IB_CM_DREQ_ERROR:
1870         case IB_CM_DREP_RECEIVED:
1871                 break;
1872
1873         default:
1874                 shost_printk(KERN_WARNING, target->scsi_host,
1875                              PFX "Unhandled CM event %d\n", event->event);
1876                 break;
1877         }
1878
1879         if (comp)
1880                 complete(&target->done);
1881
1882         return 0;
1883 }
1884
1885 static int srp_send_tsk_mgmt(struct srp_target_port *target,
1886                              u64 req_tag, unsigned int lun, u8 func)
1887 {
1888         struct srp_rport *rport = target->rport;
1889         struct ib_device *dev = target->srp_host->srp_dev->dev;
1890         struct srp_iu *iu;
1891         struct srp_tsk_mgmt *tsk_mgmt;
1892
1893         if (!target->connected || target->qp_in_error)
1894                 return -1;
1895
1896         init_completion(&target->tsk_mgmt_done);
1897
1898         /*
1899          * Lock the rport mutex to avoid that srp_create_target_ib() is
1900          * invoked while a task management function is being sent.
1901          */
1902         mutex_lock(&rport->mutex);
1903         spin_lock_irq(&target->lock);
1904         iu = __srp_get_tx_iu(target, SRP_IU_TSK_MGMT);
1905         spin_unlock_irq(&target->lock);
1906
1907         if (!iu) {
1908                 mutex_unlock(&rport->mutex);
1909
1910                 return -1;
1911         }
1912
1913         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
1914                                    DMA_TO_DEVICE);
1915         tsk_mgmt = iu->buf;
1916         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
1917
1918         tsk_mgmt->opcode        = SRP_TSK_MGMT;
1919         tsk_mgmt->lun           = cpu_to_be64((u64) lun << 48);
1920         tsk_mgmt->tag           = req_tag | SRP_TAG_TSK_MGMT;
1921         tsk_mgmt->tsk_mgmt_func = func;
1922         tsk_mgmt->task_tag      = req_tag;
1923
1924         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
1925                                       DMA_TO_DEVICE);
1926         if (srp_post_send(target, iu, sizeof *tsk_mgmt)) {
1927                 srp_put_tx_iu(target, iu, SRP_IU_TSK_MGMT);
1928                 mutex_unlock(&rport->mutex);
1929
1930                 return -1;
1931         }
1932         mutex_unlock(&rport->mutex);
1933
1934         if (!wait_for_completion_timeout(&target->tsk_mgmt_done,
1935                                          msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
1936                 return -1;
1937
1938         return 0;
1939 }
1940
1941 static int srp_abort(struct scsi_cmnd *scmnd)
1942 {
1943         struct srp_target_port *target = host_to_target(scmnd->device->host);
1944         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
1945         int ret;
1946
1947         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
1948
1949         if (!req || !srp_claim_req(target, req, scmnd))
1950                 return FAILED;
1951         if (srp_send_tsk_mgmt(target, req->index, scmnd->device->lun,
1952                               SRP_TSK_ABORT_TASK) == 0)
1953                 ret = SUCCESS;
1954         else if (target->rport->state == SRP_RPORT_LOST)
1955                 ret = FAST_IO_FAIL;
1956         else
1957                 ret = FAILED;
1958         srp_free_req(target, req, scmnd, 0);
1959         scmnd->result = DID_ABORT << 16;
1960         scmnd->scsi_done(scmnd);
1961
1962         return ret;
1963 }
1964
1965 static int srp_reset_device(struct scsi_cmnd *scmnd)
1966 {
1967         struct srp_target_port *target = host_to_target(scmnd->device->host);
1968         int i;
1969
1970         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
1971
1972         if (srp_send_tsk_mgmt(target, SRP_TAG_NO_REQ, scmnd->device->lun,
1973                               SRP_TSK_LUN_RESET))
1974                 return FAILED;
1975         if (target->tsk_mgmt_status)
1976                 return FAILED;
1977
1978         for (i = 0; i < target->req_ring_size; ++i) {
1979                 struct srp_request *req = &target->req_ring[i];
1980                 if (req->scmnd && req->scmnd->device == scmnd->device)
1981                         srp_finish_req(target, req, DID_RESET << 16);
1982         }
1983
1984         return SUCCESS;
1985 }
1986
1987 static int srp_reset_host(struct scsi_cmnd *scmnd)
1988 {
1989         struct srp_target_port *target = host_to_target(scmnd->device->host);
1990
1991         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
1992
1993         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
1994 }
1995
1996 static int srp_slave_configure(struct scsi_device *sdev)
1997 {
1998         struct Scsi_Host *shost = sdev->host;
1999         struct srp_target_port *target = host_to_target(shost);
2000         struct request_queue *q = sdev->request_queue;
2001         unsigned long timeout;
2002
2003         if (sdev->type == TYPE_DISK) {
2004                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2005                 blk_queue_rq_timeout(q, timeout);
2006         }
2007
2008         return 0;
2009 }
2010
2011 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2012                            char *buf)
2013 {
2014         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2015
2016         return sprintf(buf, "0x%016llx\n",
2017                        (unsigned long long) be64_to_cpu(target->id_ext));
2018 }
2019
2020 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2021                              char *buf)
2022 {
2023         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2024
2025         return sprintf(buf, "0x%016llx\n",
2026                        (unsigned long long) be64_to_cpu(target->ioc_guid));
2027 }
2028
2029 static ssize_t show_service_id(struct device *dev,
2030                                struct device_attribute *attr, char *buf)
2031 {
2032         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2033
2034         return sprintf(buf, "0x%016llx\n",
2035                        (unsigned long long) be64_to_cpu(target->service_id));
2036 }
2037
2038 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2039                          char *buf)
2040 {
2041         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2042
2043         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->path.pkey));
2044 }
2045
2046 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2047                          char *buf)
2048 {
2049         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2050
2051         return sprintf(buf, "%pI6\n", target->path.sgid.raw);
2052 }
2053
2054 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2055                          char *buf)
2056 {
2057         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2058
2059         return sprintf(buf, "%pI6\n", target->path.dgid.raw);
2060 }
2061
2062 static ssize_t show_orig_dgid(struct device *dev,
2063                               struct device_attribute *attr, char *buf)
2064 {
2065         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2066
2067         return sprintf(buf, "%pI6\n", target->orig_dgid);
2068 }
2069
2070 static ssize_t show_req_lim(struct device *dev,
2071                             struct device_attribute *attr, char *buf)
2072 {
2073         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2074
2075         return sprintf(buf, "%d\n", target->req_lim);
2076 }
2077
2078 static ssize_t show_zero_req_lim(struct device *dev,
2079                                  struct device_attribute *attr, char *buf)
2080 {
2081         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2082
2083         return sprintf(buf, "%d\n", target->zero_req_lim);
2084 }
2085
2086 static ssize_t show_local_ib_port(struct device *dev,
2087                                   struct device_attribute *attr, char *buf)
2088 {
2089         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2090
2091         return sprintf(buf, "%d\n", target->srp_host->port);
2092 }
2093
2094 static ssize_t show_local_ib_device(struct device *dev,
2095                                     struct device_attribute *attr, char *buf)
2096 {
2097         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2098
2099         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2100 }
2101
2102 static ssize_t show_comp_vector(struct device *dev,
2103                                 struct device_attribute *attr, char *buf)
2104 {
2105         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2106
2107         return sprintf(buf, "%d\n", target->comp_vector);
2108 }
2109
2110 static ssize_t show_tl_retry_count(struct device *dev,
2111                                    struct device_attribute *attr, char *buf)
2112 {
2113         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2114
2115         return sprintf(buf, "%d\n", target->tl_retry_count);
2116 }
2117
2118 static ssize_t show_cmd_sg_entries(struct device *dev,
2119                                    struct device_attribute *attr, char *buf)
2120 {
2121         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2122
2123         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2124 }
2125
2126 static ssize_t show_allow_ext_sg(struct device *dev,
2127                                  struct device_attribute *attr, char *buf)
2128 {
2129         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2130
2131         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2132 }
2133
2134 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2135 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2136 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2137 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2138 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2139 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2140 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2141 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2142 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2143 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2144 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2145 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2146 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2147 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2148 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2149
2150 static struct device_attribute *srp_host_attrs[] = {
2151         &dev_attr_id_ext,
2152         &dev_attr_ioc_guid,
2153         &dev_attr_service_id,
2154         &dev_attr_pkey,
2155         &dev_attr_sgid,
2156         &dev_attr_dgid,
2157         &dev_attr_orig_dgid,
2158         &dev_attr_req_lim,
2159         &dev_attr_zero_req_lim,
2160         &dev_attr_local_ib_port,
2161         &dev_attr_local_ib_device,
2162         &dev_attr_comp_vector,
2163         &dev_attr_tl_retry_count,
2164         &dev_attr_cmd_sg_entries,
2165         &dev_attr_allow_ext_sg,
2166         NULL
2167 };
2168
2169 static struct scsi_host_template srp_template = {
2170         .module                         = THIS_MODULE,
2171         .name                           = "InfiniBand SRP initiator",
2172         .proc_name                      = DRV_NAME,
2173         .slave_configure                = srp_slave_configure,
2174         .info                           = srp_target_info,
2175         .queuecommand                   = srp_queuecommand,
2176         .eh_abort_handler               = srp_abort,
2177         .eh_device_reset_handler        = srp_reset_device,
2178         .eh_host_reset_handler          = srp_reset_host,
2179         .skip_settle_delay              = true,
2180         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2181         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2182         .this_id                        = -1,
2183         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2184         .use_clustering                 = ENABLE_CLUSTERING,
2185         .shost_attrs                    = srp_host_attrs
2186 };
2187
2188 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2189 {
2190         struct srp_rport_identifiers ids;
2191         struct srp_rport *rport;
2192
2193         sprintf(target->target_name, "SRP.T10:%016llX",
2194                  (unsigned long long) be64_to_cpu(target->id_ext));
2195
2196         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2197                 return -ENODEV;
2198
2199         memcpy(ids.port_id, &target->id_ext, 8);
2200         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2201         ids.roles = SRP_RPORT_ROLE_TARGET;
2202         rport = srp_rport_add(target->scsi_host, &ids);
2203         if (IS_ERR(rport)) {
2204                 scsi_remove_host(target->scsi_host);
2205                 return PTR_ERR(rport);
2206         }
2207
2208         rport->lld_data = target;
2209         target->rport = rport;
2210
2211         spin_lock(&host->target_lock);
2212         list_add_tail(&target->list, &host->target_list);
2213         spin_unlock(&host->target_lock);
2214
2215         target->state = SRP_TARGET_LIVE;
2216
2217         scsi_scan_target(&target->scsi_host->shost_gendev,
2218                          0, target->scsi_id, SCAN_WILD_CARD, 0);
2219
2220         return 0;
2221 }
2222
2223 static void srp_release_dev(struct device *dev)
2224 {
2225         struct srp_host *host =
2226                 container_of(dev, struct srp_host, dev);
2227
2228         complete(&host->released);
2229 }
2230
2231 static struct class srp_class = {
2232         .name    = "infiniband_srp",
2233         .dev_release = srp_release_dev
2234 };
2235
2236 /**
2237  * srp_conn_unique() - check whether the connection to a target is unique
2238  */
2239 static bool srp_conn_unique(struct srp_host *host,
2240                             struct srp_target_port *target)
2241 {
2242         struct srp_target_port *t;
2243         bool ret = false;
2244
2245         if (target->state == SRP_TARGET_REMOVED)
2246                 goto out;
2247
2248         ret = true;
2249
2250         spin_lock(&host->target_lock);
2251         list_for_each_entry(t, &host->target_list, list) {
2252                 if (t != target &&
2253                     target->id_ext == t->id_ext &&
2254                     target->ioc_guid == t->ioc_guid &&
2255                     target->initiator_ext == t->initiator_ext) {
2256                         ret = false;
2257                         break;
2258                 }
2259         }
2260         spin_unlock(&host->target_lock);
2261
2262 out:
2263         return ret;
2264 }
2265
2266 /*
2267  * Target ports are added by writing
2268  *
2269  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2270  *     pkey=<P_Key>,service_id=<service ID>
2271  *
2272  * to the add_target sysfs attribute.
2273  */
2274 enum {
2275         SRP_OPT_ERR             = 0,
2276         SRP_OPT_ID_EXT          = 1 << 0,
2277         SRP_OPT_IOC_GUID        = 1 << 1,
2278         SRP_OPT_DGID            = 1 << 2,
2279         SRP_OPT_PKEY            = 1 << 3,
2280         SRP_OPT_SERVICE_ID      = 1 << 4,
2281         SRP_OPT_MAX_SECT        = 1 << 5,
2282         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
2283         SRP_OPT_IO_CLASS        = 1 << 7,
2284         SRP_OPT_INITIATOR_EXT   = 1 << 8,
2285         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
2286         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
2287         SRP_OPT_SG_TABLESIZE    = 1 << 11,
2288         SRP_OPT_COMP_VECTOR     = 1 << 12,
2289         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
2290         SRP_OPT_QUEUE_SIZE      = 1 << 14,
2291         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
2292                                    SRP_OPT_IOC_GUID     |
2293                                    SRP_OPT_DGID         |
2294                                    SRP_OPT_PKEY         |
2295                                    SRP_OPT_SERVICE_ID),
2296 };
2297
2298 static const match_table_t srp_opt_tokens = {
2299         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
2300         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
2301         { SRP_OPT_DGID,                 "dgid=%s"               },
2302         { SRP_OPT_PKEY,                 "pkey=%x"               },
2303         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
2304         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
2305         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
2306         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
2307         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
2308         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
2309         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
2310         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
2311         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
2312         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
2313         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
2314         { SRP_OPT_ERR,                  NULL                    }
2315 };
2316
2317 static int srp_parse_options(const char *buf, struct srp_target_port *target)
2318 {
2319         char *options, *sep_opt;
2320         char *p;
2321         char dgid[3];
2322         substring_t args[MAX_OPT_ARGS];
2323         int opt_mask = 0;
2324         int token;
2325         int ret = -EINVAL;
2326         int i;
2327
2328         options = kstrdup(buf, GFP_KERNEL);
2329         if (!options)
2330                 return -ENOMEM;
2331
2332         sep_opt = options;
2333         while ((p = strsep(&sep_opt, ",")) != NULL) {
2334                 if (!*p)
2335                         continue;
2336
2337                 token = match_token(p, srp_opt_tokens, args);
2338                 opt_mask |= token;
2339
2340                 switch (token) {
2341                 case SRP_OPT_ID_EXT:
2342                         p = match_strdup(args);
2343                         if (!p) {
2344                                 ret = -ENOMEM;
2345                                 goto out;
2346                         }
2347                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2348                         kfree(p);
2349                         break;
2350
2351                 case SRP_OPT_IOC_GUID:
2352                         p = match_strdup(args);
2353                         if (!p) {
2354                                 ret = -ENOMEM;
2355                                 goto out;
2356                         }
2357                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2358                         kfree(p);
2359                         break;
2360
2361                 case SRP_OPT_DGID:
2362                         p = match_strdup(args);
2363                         if (!p) {
2364                                 ret = -ENOMEM;
2365                                 goto out;
2366                         }
2367                         if (strlen(p) != 32) {
2368                                 pr_warn("bad dest GID parameter '%s'\n", p);
2369                                 kfree(p);
2370                                 goto out;
2371                         }
2372
2373                         for (i = 0; i < 16; ++i) {
2374                                 strlcpy(dgid, p + i * 2, 3);
2375                                 target->path.dgid.raw[i] = simple_strtoul(dgid, NULL, 16);
2376                         }
2377                         kfree(p);
2378                         memcpy(target->orig_dgid, target->path.dgid.raw, 16);
2379                         break;
2380
2381                 case SRP_OPT_PKEY:
2382                         if (match_hex(args, &token)) {
2383                                 pr_warn("bad P_Key parameter '%s'\n", p);
2384                                 goto out;
2385                         }
2386                         target->path.pkey = cpu_to_be16(token);
2387                         break;
2388
2389                 case SRP_OPT_SERVICE_ID:
2390                         p = match_strdup(args);
2391                         if (!p) {
2392                                 ret = -ENOMEM;
2393                                 goto out;
2394                         }
2395                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
2396                         target->path.service_id = target->service_id;
2397                         kfree(p);
2398                         break;
2399
2400                 case SRP_OPT_MAX_SECT:
2401                         if (match_int(args, &token)) {
2402                                 pr_warn("bad max sect parameter '%s'\n", p);
2403                                 goto out;
2404                         }
2405                         target->scsi_host->max_sectors = token;
2406                         break;
2407
2408                 case SRP_OPT_QUEUE_SIZE:
2409                         if (match_int(args, &token) || token < 1) {
2410                                 pr_warn("bad queue_size parameter '%s'\n", p);
2411                                 goto out;
2412                         }
2413                         target->scsi_host->can_queue = token;
2414                         target->queue_size = token + SRP_RSP_SQ_SIZE +
2415                                              SRP_TSK_MGMT_SQ_SIZE;
2416                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2417                                 target->scsi_host->cmd_per_lun = token;
2418                         break;
2419
2420                 case SRP_OPT_MAX_CMD_PER_LUN:
2421                         if (match_int(args, &token) || token < 1) {
2422                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
2423                                         p);
2424                                 goto out;
2425                         }
2426                         target->scsi_host->cmd_per_lun = token;
2427                         break;
2428
2429                 case SRP_OPT_IO_CLASS:
2430                         if (match_hex(args, &token)) {
2431                                 pr_warn("bad IO class parameter '%s'\n", p);
2432                                 goto out;
2433                         }
2434                         if (token != SRP_REV10_IB_IO_CLASS &&
2435                             token != SRP_REV16A_IB_IO_CLASS) {
2436                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
2437                                         token, SRP_REV10_IB_IO_CLASS,
2438                                         SRP_REV16A_IB_IO_CLASS);
2439                                 goto out;
2440                         }
2441                         target->io_class = token;
2442                         break;
2443
2444                 case SRP_OPT_INITIATOR_EXT:
2445                         p = match_strdup(args);
2446                         if (!p) {
2447                                 ret = -ENOMEM;
2448                                 goto out;
2449                         }
2450                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2451                         kfree(p);
2452                         break;
2453
2454                 case SRP_OPT_CMD_SG_ENTRIES:
2455                         if (match_int(args, &token) || token < 1 || token > 255) {
2456                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
2457                                         p);
2458                                 goto out;
2459                         }
2460                         target->cmd_sg_cnt = token;
2461                         break;
2462
2463                 case SRP_OPT_ALLOW_EXT_SG:
2464                         if (match_int(args, &token)) {
2465                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
2466                                 goto out;
2467                         }
2468                         target->allow_ext_sg = !!token;
2469                         break;
2470
2471                 case SRP_OPT_SG_TABLESIZE:
2472                         if (match_int(args, &token) || token < 1 ||
2473                                         token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
2474                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
2475                                         p);
2476                                 goto out;
2477                         }
2478                         target->sg_tablesize = token;
2479                         break;
2480
2481                 case SRP_OPT_COMP_VECTOR:
2482                         if (match_int(args, &token) || token < 0) {
2483                                 pr_warn("bad comp_vector parameter '%s'\n", p);
2484                                 goto out;
2485                         }
2486                         target->comp_vector = token;
2487                         break;
2488
2489                 case SRP_OPT_TL_RETRY_COUNT:
2490                         if (match_int(args, &token) || token < 2 || token > 7) {
2491                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
2492                                         p);
2493                                 goto out;
2494                         }
2495                         target->tl_retry_count = token;
2496                         break;
2497
2498                 default:
2499                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
2500                                 p);
2501                         goto out;
2502                 }
2503         }
2504
2505         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
2506                 ret = 0;
2507         else
2508                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
2509                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
2510                             !(srp_opt_tokens[i].token & opt_mask))
2511                                 pr_warn("target creation request is missing parameter '%s'\n",
2512                                         srp_opt_tokens[i].pattern);
2513
2514         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
2515             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2516                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
2517                         target->scsi_host->cmd_per_lun,
2518                         target->scsi_host->can_queue);
2519
2520 out:
2521         kfree(options);
2522         return ret;
2523 }
2524
2525 static ssize_t srp_create_target(struct device *dev,
2526                                  struct device_attribute *attr,
2527                                  const char *buf, size_t count)
2528 {
2529         struct srp_host *host =
2530                 container_of(dev, struct srp_host, dev);
2531         struct Scsi_Host *target_host;
2532         struct srp_target_port *target;
2533         struct ib_device *ibdev = host->srp_dev->dev;
2534         int ret;
2535
2536         target_host = scsi_host_alloc(&srp_template,
2537                                       sizeof (struct srp_target_port));
2538         if (!target_host)
2539                 return -ENOMEM;
2540
2541         target_host->transportt  = ib_srp_transport_template;
2542         target_host->max_channel = 0;
2543         target_host->max_id      = 1;
2544         target_host->max_lun     = SRP_MAX_LUN;
2545         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
2546
2547         target = host_to_target(target_host);
2548
2549         target->io_class        = SRP_REV16A_IB_IO_CLASS;
2550         target->scsi_host       = target_host;
2551         target->srp_host        = host;
2552         target->lkey            = host->srp_dev->mr->lkey;
2553         target->rkey            = host->srp_dev->mr->rkey;
2554         target->cmd_sg_cnt      = cmd_sg_entries;
2555         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
2556         target->allow_ext_sg    = allow_ext_sg;
2557         target->tl_retry_count  = 7;
2558         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
2559
2560         ret = srp_parse_options(buf, target);
2561         if (ret)
2562                 goto err;
2563
2564         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
2565
2566         if (!srp_conn_unique(target->srp_host, target)) {
2567                 shost_printk(KERN_INFO, target->scsi_host,
2568                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
2569                              be64_to_cpu(target->id_ext),
2570                              be64_to_cpu(target->ioc_guid),
2571                              be64_to_cpu(target->initiator_ext));
2572                 ret = -EEXIST;
2573                 goto err;
2574         }
2575
2576         if (!host->srp_dev->fmr_pool && !target->allow_ext_sg &&
2577                                 target->cmd_sg_cnt < target->sg_tablesize) {
2578                 pr_warn("No FMR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
2579                 target->sg_tablesize = target->cmd_sg_cnt;
2580         }
2581
2582         target_host->sg_tablesize = target->sg_tablesize;
2583         target->indirect_size = target->sg_tablesize *
2584                                 sizeof (struct srp_direct_buf);
2585         target->max_iu_len = sizeof (struct srp_cmd) +
2586                              sizeof (struct srp_indirect_buf) +
2587                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
2588
2589         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
2590         INIT_WORK(&target->remove_work, srp_remove_work);
2591         spin_lock_init(&target->lock);
2592         INIT_LIST_HEAD(&target->free_tx);
2593         ret = srp_alloc_req_data(target);
2594         if (ret)
2595                 goto err_free_mem;
2596
2597         ib_query_gid(ibdev, host->port, 0, &target->path.sgid);
2598
2599         shost_printk(KERN_DEBUG, target->scsi_host, PFX
2600                      "new target: id_ext %016llx ioc_guid %016llx pkey %04x "
2601                      "service_id %016llx dgid %pI6\n",
2602                (unsigned long long) be64_to_cpu(target->id_ext),
2603                (unsigned long long) be64_to_cpu(target->ioc_guid),
2604                be16_to_cpu(target->path.pkey),
2605                (unsigned long long) be64_to_cpu(target->service_id),
2606                target->path.dgid.raw);
2607
2608         ret = srp_create_target_ib(target);
2609         if (ret)
2610                 goto err_free_mem;
2611
2612         ret = srp_new_cm_id(target);
2613         if (ret)
2614                 goto err_free_ib;
2615
2616         ret = srp_connect_target(target);
2617         if (ret) {
2618                 shost_printk(KERN_ERR, target->scsi_host,
2619                              PFX "Connection failed\n");
2620                 goto err_cm_id;
2621         }
2622
2623         ret = srp_add_target(host, target);
2624         if (ret)
2625                 goto err_disconnect;
2626
2627         return count;
2628
2629 err_disconnect:
2630         srp_disconnect_target(target);
2631
2632 err_cm_id:
2633         ib_destroy_cm_id(target->cm_id);
2634
2635 err_free_ib:
2636         srp_free_target_ib(target);
2637
2638 err_free_mem:
2639         srp_free_req_data(target);
2640
2641 err:
2642         scsi_host_put(target_host);
2643
2644         return ret;
2645 }
2646
2647 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
2648
2649 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
2650                           char *buf)
2651 {
2652         struct srp_host *host = container_of(dev, struct srp_host, dev);
2653
2654         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
2655 }
2656
2657 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
2658
2659 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
2660                          char *buf)
2661 {
2662         struct srp_host *host = container_of(dev, struct srp_host, dev);
2663
2664         return sprintf(buf, "%d\n", host->port);
2665 }
2666
2667 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
2668
2669 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
2670 {
2671         struct srp_host *host;
2672
2673         host = kzalloc(sizeof *host, GFP_KERNEL);
2674         if (!host)
2675                 return NULL;
2676
2677         INIT_LIST_HEAD(&host->target_list);
2678         spin_lock_init(&host->target_lock);
2679         init_completion(&host->released);
2680         host->srp_dev = device;
2681         host->port = port;
2682
2683         host->dev.class = &srp_class;
2684         host->dev.parent = device->dev->dma_device;
2685         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
2686
2687         if (device_register(&host->dev))
2688                 goto free_host;
2689         if (device_create_file(&host->dev, &dev_attr_add_target))
2690                 goto err_class;
2691         if (device_create_file(&host->dev, &dev_attr_ibdev))
2692                 goto err_class;
2693         if (device_create_file(&host->dev, &dev_attr_port))
2694                 goto err_class;
2695
2696         return host;
2697
2698 err_class:
2699         device_unregister(&host->dev);
2700
2701 free_host:
2702         kfree(host);
2703
2704         return NULL;
2705 }
2706
2707 static void srp_add_one(struct ib_device *device)
2708 {
2709         struct srp_device *srp_dev;
2710         struct ib_device_attr *dev_attr;
2711         struct ib_fmr_pool_param fmr_param;
2712         struct srp_host *host;
2713         int max_pages_per_fmr, fmr_page_shift, s, e, p;
2714
2715         dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
2716         if (!dev_attr)
2717                 return;
2718
2719         if (ib_query_device(device, dev_attr)) {
2720                 pr_warn("Query device failed for %s\n", device->name);
2721                 goto free_attr;
2722         }
2723
2724         srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
2725         if (!srp_dev)
2726                 goto free_attr;
2727
2728         /*
2729          * Use the smallest page size supported by the HCA, down to a
2730          * minimum of 4096 bytes. We're unlikely to build large sglists
2731          * out of smaller entries.
2732          */
2733         fmr_page_shift          = max(12, ffs(dev_attr->page_size_cap) - 1);
2734         srp_dev->fmr_page_size  = 1 << fmr_page_shift;
2735         srp_dev->fmr_page_mask  = ~((u64) srp_dev->fmr_page_size - 1);
2736         srp_dev->fmr_max_size   = srp_dev->fmr_page_size * SRP_FMR_SIZE;
2737
2738         INIT_LIST_HEAD(&srp_dev->dev_list);
2739
2740         srp_dev->dev = device;
2741         srp_dev->pd  = ib_alloc_pd(device);
2742         if (IS_ERR(srp_dev->pd))
2743                 goto free_dev;
2744
2745         srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
2746                                     IB_ACCESS_LOCAL_WRITE |
2747                                     IB_ACCESS_REMOTE_READ |
2748                                     IB_ACCESS_REMOTE_WRITE);
2749         if (IS_ERR(srp_dev->mr))
2750                 goto err_pd;
2751
2752         for (max_pages_per_fmr = SRP_FMR_SIZE;
2753                         max_pages_per_fmr >= SRP_FMR_MIN_SIZE;
2754                         max_pages_per_fmr /= 2, srp_dev->fmr_max_size /= 2) {
2755                 memset(&fmr_param, 0, sizeof fmr_param);
2756                 fmr_param.pool_size         = SRP_FMR_POOL_SIZE;
2757                 fmr_param.dirty_watermark   = SRP_FMR_DIRTY_SIZE;
2758                 fmr_param.cache             = 1;
2759                 fmr_param.max_pages_per_fmr = max_pages_per_fmr;
2760                 fmr_param.page_shift        = fmr_page_shift;
2761                 fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
2762                                                IB_ACCESS_REMOTE_WRITE |
2763                                                IB_ACCESS_REMOTE_READ);
2764
2765                 srp_dev->fmr_pool = ib_create_fmr_pool(srp_dev->pd, &fmr_param);
2766                 if (!IS_ERR(srp_dev->fmr_pool))
2767                         break;
2768         }
2769
2770         if (IS_ERR(srp_dev->fmr_pool))
2771                 srp_dev->fmr_pool = NULL;
2772
2773         if (device->node_type == RDMA_NODE_IB_SWITCH) {
2774                 s = 0;
2775                 e = 0;
2776         } else {
2777                 s = 1;
2778                 e = device->phys_port_cnt;
2779         }
2780
2781         for (p = s; p <= e; ++p) {
2782                 host = srp_add_port(srp_dev, p);
2783                 if (host)
2784                         list_add_tail(&host->list, &srp_dev->dev_list);
2785         }
2786
2787         ib_set_client_data(device, &srp_client, srp_dev);
2788
2789         goto free_attr;
2790
2791 err_pd:
2792         ib_dealloc_pd(srp_dev->pd);
2793
2794 free_dev:
2795         kfree(srp_dev);
2796
2797 free_attr:
2798         kfree(dev_attr);
2799 }
2800
2801 static void srp_remove_one(struct ib_device *device)
2802 {
2803         struct srp_device *srp_dev;
2804         struct srp_host *host, *tmp_host;
2805         struct srp_target_port *target;
2806
2807         srp_dev = ib_get_client_data(device, &srp_client);
2808         if (!srp_dev)
2809                 return;
2810
2811         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
2812                 device_unregister(&host->dev);
2813                 /*
2814                  * Wait for the sysfs entry to go away, so that no new
2815                  * target ports can be created.
2816                  */
2817                 wait_for_completion(&host->released);
2818
2819                 /*
2820                  * Remove all target ports.
2821                  */
2822                 spin_lock(&host->target_lock);
2823                 list_for_each_entry(target, &host->target_list, list)
2824                         srp_queue_remove_work(target);
2825                 spin_unlock(&host->target_lock);
2826
2827                 /*
2828                  * Wait for target port removal tasks.
2829                  */
2830                 flush_workqueue(system_long_wq);
2831
2832                 kfree(host);
2833         }
2834
2835         if (srp_dev->fmr_pool)
2836                 ib_destroy_fmr_pool(srp_dev->fmr_pool);
2837         ib_dereg_mr(srp_dev->mr);
2838         ib_dealloc_pd(srp_dev->pd);
2839
2840         kfree(srp_dev);
2841 }
2842
2843 static struct srp_function_template ib_srp_transport_functions = {
2844         .has_rport_state         = true,
2845         .reset_timer_if_blocked  = true,
2846         .reconnect_delay         = &srp_reconnect_delay,
2847         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
2848         .dev_loss_tmo            = &srp_dev_loss_tmo,
2849         .reconnect               = srp_rport_reconnect,
2850         .rport_delete            = srp_rport_delete,
2851         .terminate_rport_io      = srp_terminate_io,
2852 };
2853
2854 static int __init srp_init_module(void)
2855 {
2856         int ret;
2857
2858         BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
2859
2860         if (srp_sg_tablesize) {
2861                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
2862                 if (!cmd_sg_entries)
2863                         cmd_sg_entries = srp_sg_tablesize;
2864         }
2865
2866         if (!cmd_sg_entries)
2867                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
2868
2869         if (cmd_sg_entries > 255) {
2870                 pr_warn("Clamping cmd_sg_entries to 255\n");
2871                 cmd_sg_entries = 255;
2872         }
2873
2874         if (!indirect_sg_entries)
2875                 indirect_sg_entries = cmd_sg_entries;
2876         else if (indirect_sg_entries < cmd_sg_entries) {
2877                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
2878                         cmd_sg_entries);
2879                 indirect_sg_entries = cmd_sg_entries;
2880         }
2881
2882         ib_srp_transport_template =
2883                 srp_attach_transport(&ib_srp_transport_functions);
2884         if (!ib_srp_transport_template)
2885                 return -ENOMEM;
2886
2887         ret = class_register(&srp_class);
2888         if (ret) {
2889                 pr_err("couldn't register class infiniband_srp\n");
2890                 srp_release_transport(ib_srp_transport_template);
2891                 return ret;
2892         }
2893
2894         ib_sa_register_client(&srp_sa_client);
2895
2896         ret = ib_register_client(&srp_client);
2897         if (ret) {
2898                 pr_err("couldn't register IB client\n");
2899                 srp_release_transport(ib_srp_transport_template);
2900                 ib_sa_unregister_client(&srp_sa_client);
2901                 class_unregister(&srp_class);
2902                 return ret;
2903         }
2904
2905         return 0;
2906 }
2907
2908 static void __exit srp_cleanup_module(void)
2909 {
2910         ib_unregister_client(&srp_client);
2911         ib_sa_unregister_client(&srp_sa_client);
2912         class_unregister(&srp_class);
2913         srp_release_transport(ib_srp_transport_template);
2914 }
2915
2916 module_init(srp_init_module);
2917 module_exit(srp_cleanup_module);