drivers/scsi/elx/efct/efct_scsi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
   4  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
   5  */
   6
   7 #include "efct_driver.h"
   8 #include "efct_hw.h"
   9
  10 #define enable_tsend_auto_resp(efct)    1
  11 #define enable_treceive_auto_resp(efct) 0
  12
  13 #define SCSI_IOFMT "[%04x][i:%04x t:%04x h:%04x]"
  14
  15 #define scsi_io_printf(io, fmt, ...) \
  16         efc_log_debug(io->efct, "[%s]" SCSI_IOFMT fmt, \
  17                 io->node->display_name, io->instance_index,\
  18                 io->init_task_tag, io->tgt_task_tag, io->hw_tag, ##__VA_ARGS__)
  19
  20 #define EFCT_LOG_ENABLE_SCSI_TRACE(efct)                \
  21                 (((efct) != NULL) ? (((efct)->logmask & (1U << 2)) != 0) : 0)
  22
  23 #define scsi_io_trace(io, fmt, ...) \
  24         do { \
  25                 if (EFCT_LOG_ENABLE_SCSI_TRACE(io->efct)) \
  26                         scsi_io_printf(io, fmt, ##__VA_ARGS__); \
  27         } while (0)
  28
  29 struct efct_io *
  30 efct_scsi_io_alloc(struct efct_node *node)
  31 {
  32         struct efct *efct;
  33         struct efct_xport *xport;
  34         struct efct_io *io;
  35         unsigned long flags = 0;
  36
  37         efct = node->efct;
  38
  39         xport = efct->xport;
  40
  41         spin_lock_irqsave(&node->active_ios_lock, flags);
  42
  43         io = efct_io_pool_io_alloc(efct->xport->io_pool);
  44         if (!io) {
  45                 efc_log_err(efct, "IO alloc Failed\n");
  46                 atomic_add_return(1, &xport->io_alloc_failed_count);
  47                 spin_unlock_irqrestore(&node->active_ios_lock, flags);
  48                 return NULL;
  49         }
  50
  51         /* initialize refcount */
  52         kref_init(&io->ref);
  53         io->release = _efct_scsi_io_free;
  54
  55         /* set generic fields */
  56         io->efct = efct;
  57         io->node = node;
  58         kref_get(&node->ref);
  59
  60         /* set type and name */
  61         io->io_type = EFCT_IO_TYPE_IO;
  62         io->display_name = "scsi_io";
  63
  64         io->cmd_ini = false;
  65         io->cmd_tgt = true;
  66
  67         /* Add to node's active_ios list */
  68         INIT_LIST_HEAD(&io->list_entry);
  69         list_add(&io->list_entry, &node->active_ios);
  70
  71         spin_unlock_irqrestore(&node->active_ios_lock, flags);
  72
  73         return io;
  74 }
  75
  76 void
  77 _efct_scsi_io_free(struct kref *arg)
  78 {
  79         struct efct_io *io = container_of(arg, struct efct_io, ref);
  80         struct efct *efct = io->efct;
  81         struct efct_node *node = io->node;
  82         unsigned long flags = 0;
  83
  84         scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
  85
  86         if (io->io_free) {
  87                 efc_log_err(efct, "IO already freed.\n");
  88                 return;
  89         }
  90
  91         spin_lock_irqsave(&node->active_ios_lock, flags);
  92         list_del_init(&io->list_entry);
  93         spin_unlock_irqrestore(&node->active_ios_lock, flags);
  94
  95         kref_put(&node->ref, node->release);
  96         io->node = NULL;
  97         efct_io_pool_io_free(efct->xport->io_pool, io);
  98 }
  99
 100 void
 101 efct_scsi_io_free(struct efct_io *io)
 102 {
 103         scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
 104         WARN_ON(!refcount_read(&io->ref.refcount));
 105         kref_put(&io->ref, io->release);
 106 }
 107
 108 static void
 109 efct_target_io_cb(struct efct_hw_io *hio, u32 length, int status,
 110                   u32 ext_status, void *app)
 111 {
 112         u32 flags = 0;
 113         struct efct_io *io = app;
 114         struct efct *efct;
 115         enum efct_scsi_io_status scsi_stat = EFCT_SCSI_STATUS_GOOD;
 116         efct_scsi_io_cb_t cb;
 117
 118         if (!io || !io->efct) {
 119                 pr_err("%s: IO can not be NULL\n", __func__);
 120                 return;
 121         }
 122
 123         scsi_io_trace(io, "status x%x ext_status x%x\n", status, ext_status);
 124
 125         efct = io->efct;
 126
 127         io->transferred += length;
 128
 129         if (!io->scsi_tgt_cb) {
 130                 efct_scsi_check_pending(efct);
 131                 return;
 132         }
 133
 134         /* Call target server completion */
 135         cb = io->scsi_tgt_cb;
 136
 137         /* Clear the callback before invoking the callback */
 138         io->scsi_tgt_cb = NULL;
 139
 140         /* if status was good, and auto-good-response was set,
 141          * then callback target-server with IO_CMPL_RSP_SENT,
 142          * otherwise send IO_CMPL
 143          */
 144         if (status == 0 && io->auto_resp)
 145                 flags |= EFCT_SCSI_IO_CMPL_RSP_SENT;
 146         else
 147                 flags |= EFCT_SCSI_IO_CMPL;
 148
 149         switch (status) {
 150         case SLI4_FC_WCQE_STATUS_SUCCESS:
 151                 scsi_stat = EFCT_SCSI_STATUS_GOOD;
 152                 break;
 153         case SLI4_FC_WCQE_STATUS_DI_ERROR:
 154                 if (ext_status & SLI4_FC_DI_ERROR_GE)
 155                         scsi_stat = EFCT_SCSI_STATUS_DIF_GUARD_ERR;
 156                 else if (ext_status & SLI4_FC_DI_ERROR_AE)
 157                         scsi_stat = EFCT_SCSI_STATUS_DIF_APP_TAG_ERROR;
 158                 else if (ext_status & SLI4_FC_DI_ERROR_RE)
 159                         scsi_stat = EFCT_SCSI_STATUS_DIF_REF_TAG_ERROR;
 160                 else
 161                         scsi_stat = EFCT_SCSI_STATUS_DIF_UNKNOWN_ERROR;
 162                 break;
 163         case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
 164                 switch (ext_status) {
 165                 case SLI4_FC_LOCAL_REJECT_INVALID_RELOFFSET:
 166                 case SLI4_FC_LOCAL_REJECT_ABORT_REQUESTED:
 167                         scsi_stat = EFCT_SCSI_STATUS_ABORTED;
 168                         break;
 169                 case SLI4_FC_LOCAL_REJECT_INVALID_RPI:
 170                         scsi_stat = EFCT_SCSI_STATUS_NEXUS_LOST;
 171                         break;
 172                 case SLI4_FC_LOCAL_REJECT_NO_XRI:
 173                         scsi_stat = EFCT_SCSI_STATUS_NO_IO;
 174                         break;
 175                 default:
 176                         /*we have seen 0x0d(TX_DMA_FAILED err)*/
 177                         scsi_stat = EFCT_SCSI_STATUS_ERROR;
 178                         break;
 179                 }
 180                 break;
 181
 182         case SLI4_FC_WCQE_STATUS_TARGET_WQE_TIMEOUT:
 183                 /* target IO timed out */
 184                 scsi_stat = EFCT_SCSI_STATUS_TIMEDOUT_AND_ABORTED;
 185                 break;
 186
 187         case SLI4_FC_WCQE_STATUS_SHUTDOWN:
 188                 /* Target IO cancelled by HW */
 189                 scsi_stat = EFCT_SCSI_STATUS_SHUTDOWN;
 190                 break;
 191
 192         default:
 193                 scsi_stat = EFCT_SCSI_STATUS_ERROR;
 194                 break;
 195         }
 196
 197         cb(io, scsi_stat, flags, io->scsi_tgt_cb_arg);
 198
 199         efct_scsi_check_pending(efct);
 200 }
 201
 202 static int
 203 efct_scsi_build_sgls(struct efct_hw *hw, struct efct_hw_io *hio,
 204                      struct efct_scsi_sgl *sgl, u32 sgl_count,
 205                      enum efct_hw_io_type type)
 206 {
 207         int rc;
 208         u32 i;
 209         struct efct *efct = hw->os;
 210
 211         /* Initialize HW SGL */
 212         rc = efct_hw_io_init_sges(hw, hio, type);
 213         if (rc) {
 214                 efc_log_err(efct, "efct_hw_io_init_sges failed: %d\n", rc);
 215                 return -EIO;
 216         }
 217
 218         for (i = 0; i < sgl_count; i++) {
 219                 /* Add data SGE */
 220                 rc = efct_hw_io_add_sge(hw, hio, sgl[i].addr, sgl[i].len);
 221                 if (rc) {
 222                         efc_log_err(efct, "add sge failed cnt=%d rc=%d\n",
 223                                     sgl_count, rc);
 224                         return rc;
 225                 }
 226         }
 227
 228         return 0;
 229 }
 230
 231 static void efc_log_sgl(struct efct_io *io)
 232 {
 233         struct efct_hw_io *hio = io->hio;
 234         struct sli4_sge *data = NULL;
 235         u32 *dword = NULL;
 236         u32 i;
 237         u32 n_sge;
 238
 239         scsi_io_trace(io, "def_sgl at 0x%x 0x%08x\n",
 240                       upper_32_bits(hio->def_sgl.phys),
 241                       lower_32_bits(hio->def_sgl.phys));
 242         n_sge = (hio->sgl == &hio->def_sgl) ? hio->n_sge : hio->def_sgl_count;
 243         for (i = 0, data = hio->def_sgl.virt; i < n_sge; i++, data++) {
 244                 dword = (u32 *)data;
 245
 246                 scsi_io_trace(io, "SGL %2d 0x%08x 0x%08x 0x%08x 0x%08x\n",
 247                               i, dword[0], dword[1], dword[2], dword[3]);
 248
 249                 if (dword[2] & (1U << 31))
 250                         break;
 251         }
 252 }
 253
 254 static void
 255 efct_scsi_check_pending_async_cb(struct efct_hw *hw, int status,
 256                                  u8 *mqe, void *arg)
 257 {
 258         struct efct_io *io = arg;
 259
 260         if (io) {
 261                 efct_hw_done_t cb = io->hw_cb;
 262
 263                 if (!io->hw_cb)
 264                         return;
 265
 266                 io->hw_cb = NULL;
 267                 (cb)(io->hio, 0, SLI4_FC_WCQE_STATUS_DISPATCH_ERROR, 0, io);
 268         }
 269 }
 270
 271 static int
 272 efct_scsi_io_dispatch_hw_io(struct efct_io *io, struct efct_hw_io *hio)
 273 {
 274         int rc = 0;
 275         struct efct *efct = io->efct;
 276
 277         /* Got a HW IO;
 278          * update ini/tgt_task_tag with HW IO info and dispatch
 279          */
 280         io->hio = hio;
 281         if (io->cmd_tgt)
 282                 io->tgt_task_tag = hio->indicator;
 283         else if (io->cmd_ini)
 284                 io->init_task_tag = hio->indicator;
 285         io->hw_tag = hio->reqtag;
 286
 287         hio->eq = io->hw_priv;
 288
 289         /* Copy WQ steering */
 290         switch (io->wq_steering) {
 291         case EFCT_SCSI_WQ_STEERING_CLASS >> EFCT_SCSI_WQ_STEERING_SHIFT:
 292                 hio->wq_steering = EFCT_HW_WQ_STEERING_CLASS;
 293                 break;
 294         case EFCT_SCSI_WQ_STEERING_REQUEST >> EFCT_SCSI_WQ_STEERING_SHIFT:
 295                 hio->wq_steering = EFCT_HW_WQ_STEERING_REQUEST;
 296                 break;
 297         case EFCT_SCSI_WQ_STEERING_CPU >> EFCT_SCSI_WQ_STEERING_SHIFT:
 298                 hio->wq_steering = EFCT_HW_WQ_STEERING_CPU;
 299                 break;
 300         }
 301
 302         switch (io->io_type) {
 303         case EFCT_IO_TYPE_IO:
 304                 rc = efct_scsi_build_sgls(&efct->hw, io->hio,
 305                                           io->sgl, io->sgl_count, io->hio_type);
 306                 if (rc)
 307                         break;
 308
 309                 if (EFCT_LOG_ENABLE_SCSI_TRACE(efct))
 310                         efc_log_sgl(io);
 311
 312                 if (io->app_id)
 313                         io->iparam.fcp_tgt.app_id = io->app_id;
 314
 315                 io->iparam.fcp_tgt.vpi = io->node->vpi;
 316                 io->iparam.fcp_tgt.rpi = io->node->rpi;
 317                 io->iparam.fcp_tgt.s_id = io->node->port_fc_id;
 318                 io->iparam.fcp_tgt.d_id = io->node->node_fc_id;
 319                 io->iparam.fcp_tgt.xmit_len = io->wire_len;
 320
 321                 rc = efct_hw_io_send(&io->efct->hw, io->hio_type, io->hio,
 322                                      &io->iparam, io->hw_cb, io);
 323                 break;
 324         default:
 325                 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
 326                 rc = -EIO;
 327                 break;
 328         }
 329         return rc;
 330 }
 331
 332 static int
 333 efct_scsi_io_dispatch_no_hw_io(struct efct_io *io)
 334 {
 335         int rc;
 336
 337         switch (io->io_type) {
 338         case EFCT_IO_TYPE_ABORT: {
 339                 struct efct_hw_io *hio_to_abort = NULL;
 340
 341                 hio_to_abort = io->io_to_abort->hio;
 342
 343                 if (!hio_to_abort) {
 344                         /*
 345                          * If "IO to abort" does not have an
 346                          * associated HW IO, immediately make callback with
 347                          * success. The command must have been sent to
 348                          * the backend, but the data phase has not yet
 349                          * started, so we don't have a HW IO.
 350                          *
 351                          * Note: since the backend shims should be
 352                          * taking a reference on io_to_abort, it should not
 353                          * be possible to have been completed and freed by
 354                          * the backend before the abort got here.
 355                          */
 356                         scsi_io_printf(io, "IO: not active\n");
 357                         ((efct_hw_done_t)io->hw_cb)(io->hio, 0,
 358                                         SLI4_FC_WCQE_STATUS_SUCCESS, 0, io);
 359                         rc = 0;
 360                         break;
 361                 }
 362
 363                 /* HW IO is valid, abort it */
 364                 scsi_io_printf(io, "aborting\n");
 365                 rc = efct_hw_io_abort(&io->efct->hw, hio_to_abort,
 366                                       io->send_abts, io->hw_cb, io);
 367                 if (rc) {
 368                         int status = SLI4_FC_WCQE_STATUS_SUCCESS;
 369                         efct_hw_done_t cb = io->hw_cb;
 370
 371                         if (rc != -ENOENT && rc != -EINPROGRESS) {
 372                                 status = -1;
 373                                 scsi_io_printf(io, "Failed to abort IO rc=%d\n",
 374                                                rc);
 375                         }
 376                         cb(io->hio, 0, status, 0, io);
 377                         rc = 0;
 378                 }
 379
 380                 break;
 381         }
 382         default:
 383                 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
 384                 rc = -EIO;
 385                 break;
 386         }
 387         return rc;
 388 }
 389
 390 static struct efct_io *
 391 efct_scsi_dispatch_pending(struct efct *efct)
 392 {
 393         struct efct_xport *xport = efct->xport;
 394         struct efct_io *io = NULL;
 395         struct efct_hw_io *hio;
 396         unsigned long flags = 0;
 397         int status;
 398
 399         spin_lock_irqsave(&xport->io_pending_lock, flags);
 400
 401         if (!list_empty(&xport->io_pending_list)) {
 402                 io = list_first_entry(&xport->io_pending_list, struct efct_io,
 403                                       io_pending_link);
 404                 list_del_init(&io->io_pending_link);
 405         }
 406
 407         if (!io) {
 408                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 409                 return NULL;
 410         }
 411
 412         if (io->io_type == EFCT_IO_TYPE_ABORT) {
 413                 hio = NULL;
 414         } else {
 415                 hio = efct_hw_io_alloc(&efct->hw);
 416                 if (!hio) {
 417                         /*
 418                          * No HW IO available.Put IO back on
 419                          * the front of pending list
 420                          */
 421                         list_add(&xport->io_pending_list, &io->io_pending_link);
 422                         io = NULL;
 423                 } else {
 424                         hio->eq = io->hw_priv;
 425                 }
 426         }
 427
 428         /* Must drop the lock before dispatching the IO */
 429         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 430
 431         if (!io)
 432                 return NULL;
 433
 434         /*
 435          * We pulled an IO off the pending list,
 436          * and either got an HW IO or don't need one
 437          */
 438         atomic_sub_return(1, &xport->io_pending_count);
 439         if (!hio)
 440                 status = efct_scsi_io_dispatch_no_hw_io(io);
 441         else
 442                 status = efct_scsi_io_dispatch_hw_io(io, hio);
 443         if (status) {
 444                 /*
 445                  * Invoke the HW callback, but do so in the
 446                  * separate execution context,provided by the
 447                  * NOP mailbox completion processing context
 448                  * by using efct_hw_async_call()
 449                  */
 450                 if (efct_hw_async_call(&efct->hw,
 451                                        efct_scsi_check_pending_async_cb, io)) {
 452                         efc_log_debug(efct, "call hw async failed\n");
 453                 }
 454         }
 455
 456         return io;
 457 }
 458
 459 void
 460 efct_scsi_check_pending(struct efct *efct)
 461 {
 462         struct efct_xport *xport = efct->xport;
 463         struct efct_io *io = NULL;
 464         int count = 0;
 465         unsigned long flags = 0;
 466         int dispatch = 0;
 467
 468         /* Guard against recursion */
 469         if (atomic_add_return(1, &xport->io_pending_recursing)) {
 470                 /* This function is already running.  Decrement and return. */
 471                 atomic_sub_return(1, &xport->io_pending_recursing);
 472                 return;
 473         }
 474
 475         while (efct_scsi_dispatch_pending(efct))
 476                 count++;
 477
 478         if (count) {
 479                 atomic_sub_return(1, &xport->io_pending_recursing);
 480                 return;
 481         }
 482
 483         /*
 484          * If nothing was removed from the list,
 485          * we might be in a case where we need to abort an
 486          * active IO and the abort is on the pending list.
 487          * Look for an abort we can dispatch.
 488          */
 489
 490         spin_lock_irqsave(&xport->io_pending_lock, flags);
 491
 492         list_for_each_entry(io, &xport->io_pending_list, io_pending_link) {
 493                 if (io->io_type == EFCT_IO_TYPE_ABORT && io->io_to_abort->hio) {
 494                         /* This IO has a HW IO, so it is
 495                          * active.  Dispatch the abort.
 496                          */
 497                         dispatch = 1;
 498                         list_del_init(&io->io_pending_link);
 499                         atomic_sub_return(1, &xport->io_pending_count);
 500                         break;
 501                 }
 502         }
 503
 504         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 505
 506         if (dispatch) {
 507                 if (efct_scsi_io_dispatch_no_hw_io(io)) {
 508                         if (efct_hw_async_call(&efct->hw,
 509                                 efct_scsi_check_pending_async_cb, io)) {
 510                                 efc_log_debug(efct, "hw async failed\n");
 511                         }
 512                 }
 513         }
 514
 515         atomic_sub_return(1, &xport->io_pending_recursing);
 516 }
 517
 518 int
 519 efct_scsi_io_dispatch(struct efct_io *io, void *cb)
 520 {
 521         struct efct_hw_io *hio;
 522         struct efct *efct = io->efct;
 523         struct efct_xport *xport = efct->xport;
 524         unsigned long flags = 0;
 525
 526         io->hw_cb = cb;
 527
 528         /*
 529          * if this IO already has a HW IO, then this is either
 530          * not the first phase of the IO. Send it to the HW.
 531          */
 532         if (io->hio)
 533                 return efct_scsi_io_dispatch_hw_io(io, io->hio);
 534
 535         /*
 536          * We don't already have a HW IO associated with the IO. First check
 537          * the pending list. If not empty, add IO to the tail and process the
 538          * pending list.
 539          */
 540         spin_lock_irqsave(&xport->io_pending_lock, flags);
 541         if (!list_empty(&xport->io_pending_list)) {
 542                 /*
 543                  * If this is a low latency request,
 544                  * the put at the front of the IO pending
 545                  * queue, otherwise put it at the end of the queue.
 546                  */
 547                 if (io->low_latency) {
 548                         INIT_LIST_HEAD(&io->io_pending_link);
 549                         list_add(&xport->io_pending_list, &io->io_pending_link);
 550                 } else {
 551                         INIT_LIST_HEAD(&io->io_pending_link);
 552                         list_add_tail(&io->io_pending_link,
 553                                       &xport->io_pending_list);
 554                 }
 555                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 556                 atomic_add_return(1, &xport->io_pending_count);
 557                 atomic_add_return(1, &xport->io_total_pending);
 558
 559                 /* process pending list */
 560                 efct_scsi_check_pending(efct);
 561                 return 0;
 562         }
 563         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 564
 565         /*
 566          * We don't have a HW IO associated with the IO and there's nothing
 567          * on the pending list. Attempt to allocate a HW IO and dispatch it.
 568          */
 569         hio = efct_hw_io_alloc(&io->efct->hw);
 570         if (!hio) {
 571                 /* Couldn't get a HW IO. Save this IO on the pending list */
 572                 spin_lock_irqsave(&xport->io_pending_lock, flags);
 573                 INIT_LIST_HEAD(&io->io_pending_link);
 574                 list_add_tail(&io->io_pending_link, &xport->io_pending_list);
 575                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 576
 577                 atomic_add_return(1, &xport->io_total_pending);
 578                 atomic_add_return(1, &xport->io_pending_count);
 579                 return 0;
 580         }
 581
 582         /* We successfully allocated a HW IO; dispatch to HW */
 583         return efct_scsi_io_dispatch_hw_io(io, hio);
 584 }
 585
 586 int
 587 efct_scsi_io_dispatch_abort(struct efct_io *io, void *cb)
 588 {
 589         struct efct *efct = io->efct;
 590         struct efct_xport *xport = efct->xport;
 591         unsigned long flags = 0;
 592
 593         io->hw_cb = cb;
 594
 595         /*
 596          * For aborts, we don't need a HW IO, but we still want
 597          * to pass through the pending list to preserve ordering.
 598          * Thus, if the pending list is not empty, add this abort
 599          * to the pending list and process the pending list.
 600          */
 601         spin_lock_irqsave(&xport->io_pending_lock, flags);
 602         if (!list_empty(&xport->io_pending_list)) {
 603                 INIT_LIST_HEAD(&io->io_pending_link);
 604                 list_add_tail(&io->io_pending_link, &xport->io_pending_list);
 605                 spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 606                 atomic_add_return(1, &xport->io_pending_count);
 607                 atomic_add_return(1, &xport->io_total_pending);
 608
 609                 /* process pending list */
 610                 efct_scsi_check_pending(efct);
 611                 return 0;
 612         }
 613         spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 614
 615         /* nothing on pending list, dispatch abort */
 616         return efct_scsi_io_dispatch_no_hw_io(io);
 617 }
 618
 619 static inline int
 620 efct_scsi_xfer_data(struct efct_io *io, u32 flags,
 621                     struct efct_scsi_sgl *sgl, u32 sgl_count, u64 xwire_len,
 622                     enum efct_hw_io_type type, int enable_ar,
 623                     efct_scsi_io_cb_t cb, void *arg)
 624 {
 625         struct efct *efct;
 626         size_t residual = 0;
 627
 628         io->sgl_count = sgl_count;
 629
 630         efct = io->efct;
 631
 632         scsi_io_trace(io, "%s wire_len %llu\n",
 633                       (type == EFCT_HW_IO_TARGET_READ) ? "send" : "recv",
 634                       xwire_len);
 635
 636         io->hio_type = type;
 637
 638         io->scsi_tgt_cb = cb;
 639         io->scsi_tgt_cb_arg = arg;
 640
 641         residual = io->exp_xfer_len - io->transferred;
 642         io->wire_len = (xwire_len < residual) ? xwire_len : residual;
 643         residual = (xwire_len - io->wire_len);
 644
 645         memset(&io->iparam, 0, sizeof(io->iparam));
 646         io->iparam.fcp_tgt.ox_id = io->init_task_tag;
 647         io->iparam.fcp_tgt.offset = io->transferred;
 648         io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
 649         io->iparam.fcp_tgt.timeout = io->timeout;
 650
 651         /* if this is the last data phase and there is no residual, enable
 652          * auto-good-response
 653          */
 654         if (enable_ar && (flags & EFCT_SCSI_LAST_DATAPHASE) && residual == 0 &&
 655             ((io->transferred + io->wire_len) == io->exp_xfer_len) &&
 656             (!(flags & EFCT_SCSI_NO_AUTO_RESPONSE))) {
 657                 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
 658                 io->auto_resp = true;
 659         } else {
 660                 io->auto_resp = false;
 661         }
 662
 663         /* save this transfer length */
 664         io->xfer_req = io->wire_len;
 665
 666         /* Adjust the transferred count to account for overrun
 667          * when the residual is calculated in efct_scsi_send_resp
 668          */
 669         io->transferred += residual;
 670
 671         /* Adjust the SGL size if there is overrun */
 672
 673         if (residual) {
 674                 struct efct_scsi_sgl  *sgl_ptr = &io->sgl[sgl_count - 1];
 675
 676                 while (residual) {
 677                         size_t len = sgl_ptr->len;
 678
 679                         if (len > residual) {
 680                                 sgl_ptr->len = len - residual;
 681                                 residual = 0;
 682                         } else {
 683                                 sgl_ptr->len = 0;
 684                                 residual -= len;
 685                                 io->sgl_count--;
 686                         }
 687                         sgl_ptr--;
 688                 }
 689         }
 690
 691         /* Set latency and WQ steering */
 692         io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
 693         io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
 694                                 EFCT_SCSI_WQ_STEERING_SHIFT;
 695         io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
 696                                 EFCT_SCSI_WQ_CLASS_SHIFT;
 697
 698         if (efct->xport) {
 699                 struct efct_xport *xport = efct->xport;
 700
 701                 if (type == EFCT_HW_IO_TARGET_READ) {
 702                         xport->fcp_stats.input_requests++;
 703                         xport->fcp_stats.input_bytes += xwire_len;
 704                 } else if (type == EFCT_HW_IO_TARGET_WRITE) {
 705                         xport->fcp_stats.output_requests++;
 706                         xport->fcp_stats.output_bytes += xwire_len;
 707                 }
 708         }
 709         return efct_scsi_io_dispatch(io, efct_target_io_cb);
 710 }
 711
 712 int
 713 efct_scsi_send_rd_data(struct efct_io *io, u32 flags,
 714                        struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
 715                        efct_scsi_io_cb_t cb, void *arg)
 716 {
 717         return efct_scsi_xfer_data(io, flags, sgl, sgl_count,
 718                                    len, EFCT_HW_IO_TARGET_READ,
 719                                    enable_tsend_auto_resp(io->efct), cb, arg);
 720 }
 721
 722 int
 723 efct_scsi_recv_wr_data(struct efct_io *io, u32 flags,
 724                        struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
 725                        efct_scsi_io_cb_t cb, void *arg)
 726 {
 727         return efct_scsi_xfer_data(io, flags, sgl, sgl_count, len,
 728                                    EFCT_HW_IO_TARGET_WRITE,
 729                                    enable_treceive_auto_resp(io->efct), cb, arg);
 730 }
 731
 732 int
 733 efct_scsi_send_resp(struct efct_io *io, u32 flags,
 734                     struct efct_scsi_cmd_resp *rsp,
 735                     efct_scsi_io_cb_t cb, void *arg)
 736 {
 737         struct efct *efct;
 738         int residual;
 739         /* Always try auto resp */
 740         bool auto_resp = true;
 741         u8 scsi_status = 0;
 742         u16 scsi_status_qualifier = 0;
 743         u8 *sense_data = NULL;
 744         u32 sense_data_length = 0;
 745
 746         efct = io->efct;
 747
 748         if (rsp) {
 749                 scsi_status = rsp->scsi_status;
 750                 scsi_status_qualifier = rsp->scsi_status_qualifier;
 751                 sense_data = rsp->sense_data;
 752                 sense_data_length = rsp->sense_data_length;
 753                 residual = rsp->residual;
 754         } else {
 755                 residual = io->exp_xfer_len - io->transferred;
 756         }
 757
 758         io->wire_len = 0;
 759         io->hio_type = EFCT_HW_IO_TARGET_RSP;
 760
 761         io->scsi_tgt_cb = cb;
 762         io->scsi_tgt_cb_arg = arg;
 763
 764         memset(&io->iparam, 0, sizeof(io->iparam));
 765         io->iparam.fcp_tgt.ox_id = io->init_task_tag;
 766         io->iparam.fcp_tgt.offset = 0;
 767         io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
 768         io->iparam.fcp_tgt.timeout = io->timeout;
 769
 770         /* Set low latency queueing request */
 771         io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
 772         io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
 773                                 EFCT_SCSI_WQ_STEERING_SHIFT;
 774         io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
 775                                 EFCT_SCSI_WQ_CLASS_SHIFT;
 776
 777         if (scsi_status != 0 || residual || sense_data_length) {
 778                 struct fcp_resp_with_ext *fcprsp = io->rspbuf.virt;
 779                 u8 *sns_data;
 780
 781                 if (!fcprsp) {
 782                         efc_log_err(efct, "NULL response buffer\n");
 783                         return -EIO;
 784                 }
 785
 786                 sns_data = (u8 *)io->rspbuf.virt + sizeof(*fcprsp);
 787
 788                 auto_resp = false;
 789
 790                 memset(fcprsp, 0, sizeof(*fcprsp));
 791
 792                 io->wire_len += sizeof(*fcprsp);
 793
 794                 fcprsp->resp.fr_status = scsi_status;
 795                 fcprsp->resp.fr_retry_delay =
 796                         cpu_to_be16(scsi_status_qualifier);
 797
 798                 /* set residual status if necessary */
 799                 if (residual != 0) {
 800                         /* FCP: if data transferred is less than the
 801                          * amount expected, then this is an underflow.
 802                          * If data transferred would have been greater
 803                          * than the amount expected this is an overflow
 804                          */
 805                         if (residual > 0) {
 806                                 fcprsp->resp.fr_flags |= FCP_RESID_UNDER;
 807                                 fcprsp->ext.fr_resid =  cpu_to_be32(residual);
 808                         } else {
 809                                 fcprsp->resp.fr_flags |= FCP_RESID_OVER;
 810                                 fcprsp->ext.fr_resid = cpu_to_be32(-residual);
 811                         }
 812                 }
 813
 814                 if (EFCT_SCSI_SNS_BUF_VALID(sense_data) && sense_data_length) {
 815                         if (sense_data_length > SCSI_SENSE_BUFFERSIZE) {
 816                                 efc_log_err(efct, "Sense exceeds max size.\n");
 817                                 return -EIO;
 818                         }
 819
 820                         fcprsp->resp.fr_flags |= FCP_SNS_LEN_VAL;
 821                         memcpy(sns_data, sense_data, sense_data_length);
 822                         fcprsp->ext.fr_sns_len = cpu_to_be32(sense_data_length);
 823                         io->wire_len += sense_data_length;
 824                 }
 825
 826                 io->sgl[0].addr = io->rspbuf.phys;
 827                 io->sgl[0].dif_addr = 0;
 828                 io->sgl[0].len = io->wire_len;
 829                 io->sgl_count = 1;
 830         }
 831
 832         if (auto_resp)
 833                 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
 834
 835         return efct_scsi_io_dispatch(io, efct_target_io_cb);
 836 }
 837
 838 static int
 839 efct_target_bls_resp_cb(struct efct_hw_io *hio, u32 length, int status,
 840                         u32 ext_status, void *app)
 841 {
 842         struct efct_io *io = app;
 843         struct efct *efct;
 844         enum efct_scsi_io_status bls_status;
 845
 846         efct = io->efct;
 847
 848         /* BLS isn't really a "SCSI" concept, but use SCSI status */
 849         if (status) {
 850                 io_error_log(io, "s=%#x x=%#x\n", status, ext_status);
 851                 bls_status = EFCT_SCSI_STATUS_ERROR;
 852         } else {
 853                 bls_status = EFCT_SCSI_STATUS_GOOD;
 854         }
 855
 856         if (io->bls_cb) {
 857                 efct_scsi_io_cb_t bls_cb = io->bls_cb;
 858                 void *bls_cb_arg = io->bls_cb_arg;
 859
 860                 io->bls_cb = NULL;
 861                 io->bls_cb_arg = NULL;
 862
 863                 /* invoke callback */
 864                 bls_cb(io, bls_status, 0, bls_cb_arg);
 865         }
 866
 867         efct_scsi_check_pending(efct);
 868         return 0;
 869 }
 870
 871 static int
 872 efct_target_send_bls_resp(struct efct_io *io,
 873                           efct_scsi_io_cb_t cb, void *arg)
 874 {
 875         struct efct_node *node = io->node;
 876         struct sli_bls_params *bls = &io->iparam.bls;
 877         struct efct *efct = node->efct;
 878         struct fc_ba_acc *acc;
 879         int rc;
 880
 881         /* fill out IO structure with everything needed to send BA_ACC */
 882         memset(&io->iparam, 0, sizeof(io->iparam));
 883         bls->ox_id = io->init_task_tag;
 884         bls->rx_id = io->abort_rx_id;
 885         bls->vpi = io->node->vpi;
 886         bls->rpi = io->node->rpi;
 887         bls->s_id = U32_MAX;
 888         bls->d_id = io->node->node_fc_id;
 889         bls->rpi_registered = true;
 890
 891         acc = (void *)bls->payload;
 892         acc->ba_ox_id = cpu_to_be16(bls->ox_id);
 893         acc->ba_rx_id = cpu_to_be16(bls->rx_id);
 894         acc->ba_high_seq_cnt = cpu_to_be16(U16_MAX);
 895
 896         /* generic io fields have already been populated */
 897
 898         /* set type and BLS-specific fields */
 899         io->io_type = EFCT_IO_TYPE_BLS_RESP;
 900         io->display_name = "bls_rsp";
 901         io->hio_type = EFCT_HW_BLS_ACC;
 902         io->bls_cb = cb;
 903         io->bls_cb_arg = arg;
 904
 905         /* dispatch IO */
 906         rc = efct_hw_bls_send(efct, FC_RCTL_BA_ACC, bls,
 907                               efct_target_bls_resp_cb, io);
 908         return rc;
 909 }
 910
 911 static int efct_bls_send_rjt_cb(struct efct_hw_io *hio, u32 length, int status,
 912                                 u32 ext_status, void *app)
 913 {
 914         struct efct_io *io = app;
 915
 916         efct_scsi_io_free(io);
 917         return 0;
 918 }
 919
 920 struct efct_io *
 921 efct_bls_send_rjt(struct efct_io *io, struct fc_frame_header *hdr)
 922 {
 923         struct efct_node *node = io->node;
 924         struct sli_bls_params *bls = &io->iparam.bls;
 925         struct efct *efct = node->efct;
 926         struct fc_ba_rjt *acc;
 927         int rc;
 928
 929         /* fill out BLS Response-specific fields */
 930         io->io_type = EFCT_IO_TYPE_BLS_RESP;
 931         io->display_name = "ba_rjt";
 932         io->hio_type = EFCT_HW_BLS_RJT;
 933         io->init_task_tag = be16_to_cpu(hdr->fh_ox_id);
 934
 935         /* fill out iparam fields */
 936         memset(&io->iparam, 0, sizeof(io->iparam));
 937         bls->ox_id = be16_to_cpu(hdr->fh_ox_id);
 938         bls->rx_id = be16_to_cpu(hdr->fh_rx_id);
 939         bls->vpi = io->node->vpi;
 940         bls->rpi = io->node->rpi;
 941         bls->s_id = U32_MAX;
 942         bls->d_id = io->node->node_fc_id;
 943         bls->rpi_registered = true;
 944
 945         acc = (void *)bls->payload;
 946         acc->br_reason = ELS_RJT_UNAB;
 947         acc->br_explan = ELS_EXPL_NONE;
 948
 949         rc = efct_hw_bls_send(efct, FC_RCTL_BA_RJT, bls, efct_bls_send_rjt_cb,
 950                               io);
 951         if (rc) {
 952                 efc_log_err(efct, "efct_scsi_io_dispatch() failed: %d\n", rc);
 953                 efct_scsi_io_free(io);
 954                 io = NULL;
 955         }
 956         return io;
 957 }
 958
 959 int
 960 efct_scsi_send_tmf_resp(struct efct_io *io,
 961                         enum efct_scsi_tmf_resp rspcode,
 962                         u8 addl_rsp_info[3],
 963                         efct_scsi_io_cb_t cb, void *arg)
 964 {
 965         int rc;
 966         struct {
 967                 struct fcp_resp_with_ext rsp_ext;
 968                 struct fcp_resp_rsp_info info;
 969         } *fcprsp;
 970         u8 fcp_rspcode;
 971
 972         io->wire_len = 0;
 973
 974         switch (rspcode) {
 975         case EFCT_SCSI_TMF_FUNCTION_COMPLETE:
 976                 fcp_rspcode = FCP_TMF_CMPL;
 977                 break;
 978         case EFCT_SCSI_TMF_FUNCTION_SUCCEEDED:
 979         case EFCT_SCSI_TMF_FUNCTION_IO_NOT_FOUND:
 980                 fcp_rspcode = FCP_TMF_CMPL;
 981                 break;
 982         case EFCT_SCSI_TMF_FUNCTION_REJECTED:
 983                 fcp_rspcode = FCP_TMF_REJECTED;
 984                 break;
 985         case EFCT_SCSI_TMF_INCORRECT_LOGICAL_UNIT_NUMBER:
 986                 fcp_rspcode = FCP_TMF_INVALID_LUN;
 987                 break;
 988         case EFCT_SCSI_TMF_SERVICE_DELIVERY:
 989                 fcp_rspcode = FCP_TMF_FAILED;
 990                 break;
 991         default:
 992                 fcp_rspcode = FCP_TMF_REJECTED;
 993                 break;
 994         }
 995
 996         io->hio_type = EFCT_HW_IO_TARGET_RSP;
 997
 998         io->scsi_tgt_cb = cb;
 999         io->scsi_tgt_cb_arg = arg;
1000
1001         if (io->tmf_cmd == EFCT_SCSI_TMF_ABORT_TASK) {
1002                 rc = efct_target_send_bls_resp(io, cb, arg);
1003                 return rc;
1004         }
1005
1006         /* populate the FCP TMF response */
1007         fcprsp = io->rspbuf.virt;
1008         memset(fcprsp, 0, sizeof(*fcprsp));
1009
1010         fcprsp->rsp_ext.resp.fr_flags |= FCP_SNS_LEN_VAL;
1011
1012         if (addl_rsp_info) {
1013                 memcpy(fcprsp->info._fr_resvd, addl_rsp_info,
1014                        sizeof(fcprsp->info._fr_resvd));
1015         }
1016         fcprsp->info.rsp_code = fcp_rspcode;
1017
1018         io->wire_len = sizeof(*fcprsp);
1019
1020         fcprsp->rsp_ext.ext.fr_rsp_len =
1021                         cpu_to_be32(sizeof(struct fcp_resp_rsp_info));
1022
1023         io->sgl[0].addr = io->rspbuf.phys;
1024         io->sgl[0].dif_addr = 0;
1025         io->sgl[0].len = io->wire_len;
1026         io->sgl_count = 1;
1027
1028         memset(&io->iparam, 0, sizeof(io->iparam));
1029         io->iparam.fcp_tgt.ox_id = io->init_task_tag;
1030         io->iparam.fcp_tgt.offset = 0;
1031         io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
1032         io->iparam.fcp_tgt.timeout = io->timeout;
1033
1034         rc = efct_scsi_io_dispatch(io, efct_target_io_cb);
1035
1036         return rc;
1037 }
1038
1039 static int
1040 efct_target_abort_cb(struct efct_hw_io *hio, u32 length, int status,
1041                      u32 ext_status, void *app)
1042 {
1043         struct efct_io *io = app;
1044         struct efct *efct;
1045         enum efct_scsi_io_status scsi_status;
1046         efct_scsi_io_cb_t abort_cb;
1047         void *abort_cb_arg;
1048
1049         efct = io->efct;
1050
1051         if (!io->abort_cb)
1052                 goto done;
1053
1054         abort_cb = io->abort_cb;
1055         abort_cb_arg = io->abort_cb_arg;
1056
1057         io->abort_cb = NULL;
1058         io->abort_cb_arg = NULL;
1059
1060         switch (status) {
1061         case SLI4_FC_WCQE_STATUS_SUCCESS:
1062                 scsi_status = EFCT_SCSI_STATUS_GOOD;
1063                 break;
1064         case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
1065                 switch (ext_status) {
1066                 case SLI4_FC_LOCAL_REJECT_NO_XRI:
1067                         scsi_status = EFCT_SCSI_STATUS_NO_IO;
1068                         break;
1069                 case SLI4_FC_LOCAL_REJECT_ABORT_IN_PROGRESS:
1070                         scsi_status = EFCT_SCSI_STATUS_ABORT_IN_PROGRESS;
1071                         break;
1072                 default:
1073                         /*we have seen 0x15 (abort in progress)*/
1074                         scsi_status = EFCT_SCSI_STATUS_ERROR;
1075                         break;
1076                 }
1077                 break;
1078         case SLI4_FC_WCQE_STATUS_FCP_RSP_FAILURE:
1079                 scsi_status = EFCT_SCSI_STATUS_CHECK_RESPONSE;
1080                 break;
1081         default:
1082                 scsi_status = EFCT_SCSI_STATUS_ERROR;
1083                 break;
1084         }
1085         /* invoke callback */
1086         abort_cb(io->io_to_abort, scsi_status, 0, abort_cb_arg);
1087
1088 done:
1089         /* done with IO to abort,efct_ref_get(): efct_scsi_tgt_abort_io() */
1090         kref_put(&io->io_to_abort->ref, io->io_to_abort->release);
1091
1092         efct_io_pool_io_free(efct->xport->io_pool, io);
1093
1094         efct_scsi_check_pending(efct);
1095         return 0;
1096 }
1097
1098 int
1099 efct_scsi_tgt_abort_io(struct efct_io *io, efct_scsi_io_cb_t cb, void *arg)
1100 {
1101         struct efct *efct;
1102         struct efct_xport *xport;
1103         int rc;
1104         struct efct_io *abort_io = NULL;
1105
1106         efct = io->efct;
1107         xport = efct->xport;
1108
1109         /* take a reference on IO being aborted */
1110         if (kref_get_unless_zero(&io->ref) == 0) {
1111                 /* command no longer active */
1112                 scsi_io_printf(io, "command no longer active\n");
1113                 return -EIO;
1114         }
1115
1116         /*
1117          * allocate a new IO to send the abort request. Use efct_io_alloc()
1118          * directly, as we need an IO object that will not fail allocation
1119          * due to allocations being disabled (in efct_scsi_io_alloc())
1120          */
1121         abort_io = efct_io_pool_io_alloc(efct->xport->io_pool);
1122         if (!abort_io) {
1123                 atomic_add_return(1, &xport->io_alloc_failed_count);
1124                 kref_put(&io->ref, io->release);
1125                 return -EIO;
1126         }
1127
1128         /* Save the target server callback and argument */
1129         /* set generic fields */
1130         abort_io->cmd_tgt = true;
1131         abort_io->node = io->node;
1132
1133         /* set type and abort-specific fields */
1134         abort_io->io_type = EFCT_IO_TYPE_ABORT;
1135         abort_io->display_name = "tgt_abort";
1136         abort_io->io_to_abort = io;
1137         abort_io->send_abts = false;
1138         abort_io->abort_cb = cb;
1139         abort_io->abort_cb_arg = arg;
1140
1141         /* now dispatch IO */
1142         rc = efct_scsi_io_dispatch_abort(abort_io, efct_target_abort_cb);
1143         if (rc)
1144                 kref_put(&io->ref, io->release);
1145         return rc;
1146 }
1147
1148 void
1149 efct_scsi_io_complete(struct efct_io *io)
1150 {
1151         if (io->io_free) {
1152                 efc_log_debug(io->efct, "completion for non-busy io tag 0x%x\n",
1153                               io->tag);
1154                 return;
1155         }
1156
1157         scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
1158         kref_put(&io->ref, io->release);
1159 }