drivers/scsi/lpfc/lpfc_scsi.c

   1 /*******************************************************************
   2  * This file is part of the Emulex Linux Device Driver for         *
   3  * Fibre Channel Host Bus Adapters.                                *
   4  * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
   5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
   6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
   7  * EMULEX and SLI are trademarks of Emulex.                        *
   8  * www.broadcom.com                                                *
   9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
  10  *                                                                 *
  11  * This program is free software; you can redistribute it and/or   *
  12  * modify it under the terms of version 2 of the GNU General       *
  13  * Public License as published by the Free Software Foundation.    *
  14  * This program is distributed in the hope that it will be useful. *
  15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  20  * more details, a copy of which can be found in the file COPYING  *
  21  * included with this package.                                     *
  22  *******************************************************************/
  23 #include <linux/pci.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/export.h>
  27 #include <linux/delay.h>
  28 #include <asm/unaligned.h>
  29 #include <linux/t10-pi.h>
  30 #include <linux/crc-t10dif.h>
  31 #include <linux/blk-cgroup.h>
  32 #include <net/checksum.h>
  33
  34 #include <scsi/scsi.h>
  35 #include <scsi/scsi_device.h>
  36 #include <scsi/scsi_eh.h>
  37 #include <scsi/scsi_host.h>
  38 #include <scsi/scsi_tcq.h>
  39 #include <scsi/scsi_transport_fc.h>
  40
  41 #include "lpfc_version.h"
  42 #include "lpfc_hw4.h"
  43 #include "lpfc_hw.h"
  44 #include "lpfc_sli.h"
  45 #include "lpfc_sli4.h"
  46 #include "lpfc_nl.h"
  47 #include "lpfc_disc.h"
  48 #include "lpfc.h"
  49 #include "lpfc_scsi.h"
  50 #include "lpfc_logmsg.h"
  51 #include "lpfc_crtn.h"
  52 #include "lpfc_vport.h"
  53
  54 #define LPFC_RESET_WAIT  2
  55 #define LPFC_ABORT_WAIT  2
  56
  57 static char *dif_op_str[] = {
  58         "PROT_NORMAL",
  59         "PROT_READ_INSERT",
  60         "PROT_WRITE_STRIP",
  61         "PROT_READ_STRIP",
  62         "PROT_WRITE_INSERT",
  63         "PROT_READ_PASS",
  64         "PROT_WRITE_PASS",
  65 };
  66
  67 struct scsi_dif_tuple {
  68         __be16 guard_tag;       /* Checksum */
  69         __be16 app_tag;         /* Opaque storage */
  70         __be32 ref_tag;         /* Target LBA or indirect LBA */
  71 };
  72
  73 static struct lpfc_rport_data *
  74 lpfc_rport_data_from_scsi_device(struct scsi_device *sdev)
  75 {
  76         struct lpfc_vport *vport = (struct lpfc_vport *)sdev->host->hostdata;
  77
  78         if (vport->phba->cfg_fof)
  79                 return ((struct lpfc_device_data *)sdev->hostdata)->rport_data;
  80         else
  81                 return (struct lpfc_rport_data *)sdev->hostdata;
  82 }
  83
  84 static void
  85 lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb);
  86 static void
  87 lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb);
  88 static int
  89 lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc);
  90
  91 /**
  92  * lpfc_sli4_set_rsp_sgl_last - Set the last bit in the response sge.
  93  * @phba: Pointer to HBA object.
  94  * @lpfc_cmd: lpfc scsi command object pointer.
  95  *
  96  * This function is called from the lpfc_prep_task_mgmt_cmd function to
  97  * set the last bit in the response sge entry.
  98  **/
  99 static void
 100 lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba,
 101                                 struct lpfc_io_buf *lpfc_cmd)
 102 {
 103         struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
 104         if (sgl) {
 105                 sgl += 1;
 106                 sgl->word2 = le32_to_cpu(sgl->word2);
 107                 bf_set(lpfc_sli4_sge_last, sgl, 1);
 108                 sgl->word2 = cpu_to_le32(sgl->word2);
 109         }
 110 }
 111
 112 #define LPFC_INVALID_REFTAG ((u32)-1)
 113
 114 /**
 115  * lpfc_rampdown_queue_depth - Post RAMP_DOWN_QUEUE event to worker thread
 116  * @phba: The Hba for which this call is being executed.
 117  *
 118  * This routine is called when there is resource error in driver or firmware.
 119  * This routine posts WORKER_RAMP_DOWN_QUEUE event for @phba. This routine
 120  * posts at most 1 event each second. This routine wakes up worker thread of
 121  * @phba to process WORKER_RAM_DOWN_EVENT event.
 122  *
 123  * This routine should be called with no lock held.
 124  **/
 125 void
 126 lpfc_rampdown_queue_depth(struct lpfc_hba *phba)
 127 {
 128         unsigned long flags;
 129         uint32_t evt_posted;
 130         unsigned long expires;
 131
 132         spin_lock_irqsave(&phba->hbalock, flags);
 133         atomic_inc(&phba->num_rsrc_err);
 134         phba->last_rsrc_error_time = jiffies;
 135
 136         expires = phba->last_ramp_down_time + QUEUE_RAMP_DOWN_INTERVAL;
 137         if (time_after(expires, jiffies)) {
 138                 spin_unlock_irqrestore(&phba->hbalock, flags);
 139                 return;
 140         }
 141
 142         phba->last_ramp_down_time = jiffies;
 143
 144         spin_unlock_irqrestore(&phba->hbalock, flags);
 145
 146         spin_lock_irqsave(&phba->pport->work_port_lock, flags);
 147         evt_posted = phba->pport->work_port_events & WORKER_RAMP_DOWN_QUEUE;
 148         if (!evt_posted)
 149                 phba->pport->work_port_events |= WORKER_RAMP_DOWN_QUEUE;
 150         spin_unlock_irqrestore(&phba->pport->work_port_lock, flags);
 151
 152         if (!evt_posted)
 153                 lpfc_worker_wake_up(phba);
 154         return;
 155 }
 156
 157 /**
 158  * lpfc_ramp_down_queue_handler - WORKER_RAMP_DOWN_QUEUE event handler
 159  * @phba: The Hba for which this call is being executed.
 160  *
 161  * This routine is called to  process WORKER_RAMP_DOWN_QUEUE event for worker
 162  * thread.This routine reduces queue depth for all scsi device on each vport
 163  * associated with @phba.
 164  **/
 165 void
 166 lpfc_ramp_down_queue_handler(struct lpfc_hba *phba)
 167 {
 168         struct lpfc_vport **vports;
 169         struct Scsi_Host  *shost;
 170         struct scsi_device *sdev;
 171         unsigned long new_queue_depth;
 172         unsigned long num_rsrc_err, num_cmd_success;
 173         int i;
 174
 175         num_rsrc_err = atomic_read(&phba->num_rsrc_err);
 176         num_cmd_success = atomic_read(&phba->num_cmd_success);
 177
 178         /*
 179          * The error and success command counters are global per
 180          * driver instance.  If another handler has already
 181          * operated on this error event, just exit.
 182          */
 183         if (num_rsrc_err == 0)
 184                 return;
 185
 186         vports = lpfc_create_vport_work_array(phba);
 187         if (vports != NULL)
 188                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
 189                         shost = lpfc_shost_from_vport(vports[i]);
 190                         shost_for_each_device(sdev, shost) {
 191                                 new_queue_depth =
 192                                         sdev->queue_depth * num_rsrc_err /
 193                                         (num_rsrc_err + num_cmd_success);
 194                                 if (!new_queue_depth)
 195                                         new_queue_depth = sdev->queue_depth - 1;
 196                                 else
 197                                         new_queue_depth = sdev->queue_depth -
 198                                                                 new_queue_depth;
 199                                 scsi_change_queue_depth(sdev, new_queue_depth);
 200                         }
 201                 }
 202         lpfc_destroy_vport_work_array(phba, vports);
 203         atomic_set(&phba->num_rsrc_err, 0);
 204         atomic_set(&phba->num_cmd_success, 0);
 205 }
 206
 207 /**
 208  * lpfc_scsi_dev_block - set all scsi hosts to block state
 209  * @phba: Pointer to HBA context object.
 210  *
 211  * This function walks vport list and set each SCSI host to block state
 212  * by invoking fc_remote_port_delete() routine. This function is invoked
 213  * with EEH when device's PCI slot has been permanently disabled.
 214  **/
 215 void
 216 lpfc_scsi_dev_block(struct lpfc_hba *phba)
 217 {
 218         struct lpfc_vport **vports;
 219         struct Scsi_Host  *shost;
 220         struct scsi_device *sdev;
 221         struct fc_rport *rport;
 222         int i;
 223
 224         vports = lpfc_create_vport_work_array(phba);
 225         if (vports != NULL)
 226                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
 227                         shost = lpfc_shost_from_vport(vports[i]);
 228                         shost_for_each_device(sdev, shost) {
 229                                 rport = starget_to_rport(scsi_target(sdev));
 230                                 fc_remote_port_delete(rport);
 231                         }
 232                 }
 233         lpfc_destroy_vport_work_array(phba, vports);
 234 }
 235
 236 /**
 237  * lpfc_new_scsi_buf_s3 - Scsi buffer allocator for HBA with SLI3 IF spec
 238  * @vport: The virtual port for which this call being executed.
 239  * @num_to_alloc: The requested number of buffers to allocate.
 240  *
 241  * This routine allocates a scsi buffer for device with SLI-3 interface spec,
 242  * the scsi buffer contains all the necessary information needed to initiate
 243  * a SCSI I/O. The non-DMAable buffer region contains information to build
 244  * the IOCB. The DMAable region contains memory for the FCP CMND, FCP RSP,
 245  * and the initial BPL. In addition to allocating memory, the FCP CMND and
 246  * FCP RSP BDEs are setup in the BPL and the BPL BDE is setup in the IOCB.
 247  *
 248  * Return codes:
 249  *   int - number of scsi buffers that were allocated.
 250  *   0 = failure, less than num_to_alloc is a partial failure.
 251  **/
 252 static int
 253 lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc)
 254 {
 255         struct lpfc_hba *phba = vport->phba;
 256         struct lpfc_io_buf *psb;
 257         struct ulp_bde64 *bpl;
 258         IOCB_t *iocb;
 259         dma_addr_t pdma_phys_fcp_cmd;
 260         dma_addr_t pdma_phys_fcp_rsp;
 261         dma_addr_t pdma_phys_sgl;
 262         uint16_t iotag;
 263         int bcnt, bpl_size;
 264
 265         bpl_size = phba->cfg_sg_dma_buf_size -
 266                 (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp));
 267
 268         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
 269                          "9067 ALLOC %d scsi_bufs: %d (%d + %d + %d)\n",
 270                          num_to_alloc, phba->cfg_sg_dma_buf_size,
 271                          (int)sizeof(struct fcp_cmnd),
 272                          (int)sizeof(struct fcp_rsp), bpl_size);
 273
 274         for (bcnt = 0; bcnt < num_to_alloc; bcnt++) {
 275                 psb = kzalloc(sizeof(struct lpfc_io_buf), GFP_KERNEL);
 276                 if (!psb)
 277                         break;
 278
 279                 /*
 280                  * Get memory from the pci pool to map the virt space to pci
 281                  * bus space for an I/O.  The DMA buffer includes space for the
 282                  * struct fcp_cmnd, struct fcp_rsp and the number of bde's
 283                  * necessary to support the sg_tablesize.
 284                  */
 285                 psb->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
 286                                         GFP_KERNEL, &psb->dma_handle);
 287                 if (!psb->data) {
 288                         kfree(psb);
 289                         break;
 290                 }
 291
 292
 293                 /* Allocate iotag for psb->cur_iocbq. */
 294                 iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
 295                 if (iotag == 0) {
 296                         dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 297                                       psb->data, psb->dma_handle);
 298                         kfree(psb);
 299                         break;
 300                 }
 301                 psb->cur_iocbq.cmd_flag |= LPFC_IO_FCP;
 302
 303                 psb->fcp_cmnd = psb->data;
 304                 psb->fcp_rsp = psb->data + sizeof(struct fcp_cmnd);
 305                 psb->dma_sgl = psb->data + sizeof(struct fcp_cmnd) +
 306                         sizeof(struct fcp_rsp);
 307
 308                 /* Initialize local short-hand pointers. */
 309                 bpl = (struct ulp_bde64 *)psb->dma_sgl;
 310                 pdma_phys_fcp_cmd = psb->dma_handle;
 311                 pdma_phys_fcp_rsp = psb->dma_handle + sizeof(struct fcp_cmnd);
 312                 pdma_phys_sgl = psb->dma_handle + sizeof(struct fcp_cmnd) +
 313                         sizeof(struct fcp_rsp);
 314
 315                 /*
 316                  * The first two bdes are the FCP_CMD and FCP_RSP. The balance
 317                  * are sg list bdes.  Initialize the first two and leave the
 318                  * rest for queuecommand.
 319                  */
 320                 bpl[0].addrHigh = le32_to_cpu(putPaddrHigh(pdma_phys_fcp_cmd));
 321                 bpl[0].addrLow = le32_to_cpu(putPaddrLow(pdma_phys_fcp_cmd));
 322                 bpl[0].tus.f.bdeSize = sizeof(struct fcp_cmnd);
 323                 bpl[0].tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 324                 bpl[0].tus.w = le32_to_cpu(bpl[0].tus.w);
 325
 326                 /* Setup the physical region for the FCP RSP */
 327                 bpl[1].addrHigh = le32_to_cpu(putPaddrHigh(pdma_phys_fcp_rsp));
 328                 bpl[1].addrLow = le32_to_cpu(putPaddrLow(pdma_phys_fcp_rsp));
 329                 bpl[1].tus.f.bdeSize = sizeof(struct fcp_rsp);
 330                 bpl[1].tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 331                 bpl[1].tus.w = le32_to_cpu(bpl[1].tus.w);
 332
 333                 /*
 334                  * Since the IOCB for the FCP I/O is built into this
 335                  * lpfc_scsi_buf, initialize it with all known data now.
 336                  */
 337                 iocb = &psb->cur_iocbq.iocb;
 338                 iocb->un.fcpi64.bdl.ulpIoTag32 = 0;
 339                 if ((phba->sli_rev == 3) &&
 340                                 !(phba->sli3_options & LPFC_SLI3_BG_ENABLED)) {
 341                         /* fill in immediate fcp command BDE */
 342                         iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BDE_IMMED;
 343                         iocb->un.fcpi64.bdl.bdeSize = sizeof(struct fcp_cmnd);
 344                         iocb->un.fcpi64.bdl.addrLow = offsetof(IOCB_t,
 345                                         unsli3.fcp_ext.icd);
 346                         iocb->un.fcpi64.bdl.addrHigh = 0;
 347                         iocb->ulpBdeCount = 0;
 348                         iocb->ulpLe = 0;
 349                         /* fill in response BDE */
 350                         iocb->unsli3.fcp_ext.rbde.tus.f.bdeFlags =
 351                                                         BUFF_TYPE_BDE_64;
 352                         iocb->unsli3.fcp_ext.rbde.tus.f.bdeSize =
 353                                 sizeof(struct fcp_rsp);
 354                         iocb->unsli3.fcp_ext.rbde.addrLow =
 355                                 putPaddrLow(pdma_phys_fcp_rsp);
 356                         iocb->unsli3.fcp_ext.rbde.addrHigh =
 357                                 putPaddrHigh(pdma_phys_fcp_rsp);
 358                 } else {
 359                         iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BLP_64;
 360                         iocb->un.fcpi64.bdl.bdeSize =
 361                                         (2 * sizeof(struct ulp_bde64));
 362                         iocb->un.fcpi64.bdl.addrLow =
 363                                         putPaddrLow(pdma_phys_sgl);
 364                         iocb->un.fcpi64.bdl.addrHigh =
 365                                         putPaddrHigh(pdma_phys_sgl);
 366                         iocb->ulpBdeCount = 1;
 367                         iocb->ulpLe = 1;
 368                 }
 369                 iocb->ulpClass = CLASS3;
 370                 psb->status = IOSTAT_SUCCESS;
 371                 /* Put it back into the SCSI buffer list */
 372                 psb->cur_iocbq.io_buf = psb;
 373                 spin_lock_init(&psb->buf_lock);
 374                 lpfc_release_scsi_buf_s3(phba, psb);
 375
 376         }
 377
 378         return bcnt;
 379 }
 380
 381 /**
 382  * lpfc_sli4_vport_delete_fcp_xri_aborted -Remove all ndlp references for vport
 383  * @vport: pointer to lpfc vport data structure.
 384  *
 385  * This routine is invoked by the vport cleanup for deletions and the cleanup
 386  * for an ndlp on removal.
 387  **/
 388 void
 389 lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport)
 390 {
 391         struct lpfc_hba *phba = vport->phba;
 392         struct lpfc_io_buf *psb, *next_psb;
 393         struct lpfc_sli4_hdw_queue *qp;
 394         unsigned long iflag = 0;
 395         int idx;
 396
 397         if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
 398                 return;
 399
 400         spin_lock_irqsave(&phba->hbalock, iflag);
 401         for (idx = 0; idx < phba->cfg_hdw_queue; idx++) {
 402                 qp = &phba->sli4_hba.hdwq[idx];
 403
 404                 spin_lock(&qp->abts_io_buf_list_lock);
 405                 list_for_each_entry_safe(psb, next_psb,
 406                                          &qp->lpfc_abts_io_buf_list, list) {
 407                         if (psb->cur_iocbq.cmd_flag & LPFC_IO_NVME)
 408                                 continue;
 409
 410                         if (psb->rdata && psb->rdata->pnode &&
 411                             psb->rdata->pnode->vport == vport)
 412                                 psb->rdata = NULL;
 413                 }
 414                 spin_unlock(&qp->abts_io_buf_list_lock);
 415         }
 416         spin_unlock_irqrestore(&phba->hbalock, iflag);
 417 }
 418
 419 /**
 420  * lpfc_sli4_io_xri_aborted - Fast-path process of fcp xri abort
 421  * @phba: pointer to lpfc hba data structure.
 422  * @axri: pointer to the fcp xri abort wcqe structure.
 423  * @idx: index into hdwq
 424  *
 425  * This routine is invoked by the worker thread to process a SLI4 fast-path
 426  * FCP or NVME aborted xri.
 427  **/
 428 void
 429 lpfc_sli4_io_xri_aborted(struct lpfc_hba *phba,
 430                          struct sli4_wcqe_xri_aborted *axri, int idx)
 431 {
 432         u16 xri = 0;
 433         u16 rxid = 0;
 434         struct lpfc_io_buf *psb, *next_psb;
 435         struct lpfc_sli4_hdw_queue *qp;
 436         unsigned long iflag = 0;
 437         struct lpfc_iocbq *iocbq;
 438         int i;
 439         struct lpfc_nodelist *ndlp;
 440         int rrq_empty = 0;
 441         struct lpfc_sli_ring *pring = phba->sli4_hba.els_wq->pring;
 442         struct scsi_cmnd *cmd;
 443         int offline = 0;
 444
 445         if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
 446                 return;
 447         offline = pci_channel_offline(phba->pcidev);
 448         if (!offline) {
 449                 xri = bf_get(lpfc_wcqe_xa_xri, axri);
 450                 rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
 451         }
 452         qp = &phba->sli4_hba.hdwq[idx];
 453         spin_lock_irqsave(&phba->hbalock, iflag);
 454         spin_lock(&qp->abts_io_buf_list_lock);
 455         list_for_each_entry_safe(psb, next_psb,
 456                 &qp->lpfc_abts_io_buf_list, list) {
 457                 if (offline)
 458                         xri = psb->cur_iocbq.sli4_xritag;
 459                 if (psb->cur_iocbq.sli4_xritag == xri) {
 460                         list_del_init(&psb->list);
 461                         psb->flags &= ~LPFC_SBUF_XBUSY;
 462                         psb->status = IOSTAT_SUCCESS;
 463                         if (psb->cur_iocbq.cmd_flag & LPFC_IO_NVME) {
 464                                 qp->abts_nvme_io_bufs--;
 465                                 spin_unlock(&qp->abts_io_buf_list_lock);
 466                                 spin_unlock_irqrestore(&phba->hbalock, iflag);
 467                                 if (!offline) {
 468                                         lpfc_sli4_nvme_xri_aborted(phba, axri,
 469                                                                    psb);
 470                                         return;
 471                                 }
 472                                 lpfc_sli4_nvme_pci_offline_aborted(phba, psb);
 473                                 spin_lock_irqsave(&phba->hbalock, iflag);
 474                                 spin_lock(&qp->abts_io_buf_list_lock);
 475                                 continue;
 476                         }
 477                         qp->abts_scsi_io_bufs--;
 478                         spin_unlock(&qp->abts_io_buf_list_lock);
 479
 480                         if (psb->rdata && psb->rdata->pnode)
 481                                 ndlp = psb->rdata->pnode;
 482                         else
 483                                 ndlp = NULL;
 484
 485                         rrq_empty = list_empty(&phba->active_rrq_list);
 486                         spin_unlock_irqrestore(&phba->hbalock, iflag);
 487                         if (ndlp && !offline) {
 488                                 lpfc_set_rrq_active(phba, ndlp,
 489                                         psb->cur_iocbq.sli4_lxritag, rxid, 1);
 490                                 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
 491                         }
 492
 493                         if (phba->cfg_fcp_wait_abts_rsp || offline) {
 494                                 spin_lock_irqsave(&psb->buf_lock, iflag);
 495                                 cmd = psb->pCmd;
 496                                 psb->pCmd = NULL;
 497                                 spin_unlock_irqrestore(&psb->buf_lock, iflag);
 498
 499                                 /* The sdev is not guaranteed to be valid post
 500                                  * scsi_done upcall.
 501                                  */
 502                                 if (cmd)
 503                                         scsi_done(cmd);
 504
 505                                 /*
 506                                  * We expect there is an abort thread waiting
 507                                  * for command completion wake up the thread.
 508                                  */
 509                                 spin_lock_irqsave(&psb->buf_lock, iflag);
 510                                 psb->cur_iocbq.cmd_flag &=
 511                                         ~LPFC_DRIVER_ABORTED;
 512                                 if (psb->waitq)
 513                                         wake_up(psb->waitq);
 514                                 spin_unlock_irqrestore(&psb->buf_lock, iflag);
 515                         }
 516
 517                         lpfc_release_scsi_buf_s4(phba, psb);
 518                         if (rrq_empty)
 519                                 lpfc_worker_wake_up(phba);
 520                         if (!offline)
 521                                 return;
 522                         spin_lock_irqsave(&phba->hbalock, iflag);
 523                         spin_lock(&qp->abts_io_buf_list_lock);
 524                         continue;
 525                 }
 526         }
 527         spin_unlock(&qp->abts_io_buf_list_lock);
 528         if (!offline) {
 529                 for (i = 1; i <= phba->sli.last_iotag; i++) {
 530                         iocbq = phba->sli.iocbq_lookup[i];
 531
 532                         if (!(iocbq->cmd_flag & LPFC_IO_FCP) ||
 533                             (iocbq->cmd_flag & LPFC_IO_LIBDFC))
 534                                 continue;
 535                         if (iocbq->sli4_xritag != xri)
 536                                 continue;
 537                         psb = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
 538                         psb->flags &= ~LPFC_SBUF_XBUSY;
 539                         spin_unlock_irqrestore(&phba->hbalock, iflag);
 540                         if (!list_empty(&pring->txq))
 541                                 lpfc_worker_wake_up(phba);
 542                         return;
 543                 }
 544         }
 545         spin_unlock_irqrestore(&phba->hbalock, iflag);
 546 }
 547
 548 /**
 549  * lpfc_get_scsi_buf_s3 - Get a scsi buffer from lpfc_scsi_buf_list of the HBA
 550  * @phba: The HBA for which this call is being executed.
 551  * @ndlp: pointer to a node-list data structure.
 552  * @cmnd: Pointer to scsi_cmnd data structure.
 553  *
 554  * This routine removes a scsi buffer from head of @phba lpfc_scsi_buf_list list
 555  * and returns to caller.
 556  *
 557  * Return codes:
 558  *   NULL - Error
 559  *   Pointer to lpfc_scsi_buf - Success
 560  **/
 561 static struct lpfc_io_buf *
 562 lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 563                      struct scsi_cmnd *cmnd)
 564 {
 565         struct lpfc_io_buf *lpfc_cmd = NULL;
 566         struct list_head *scsi_buf_list_get = &phba->lpfc_scsi_buf_list_get;
 567         unsigned long iflag = 0;
 568
 569         spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag);
 570         list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_io_buf,
 571                          list);
 572         if (!lpfc_cmd) {
 573                 spin_lock(&phba->scsi_buf_list_put_lock);
 574                 list_splice(&phba->lpfc_scsi_buf_list_put,
 575                             &phba->lpfc_scsi_buf_list_get);
 576                 INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put);
 577                 list_remove_head(scsi_buf_list_get, lpfc_cmd,
 578                                  struct lpfc_io_buf, list);
 579                 spin_unlock(&phba->scsi_buf_list_put_lock);
 580         }
 581         spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag);
 582
 583         if (lpfc_ndlp_check_qdepth(phba, ndlp) && lpfc_cmd) {
 584                 atomic_inc(&ndlp->cmd_pending);
 585                 lpfc_cmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
 586         }
 587         return  lpfc_cmd;
 588 }
 589 /**
 590  * lpfc_get_scsi_buf_s4 - Get a scsi buffer from io_buf_list of the HBA
 591  * @phba: The HBA for which this call is being executed.
 592  * @ndlp: pointer to a node-list data structure.
 593  * @cmnd: Pointer to scsi_cmnd data structure.
 594  *
 595  * This routine removes a scsi buffer from head of @hdwq io_buf_list
 596  * and returns to caller.
 597  *
 598  * Return codes:
 599  *   NULL - Error
 600  *   Pointer to lpfc_scsi_buf - Success
 601  **/
 602 static struct lpfc_io_buf *
 603 lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 604                      struct scsi_cmnd *cmnd)
 605 {
 606         struct lpfc_io_buf *lpfc_cmd;
 607         struct lpfc_sli4_hdw_queue *qp;
 608         struct sli4_sge *sgl;
 609         dma_addr_t pdma_phys_fcp_rsp;
 610         dma_addr_t pdma_phys_fcp_cmd;
 611         uint32_t cpu, idx;
 612         int tag;
 613         struct fcp_cmd_rsp_buf *tmp = NULL;
 614
 615         cpu = raw_smp_processor_id();
 616         if (cmnd && phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
 617                 tag = blk_mq_unique_tag(scsi_cmd_to_rq(cmnd));
 618                 idx = blk_mq_unique_tag_to_hwq(tag);
 619         } else {
 620                 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
 621         }
 622
 623         lpfc_cmd = lpfc_get_io_buf(phba, ndlp, idx,
 624                                    !phba->cfg_xri_rebalancing);
 625         if (!lpfc_cmd) {
 626                 qp = &phba->sli4_hba.hdwq[idx];
 627                 qp->empty_io_bufs++;
 628                 return NULL;
 629         }
 630
 631         /* Setup key fields in buffer that may have been changed
 632          * if other protocols used this buffer.
 633          */
 634         lpfc_cmd->cur_iocbq.cmd_flag = LPFC_IO_FCP;
 635         lpfc_cmd->prot_seg_cnt = 0;
 636         lpfc_cmd->seg_cnt = 0;
 637         lpfc_cmd->timeout = 0;
 638         lpfc_cmd->flags = 0;
 639         lpfc_cmd->start_time = jiffies;
 640         lpfc_cmd->waitq = NULL;
 641         lpfc_cmd->cpu = cpu;
 642 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 643         lpfc_cmd->prot_data_type = 0;
 644 #endif
 645         tmp = lpfc_get_cmd_rsp_buf_per_hdwq(phba, lpfc_cmd);
 646         if (!tmp) {
 647                 lpfc_release_io_buf(phba, lpfc_cmd, lpfc_cmd->hdwq);
 648                 return NULL;
 649         }
 650
 651         lpfc_cmd->fcp_cmnd = tmp->fcp_cmnd;
 652         lpfc_cmd->fcp_rsp = tmp->fcp_rsp;
 653
 654         /*
 655          * The first two SGEs are the FCP_CMD and FCP_RSP.
 656          * The balance are sg list bdes. Initialize the
 657          * first two and leave the rest for queuecommand.
 658          */
 659         sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
 660         pdma_phys_fcp_cmd = tmp->fcp_cmd_rsp_dma_handle;
 661         sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_cmd));
 662         sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_cmd));
 663         sgl->word2 = le32_to_cpu(sgl->word2);
 664         bf_set(lpfc_sli4_sge_last, sgl, 0);
 665         sgl->word2 = cpu_to_le32(sgl->word2);
 666         sgl->sge_len = cpu_to_le32(sizeof(struct fcp_cmnd));
 667         sgl++;
 668
 669         /* Setup the physical region for the FCP RSP */
 670         pdma_phys_fcp_rsp = pdma_phys_fcp_cmd + sizeof(struct fcp_cmnd);
 671         sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_rsp));
 672         sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_rsp));
 673         sgl->word2 = le32_to_cpu(sgl->word2);
 674         bf_set(lpfc_sli4_sge_last, sgl, 1);
 675         sgl->word2 = cpu_to_le32(sgl->word2);
 676         sgl->sge_len = cpu_to_le32(sizeof(struct fcp_rsp));
 677
 678         if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
 679                 atomic_inc(&ndlp->cmd_pending);
 680                 lpfc_cmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
 681         }
 682         return  lpfc_cmd;
 683 }
 684 /**
 685  * lpfc_get_scsi_buf - Get a scsi buffer from lpfc_scsi_buf_list of the HBA
 686  * @phba: The HBA for which this call is being executed.
 687  * @ndlp: pointer to a node-list data structure.
 688  * @cmnd: Pointer to scsi_cmnd data structure.
 689  *
 690  * This routine removes a scsi buffer from head of @phba lpfc_scsi_buf_list list
 691  * and returns to caller.
 692  *
 693  * Return codes:
 694  *   NULL - Error
 695  *   Pointer to lpfc_scsi_buf - Success
 696  **/
 697 static struct lpfc_io_buf*
 698 lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 699                   struct scsi_cmnd *cmnd)
 700 {
 701         return  phba->lpfc_get_scsi_buf(phba, ndlp, cmnd);
 702 }
 703
 704 /**
 705  * lpfc_release_scsi_buf_s3 - Return a scsi buffer back to hba scsi buf list
 706  * @phba: The Hba for which this call is being executed.
 707  * @psb: The scsi buffer which is being released.
 708  *
 709  * This routine releases @psb scsi buffer by adding it to tail of @phba
 710  * lpfc_scsi_buf_list list.
 711  **/
 712 static void
 713 lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
 714 {
 715         unsigned long iflag = 0;
 716
 717         psb->seg_cnt = 0;
 718         psb->prot_seg_cnt = 0;
 719
 720         spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag);
 721         psb->pCmd = NULL;
 722         psb->cur_iocbq.cmd_flag = LPFC_IO_FCP;
 723         list_add_tail(&psb->list, &phba->lpfc_scsi_buf_list_put);
 724         spin_unlock_irqrestore(&phba->scsi_buf_list_put_lock, iflag);
 725 }
 726
 727 /**
 728  * lpfc_release_scsi_buf_s4: Return a scsi buffer back to hba scsi buf list.
 729  * @phba: The Hba for which this call is being executed.
 730  * @psb: The scsi buffer which is being released.
 731  *
 732  * This routine releases @psb scsi buffer by adding it to tail of @hdwq
 733  * io_buf_list list. For SLI4 XRI's are tied to the scsi buffer
 734  * and cannot be reused for at least RA_TOV amount of time if it was
 735  * aborted.
 736  **/
 737 static void
 738 lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
 739 {
 740         struct lpfc_sli4_hdw_queue *qp;
 741         unsigned long iflag = 0;
 742
 743         psb->seg_cnt = 0;
 744         psb->prot_seg_cnt = 0;
 745
 746         qp = psb->hdwq;
 747         if (psb->flags & LPFC_SBUF_XBUSY) {
 748                 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
 749                 if (!phba->cfg_fcp_wait_abts_rsp)
 750                         psb->pCmd = NULL;
 751                 list_add_tail(&psb->list, &qp->lpfc_abts_io_buf_list);
 752                 qp->abts_scsi_io_bufs++;
 753                 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
 754         } else {
 755                 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)psb, qp);
 756         }
 757 }
 758
 759 /**
 760  * lpfc_release_scsi_buf: Return a scsi buffer back to hba scsi buf list.
 761  * @phba: The Hba for which this call is being executed.
 762  * @psb: The scsi buffer which is being released.
 763  *
 764  * This routine releases @psb scsi buffer by adding it to tail of @phba
 765  * lpfc_scsi_buf_list list.
 766  **/
 767 static void
 768 lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
 769 {
 770         if ((psb->flags & LPFC_SBUF_BUMP_QDEPTH) && psb->ndlp)
 771                 atomic_dec(&psb->ndlp->cmd_pending);
 772
 773         psb->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
 774         phba->lpfc_release_scsi_buf(phba, psb);
 775 }
 776
 777 /**
 778  * lpfc_fcpcmd_to_iocb - copy the fcp_cmd data into the IOCB
 779  * @data: A pointer to the immediate command data portion of the IOCB.
 780  * @fcp_cmnd: The FCP Command that is provided by the SCSI layer.
 781  *
 782  * The routine copies the entire FCP command from @fcp_cmnd to @data while
 783  * byte swapping the data to big endian format for transmission on the wire.
 784  **/
 785 static void
 786 lpfc_fcpcmd_to_iocb(u8 *data, struct fcp_cmnd *fcp_cmnd)
 787 {
 788         int i, j;
 789
 790         for (i = 0, j = 0; i < sizeof(struct fcp_cmnd);
 791              i += sizeof(uint32_t), j++) {
 792                 ((uint32_t *)data)[j] = cpu_to_be32(((uint32_t *)fcp_cmnd)[j]);
 793         }
 794 }
 795
 796 /**
 797  * lpfc_scsi_prep_dma_buf_s3 - DMA mapping for scsi buffer to SLI3 IF spec
 798  * @phba: The Hba for which this call is being executed.
 799  * @lpfc_cmd: The scsi buffer which is going to be mapped.
 800  *
 801  * This routine does the pci dma mapping for scatter-gather list of scsi cmnd
 802  * field of @lpfc_cmd for device with SLI-3 interface spec. This routine scans
 803  * through sg elements and format the bde. This routine also initializes all
 804  * IOCB fields which are dependent on scsi command request buffer.
 805  *
 806  * Return codes:
 807  *   1 - Error
 808  *   0 - Success
 809  **/
 810 static int
 811 lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
 812 {
 813         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
 814         struct scatterlist *sgel = NULL;
 815         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
 816         struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl;
 817         struct lpfc_iocbq *iocbq = &lpfc_cmd->cur_iocbq;
 818         IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb;
 819         struct ulp_bde64 *data_bde = iocb_cmd->unsli3.fcp_ext.dbde;
 820         dma_addr_t physaddr;
 821         uint32_t num_bde = 0;
 822         int nseg, datadir = scsi_cmnd->sc_data_direction;
 823
 824         /*
 825          * There are three possibilities here - use scatter-gather segment, use
 826          * the single mapping, or neither.  Start the lpfc command prep by
 827          * bumping the bpl beyond the fcp_cmnd and fcp_rsp regions to the first
 828          * data bde entry.
 829          */
 830         bpl += 2;
 831         if (scsi_sg_count(scsi_cmnd)) {
 832                 /*
 833                  * The driver stores the segment count returned from dma_map_sg
 834                  * because this a count of dma-mappings used to map the use_sg
 835                  * pages.  They are not guaranteed to be the same for those
 836                  * architectures that implement an IOMMU.
 837                  */
 838
 839                 nseg = dma_map_sg(&phba->pcidev->dev, scsi_sglist(scsi_cmnd),
 840                                   scsi_sg_count(scsi_cmnd), datadir);
 841                 if (unlikely(!nseg))
 842                         return 1;
 843
 844                 lpfc_cmd->seg_cnt = nseg;
 845                 if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
 846                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
 847                                         "9064 BLKGRD: %s: Too many sg segments"
 848                                         " from dma_map_sg.  Config %d, seg_cnt"
 849                                         " %d\n", __func__, phba->cfg_sg_seg_cnt,
 850                                         lpfc_cmd->seg_cnt);
 851                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
 852                         lpfc_cmd->seg_cnt = 0;
 853                         scsi_dma_unmap(scsi_cmnd);
 854                         return 2;
 855                 }
 856
 857                 /*
 858                  * The driver established a maximum scatter-gather segment count
 859                  * during probe that limits the number of sg elements in any
 860                  * single scsi command.  Just run through the seg_cnt and format
 861                  * the bde's.
 862                  * When using SLI-3 the driver will try to fit all the BDEs into
 863                  * the IOCB. If it can't then the BDEs get added to a BPL as it
 864                  * does for SLI-2 mode.
 865                  */
 866                 scsi_for_each_sg(scsi_cmnd, sgel, nseg, num_bde) {
 867                         physaddr = sg_dma_address(sgel);
 868                         if (phba->sli_rev == 3 &&
 869                             !(phba->sli3_options & LPFC_SLI3_BG_ENABLED) &&
 870                             !(iocbq->cmd_flag & DSS_SECURITY_OP) &&
 871                             nseg <= LPFC_EXT_DATA_BDE_COUNT) {
 872                                 data_bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 873                                 data_bde->tus.f.bdeSize = sg_dma_len(sgel);
 874                                 data_bde->addrLow = putPaddrLow(physaddr);
 875                                 data_bde->addrHigh = putPaddrHigh(physaddr);
 876                                 data_bde++;
 877                         } else {
 878                                 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 879                                 bpl->tus.f.bdeSize = sg_dma_len(sgel);
 880                                 bpl->tus.w = le32_to_cpu(bpl->tus.w);
 881                                 bpl->addrLow =
 882                                         le32_to_cpu(putPaddrLow(physaddr));
 883                                 bpl->addrHigh =
 884                                         le32_to_cpu(putPaddrHigh(physaddr));
 885                                 bpl++;
 886                         }
 887                 }
 888         }
 889
 890         /*
 891          * Finish initializing those IOCB fields that are dependent on the
 892          * scsi_cmnd request_buffer.  Note that for SLI-2 the bdeSize is
 893          * explicitly reinitialized and for SLI-3 the extended bde count is
 894          * explicitly reinitialized since all iocb memory resources are reused.
 895          */
 896         if (phba->sli_rev == 3 &&
 897             !(phba->sli3_options & LPFC_SLI3_BG_ENABLED) &&
 898             !(iocbq->cmd_flag & DSS_SECURITY_OP)) {
 899                 if (num_bde > LPFC_EXT_DATA_BDE_COUNT) {
 900                         /*
 901                          * The extended IOCB format can only fit 3 BDE or a BPL.
 902                          * This I/O has more than 3 BDE so the 1st data bde will
 903                          * be a BPL that is filled in here.
 904                          */
 905                         physaddr = lpfc_cmd->dma_handle;
 906                         data_bde->tus.f.bdeFlags = BUFF_TYPE_BLP_64;
 907                         data_bde->tus.f.bdeSize = (num_bde *
 908                                                    sizeof(struct ulp_bde64));
 909                         physaddr += (sizeof(struct fcp_cmnd) +
 910                                      sizeof(struct fcp_rsp) +
 911                                      (2 * sizeof(struct ulp_bde64)));
 912                         data_bde->addrHigh = putPaddrHigh(physaddr);
 913                         data_bde->addrLow = putPaddrLow(physaddr);
 914                         /* ebde count includes the response bde and data bpl */
 915                         iocb_cmd->unsli3.fcp_ext.ebde_count = 2;
 916                 } else {
 917                         /* ebde count includes the response bde and data bdes */
 918                         iocb_cmd->unsli3.fcp_ext.ebde_count = (num_bde + 1);
 919                 }
 920         } else {
 921                 iocb_cmd->un.fcpi64.bdl.bdeSize =
 922                         ((num_bde + 2) * sizeof(struct ulp_bde64));
 923                 iocb_cmd->unsli3.fcp_ext.ebde_count = (num_bde + 1);
 924         }
 925         fcp_cmnd->fcpDl = cpu_to_be32(scsi_bufflen(scsi_cmnd));
 926
 927         /*
 928          * Due to difference in data length between DIF/non-DIF paths,
 929          * we need to set word 4 of IOCB here
 930          */
 931         iocb_cmd->un.fcpi.fcpi_parm = scsi_bufflen(scsi_cmnd);
 932         lpfc_fcpcmd_to_iocb(iocb_cmd->unsli3.fcp_ext.icd, fcp_cmnd);
 933         return 0;
 934 }
 935
 936 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 937
 938 /* Return BG_ERR_INIT if error injection is detected by Initiator */
 939 #define BG_ERR_INIT     0x1
 940 /* Return BG_ERR_TGT if error injection is detected by Target */
 941 #define BG_ERR_TGT      0x2
 942 /* Return BG_ERR_SWAP if swapping CSUM<-->CRC is required for error injection */
 943 #define BG_ERR_SWAP     0x10
 944 /*
 945  * Return BG_ERR_CHECK if disabling Guard/Ref/App checking is required for
 946  * error injection
 947  */
 948 #define BG_ERR_CHECK    0x20
 949
 950 /**
 951  * lpfc_bg_err_inject - Determine if we should inject an error
 952  * @phba: The Hba for which this call is being executed.
 953  * @sc: The SCSI command to examine
 954  * @reftag: (out) BlockGuard reference tag for transmitted data
 955  * @apptag: (out) BlockGuard application tag for transmitted data
 956  * @new_guard: (in) Value to replace CRC with if needed
 957  *
 958  * Returns BG_ERR_* bit mask or 0 if request ignored
 959  **/
 960 static int
 961 lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc,
 962                 uint32_t *reftag, uint16_t *apptag, uint32_t new_guard)
 963 {
 964         struct scatterlist *sgpe; /* s/g prot entry */
 965         struct lpfc_io_buf *lpfc_cmd = NULL;
 966         struct scsi_dif_tuple *src = NULL;
 967         struct lpfc_nodelist *ndlp;
 968         struct lpfc_rport_data *rdata;
 969         uint32_t op = scsi_get_prot_op(sc);
 970         uint32_t blksize;
 971         uint32_t numblks;
 972         u32 lba;
 973         int rc = 0;
 974         int blockoff = 0;
 975
 976         if (op == SCSI_PROT_NORMAL)
 977                 return 0;
 978
 979         sgpe = scsi_prot_sglist(sc);
 980         lba = scsi_prot_ref_tag(sc);
 981         if (lba == LPFC_INVALID_REFTAG)
 982                 return 0;
 983
 984         /* First check if we need to match the LBA */
 985         if (phba->lpfc_injerr_lba != LPFC_INJERR_LBA_OFF) {
 986                 blksize = scsi_prot_interval(sc);
 987                 numblks = (scsi_bufflen(sc) + blksize - 1) / blksize;
 988
 989                 /* Make sure we have the right LBA if one is specified */
 990                 if (phba->lpfc_injerr_lba < (u64)lba ||
 991                     (phba->lpfc_injerr_lba >= (u64)(lba + numblks)))
 992                         return 0;
 993                 if (sgpe) {
 994                         blockoff = phba->lpfc_injerr_lba - (u64)lba;
 995                         numblks = sg_dma_len(sgpe) /
 996                                 sizeof(struct scsi_dif_tuple);
 997                         if (numblks < blockoff)
 998                                 blockoff = numblks;
 999                 }
1000         }
1001
1002         /* Next check if we need to match the remote NPortID or WWPN */
1003         rdata = lpfc_rport_data_from_scsi_device(sc->device);
1004         if (rdata && rdata->pnode) {
1005                 ndlp = rdata->pnode;
1006
1007                 /* Make sure we have the right NPortID if one is specified */
1008                 if (phba->lpfc_injerr_nportid  &&
1009                         (phba->lpfc_injerr_nportid != ndlp->nlp_DID))
1010                         return 0;
1011
1012                 /*
1013                  * Make sure we have the right WWPN if one is specified.
1014                  * wwn[0] should be a non-zero NAA in a good WWPN.
1015                  */
1016                 if (phba->lpfc_injerr_wwpn.u.wwn[0]  &&
1017                         (memcmp(&ndlp->nlp_portname, &phba->lpfc_injerr_wwpn,
1018                                 sizeof(struct lpfc_name)) != 0))
1019                         return 0;
1020         }
1021
1022         /* Setup a ptr to the protection data if the SCSI host provides it */
1023         if (sgpe) {
1024                 src = (struct scsi_dif_tuple *)sg_virt(sgpe);
1025                 src += blockoff;
1026                 lpfc_cmd = (struct lpfc_io_buf *)sc->host_scribble;
1027         }
1028
1029         /* Should we change the Reference Tag */
1030         if (reftag) {
1031                 if (phba->lpfc_injerr_wref_cnt) {
1032                         switch (op) {
1033                         case SCSI_PROT_WRITE_PASS:
1034                                 if (src) {
1035                                         /*
1036                                          * For WRITE_PASS, force the error
1037                                          * to be sent on the wire. It should
1038                                          * be detected by the Target.
1039                                          * If blockoff != 0 error will be
1040                                          * inserted in middle of the IO.
1041                                          */
1042
1043                                         lpfc_printf_log(phba, KERN_ERR,
1044                                                         LOG_TRACE_EVENT,
1045                                         "9076 BLKGRD: Injecting reftag error: "
1046                                         "write lba x%lx + x%x oldrefTag x%x\n",
1047                                         (unsigned long)lba, blockoff,
1048                                         be32_to_cpu(src->ref_tag));
1049
1050                                         /*
1051                                          * Save the old ref_tag so we can
1052                                          * restore it on completion.
1053                                          */
1054                                         if (lpfc_cmd) {
1055                                                 lpfc_cmd->prot_data_type =
1056                                                         LPFC_INJERR_REFTAG;
1057                                                 lpfc_cmd->prot_data_segment =
1058                                                         src;
1059                                                 lpfc_cmd->prot_data =
1060                                                         src->ref_tag;
1061                                         }
1062                                         src->ref_tag = cpu_to_be32(0xDEADBEEF);
1063                                         phba->lpfc_injerr_wref_cnt--;
1064                                         if (phba->lpfc_injerr_wref_cnt == 0) {
1065                                                 phba->lpfc_injerr_nportid = 0;
1066                                                 phba->lpfc_injerr_lba =
1067                                                         LPFC_INJERR_LBA_OFF;
1068                                                 memset(&phba->lpfc_injerr_wwpn,
1069                                                   0, sizeof(struct lpfc_name));
1070                                         }
1071                                         rc = BG_ERR_TGT | BG_ERR_CHECK;
1072
1073                                         break;
1074                                 }
1075                                 fallthrough;
1076                         case SCSI_PROT_WRITE_INSERT:
1077                                 /*
1078                                  * For WRITE_INSERT, force the error
1079                                  * to be sent on the wire. It should be
1080                                  * detected by the Target.
1081                                  */
1082                                 /* DEADBEEF will be the reftag on the wire */
1083                                 *reftag = 0xDEADBEEF;
1084                                 phba->lpfc_injerr_wref_cnt--;
1085                                 if (phba->lpfc_injerr_wref_cnt == 0) {
1086                                         phba->lpfc_injerr_nportid = 0;
1087                                         phba->lpfc_injerr_lba =
1088                                         LPFC_INJERR_LBA_OFF;
1089                                         memset(&phba->lpfc_injerr_wwpn,
1090                                                 0, sizeof(struct lpfc_name));
1091                                 }
1092                                 rc = BG_ERR_TGT | BG_ERR_CHECK;
1093
1094                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1095                                         "9078 BLKGRD: Injecting reftag error: "
1096                                         "write lba x%lx\n", (unsigned long)lba);
1097                                 break;
1098                         case SCSI_PROT_WRITE_STRIP:
1099                                 /*
1100                                  * For WRITE_STRIP and WRITE_PASS,
1101                                  * force the error on data
1102                                  * being copied from SLI-Host to SLI-Port.
1103                                  */
1104                                 *reftag = 0xDEADBEEF;
1105                                 phba->lpfc_injerr_wref_cnt--;
1106                                 if (phba->lpfc_injerr_wref_cnt == 0) {
1107                                         phba->lpfc_injerr_nportid = 0;
1108                                         phba->lpfc_injerr_lba =
1109                                                 LPFC_INJERR_LBA_OFF;
1110                                         memset(&phba->lpfc_injerr_wwpn,
1111                                                 0, sizeof(struct lpfc_name));
1112                                 }
1113                                 rc = BG_ERR_INIT;
1114
1115                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1116                                         "9077 BLKGRD: Injecting reftag error: "
1117                                         "write lba x%lx\n", (unsigned long)lba);
1118                                 break;
1119                         }
1120                 }
1121                 if (phba->lpfc_injerr_rref_cnt) {
1122                         switch (op) {
1123                         case SCSI_PROT_READ_INSERT:
1124                         case SCSI_PROT_READ_STRIP:
1125                         case SCSI_PROT_READ_PASS:
1126                                 /*
1127                                  * For READ_STRIP and READ_PASS, force the
1128                                  * error on data being read off the wire. It
1129                                  * should force an IO error to the driver.
1130                                  */
1131                                 *reftag = 0xDEADBEEF;
1132                                 phba->lpfc_injerr_rref_cnt--;
1133                                 if (phba->lpfc_injerr_rref_cnt == 0) {
1134                                         phba->lpfc_injerr_nportid = 0;
1135                                         phba->lpfc_injerr_lba =
1136                                                 LPFC_INJERR_LBA_OFF;
1137                                         memset(&phba->lpfc_injerr_wwpn,
1138                                                 0, sizeof(struct lpfc_name));
1139                                 }
1140                                 rc = BG_ERR_INIT;
1141
1142                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1143                                         "9079 BLKGRD: Injecting reftag error: "
1144                                         "read lba x%lx\n", (unsigned long)lba);
1145                                 break;
1146                         }
1147                 }
1148         }
1149
1150         /* Should we change the Application Tag */
1151         if (apptag) {
1152                 if (phba->lpfc_injerr_wapp_cnt) {
1153                         switch (op) {
1154                         case SCSI_PROT_WRITE_PASS:
1155                                 if (src) {
1156                                         /*
1157                                          * For WRITE_PASS, force the error
1158                                          * to be sent on the wire. It should
1159                                          * be detected by the Target.
1160                                          * If blockoff != 0 error will be
1161                                          * inserted in middle of the IO.
1162                                          */
1163
1164                                         lpfc_printf_log(phba, KERN_ERR,
1165                                                         LOG_TRACE_EVENT,
1166                                         "9080 BLKGRD: Injecting apptag error: "
1167                                         "write lba x%lx + x%x oldappTag x%x\n",
1168                                         (unsigned long)lba, blockoff,
1169                                         be16_to_cpu(src->app_tag));
1170
1171                                         /*
1172                                          * Save the old app_tag so we can
1173                                          * restore it on completion.
1174                                          */
1175                                         if (lpfc_cmd) {
1176                                                 lpfc_cmd->prot_data_type =
1177                                                         LPFC_INJERR_APPTAG;
1178                                                 lpfc_cmd->prot_data_segment =
1179                                                         src;
1180                                                 lpfc_cmd->prot_data =
1181                                                         src->app_tag;
1182                                         }
1183                                         src->app_tag = cpu_to_be16(0xDEAD);
1184                                         phba->lpfc_injerr_wapp_cnt--;
1185                                         if (phba->lpfc_injerr_wapp_cnt == 0) {
1186                                                 phba->lpfc_injerr_nportid = 0;
1187                                                 phba->lpfc_injerr_lba =
1188                                                         LPFC_INJERR_LBA_OFF;
1189                                                 memset(&phba->lpfc_injerr_wwpn,
1190                                                   0, sizeof(struct lpfc_name));
1191                                         }
1192                                         rc = BG_ERR_TGT | BG_ERR_CHECK;
1193                                         break;
1194                                 }
1195                                 fallthrough;
1196                         case SCSI_PROT_WRITE_INSERT:
1197                                 /*
1198                                  * For WRITE_INSERT, force the
1199                                  * error to be sent on the wire. It should be
1200                                  * detected by the Target.
1201                                  */
1202                                 /* DEAD will be the apptag on the wire */
1203                                 *apptag = 0xDEAD;
1204                                 phba->lpfc_injerr_wapp_cnt--;
1205                                 if (phba->lpfc_injerr_wapp_cnt == 0) {
1206                                         phba->lpfc_injerr_nportid = 0;
1207                                         phba->lpfc_injerr_lba =
1208                                                 LPFC_INJERR_LBA_OFF;
1209                                         memset(&phba->lpfc_injerr_wwpn,
1210                                                 0, sizeof(struct lpfc_name));
1211                                 }
1212                                 rc = BG_ERR_TGT | BG_ERR_CHECK;
1213
1214                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1215                                         "0813 BLKGRD: Injecting apptag error: "
1216                                         "write lba x%lx\n", (unsigned long)lba);
1217                                 break;
1218                         case SCSI_PROT_WRITE_STRIP:
1219                                 /*
1220                                  * For WRITE_STRIP and WRITE_PASS,
1221                                  * force the error on data
1222                                  * being copied from SLI-Host to SLI-Port.
1223                                  */
1224                                 *apptag = 0xDEAD;
1225                                 phba->lpfc_injerr_wapp_cnt--;
1226                                 if (phba->lpfc_injerr_wapp_cnt == 0) {
1227                                         phba->lpfc_injerr_nportid = 0;
1228                                         phba->lpfc_injerr_lba =
1229                                                 LPFC_INJERR_LBA_OFF;
1230                                         memset(&phba->lpfc_injerr_wwpn,
1231                                                 0, sizeof(struct lpfc_name));
1232                                 }
1233                                 rc = BG_ERR_INIT;
1234
1235                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1236                                         "0812 BLKGRD: Injecting apptag error: "
1237                                         "write lba x%lx\n", (unsigned long)lba);
1238                                 break;
1239                         }
1240                 }
1241                 if (phba->lpfc_injerr_rapp_cnt) {
1242                         switch (op) {
1243                         case SCSI_PROT_READ_INSERT:
1244                         case SCSI_PROT_READ_STRIP:
1245                         case SCSI_PROT_READ_PASS:
1246                                 /*
1247                                  * For READ_STRIP and READ_PASS, force the
1248                                  * error on data being read off the wire. It
1249                                  * should force an IO error to the driver.
1250                                  */
1251                                 *apptag = 0xDEAD;
1252                                 phba->lpfc_injerr_rapp_cnt--;
1253                                 if (phba->lpfc_injerr_rapp_cnt == 0) {
1254                                         phba->lpfc_injerr_nportid = 0;
1255                                         phba->lpfc_injerr_lba =
1256                                                 LPFC_INJERR_LBA_OFF;
1257                                         memset(&phba->lpfc_injerr_wwpn,
1258                                                 0, sizeof(struct lpfc_name));
1259                                 }
1260                                 rc = BG_ERR_INIT;
1261
1262                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1263                                         "0814 BLKGRD: Injecting apptag error: "
1264                                         "read lba x%lx\n", (unsigned long)lba);
1265                                 break;
1266                         }
1267                 }
1268         }
1269
1270
1271         /* Should we change the Guard Tag */
1272         if (new_guard) {
1273                 if (phba->lpfc_injerr_wgrd_cnt) {
1274                         switch (op) {
1275                         case SCSI_PROT_WRITE_PASS:
1276                                 rc = BG_ERR_CHECK;
1277                                 fallthrough;
1278
1279                         case SCSI_PROT_WRITE_INSERT:
1280                                 /*
1281                                  * For WRITE_INSERT, force the
1282                                  * error to be sent on the wire. It should be
1283                                  * detected by the Target.
1284                                  */
1285                                 phba->lpfc_injerr_wgrd_cnt--;
1286                                 if (phba->lpfc_injerr_wgrd_cnt == 0) {
1287                                         phba->lpfc_injerr_nportid = 0;
1288                                         phba->lpfc_injerr_lba =
1289                                                 LPFC_INJERR_LBA_OFF;
1290                                         memset(&phba->lpfc_injerr_wwpn,
1291                                                 0, sizeof(struct lpfc_name));
1292                                 }
1293
1294                                 rc |= BG_ERR_TGT | BG_ERR_SWAP;
1295                                 /* Signals the caller to swap CRC->CSUM */
1296
1297                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1298                                         "0817 BLKGRD: Injecting guard error: "
1299                                         "write lba x%lx\n", (unsigned long)lba);
1300                                 break;
1301                         case SCSI_PROT_WRITE_STRIP:
1302                                 /*
1303                                  * For WRITE_STRIP and WRITE_PASS,
1304                                  * force the error on data
1305                                  * being copied from SLI-Host to SLI-Port.
1306                                  */
1307                                 phba->lpfc_injerr_wgrd_cnt--;
1308                                 if (phba->lpfc_injerr_wgrd_cnt == 0) {
1309                                         phba->lpfc_injerr_nportid = 0;
1310                                         phba->lpfc_injerr_lba =
1311                                                 LPFC_INJERR_LBA_OFF;
1312                                         memset(&phba->lpfc_injerr_wwpn,
1313                                                 0, sizeof(struct lpfc_name));
1314                                 }
1315
1316                                 rc = BG_ERR_INIT | BG_ERR_SWAP;
1317                                 /* Signals the caller to swap CRC->CSUM */
1318
1319                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1320                                         "0816 BLKGRD: Injecting guard error: "
1321                                         "write lba x%lx\n", (unsigned long)lba);
1322                                 break;
1323                         }
1324                 }
1325                 if (phba->lpfc_injerr_rgrd_cnt) {
1326                         switch (op) {
1327                         case SCSI_PROT_READ_INSERT:
1328                         case SCSI_PROT_READ_STRIP:
1329                         case SCSI_PROT_READ_PASS:
1330                                 /*
1331                                  * For READ_STRIP and READ_PASS, force the
1332                                  * error on data being read off the wire. It
1333                                  * should force an IO error to the driver.
1334                                  */
1335                                 phba->lpfc_injerr_rgrd_cnt--;
1336                                 if (phba->lpfc_injerr_rgrd_cnt == 0) {
1337                                         phba->lpfc_injerr_nportid = 0;
1338                                         phba->lpfc_injerr_lba =
1339                                                 LPFC_INJERR_LBA_OFF;
1340                                         memset(&phba->lpfc_injerr_wwpn,
1341                                                 0, sizeof(struct lpfc_name));
1342                                 }
1343
1344                                 rc = BG_ERR_INIT | BG_ERR_SWAP;
1345                                 /* Signals the caller to swap CRC->CSUM */
1346
1347                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1348                                         "0818 BLKGRD: Injecting guard error: "
1349                                         "read lba x%lx\n", (unsigned long)lba);
1350                         }
1351                 }
1352         }
1353
1354         return rc;
1355 }
1356 #endif
1357
1358 /**
1359  * lpfc_sc_to_bg_opcodes - Determine the BlockGuard opcodes to be used with
1360  * the specified SCSI command.
1361  * @phba: The Hba for which this call is being executed.
1362  * @sc: The SCSI command to examine
1363  * @txop: (out) BlockGuard operation for transmitted data
1364  * @rxop: (out) BlockGuard operation for received data
1365  *
1366  * Returns: zero on success; non-zero if tx and/or rx op cannot be determined
1367  *
1368  **/
1369 static int
1370 lpfc_sc_to_bg_opcodes(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1371                 uint8_t *txop, uint8_t *rxop)
1372 {
1373         uint8_t ret = 0;
1374
1375         if (sc->prot_flags & SCSI_PROT_IP_CHECKSUM) {
1376                 switch (scsi_get_prot_op(sc)) {
1377                 case SCSI_PROT_READ_INSERT:
1378                 case SCSI_PROT_WRITE_STRIP:
1379                         *rxop = BG_OP_IN_NODIF_OUT_CSUM;
1380                         *txop = BG_OP_IN_CSUM_OUT_NODIF;
1381                         break;
1382
1383                 case SCSI_PROT_READ_STRIP:
1384                 case SCSI_PROT_WRITE_INSERT:
1385                         *rxop = BG_OP_IN_CRC_OUT_NODIF;
1386                         *txop = BG_OP_IN_NODIF_OUT_CRC;
1387                         break;
1388
1389                 case SCSI_PROT_READ_PASS:
1390                 case SCSI_PROT_WRITE_PASS:
1391                         *rxop = BG_OP_IN_CRC_OUT_CSUM;
1392                         *txop = BG_OP_IN_CSUM_OUT_CRC;
1393                         break;
1394
1395                 case SCSI_PROT_NORMAL:
1396                 default:
1397                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1398                                 "9063 BLKGRD: Bad op/guard:%d/IP combination\n",
1399                                         scsi_get_prot_op(sc));
1400                         ret = 1;
1401                         break;
1402
1403                 }
1404         } else {
1405                 switch (scsi_get_prot_op(sc)) {
1406                 case SCSI_PROT_READ_STRIP:
1407                 case SCSI_PROT_WRITE_INSERT:
1408                         *rxop = BG_OP_IN_CRC_OUT_NODIF;
1409                         *txop = BG_OP_IN_NODIF_OUT_CRC;
1410                         break;
1411
1412                 case SCSI_PROT_READ_PASS:
1413                 case SCSI_PROT_WRITE_PASS:
1414                         *rxop = BG_OP_IN_CRC_OUT_CRC;
1415                         *txop = BG_OP_IN_CRC_OUT_CRC;
1416                         break;
1417
1418                 case SCSI_PROT_READ_INSERT:
1419                 case SCSI_PROT_WRITE_STRIP:
1420                         *rxop = BG_OP_IN_NODIF_OUT_CRC;
1421                         *txop = BG_OP_IN_CRC_OUT_NODIF;
1422                         break;
1423
1424                 case SCSI_PROT_NORMAL:
1425                 default:
1426                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1427                                 "9075 BLKGRD: Bad op/guard:%d/CRC combination\n",
1428                                         scsi_get_prot_op(sc));
1429                         ret = 1;
1430                         break;
1431                 }
1432         }
1433
1434         return ret;
1435 }
1436
1437 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1438 /**
1439  * lpfc_bg_err_opcodes - reDetermine the BlockGuard opcodes to be used with
1440  * the specified SCSI command in order to force a guard tag error.
1441  * @phba: The Hba for which this call is being executed.
1442  * @sc: The SCSI command to examine
1443  * @txop: (out) BlockGuard operation for transmitted data
1444  * @rxop: (out) BlockGuard operation for received data
1445  *
1446  * Returns: zero on success; non-zero if tx and/or rx op cannot be determined
1447  *
1448  **/
1449 static int
1450 lpfc_bg_err_opcodes(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1451                 uint8_t *txop, uint8_t *rxop)
1452 {
1453
1454         if (sc->prot_flags & SCSI_PROT_IP_CHECKSUM) {
1455                 switch (scsi_get_prot_op(sc)) {
1456                 case SCSI_PROT_READ_INSERT:
1457                 case SCSI_PROT_WRITE_STRIP:
1458                         *rxop = BG_OP_IN_NODIF_OUT_CRC;
1459                         *txop = BG_OP_IN_CRC_OUT_NODIF;
1460                         break;
1461
1462                 case SCSI_PROT_READ_STRIP:
1463                 case SCSI_PROT_WRITE_INSERT:
1464                         *rxop = BG_OP_IN_CSUM_OUT_NODIF;
1465                         *txop = BG_OP_IN_NODIF_OUT_CSUM;
1466                         break;
1467
1468                 case SCSI_PROT_READ_PASS:
1469                 case SCSI_PROT_WRITE_PASS:
1470                         *rxop = BG_OP_IN_CSUM_OUT_CRC;
1471                         *txop = BG_OP_IN_CRC_OUT_CSUM;
1472                         break;
1473
1474                 case SCSI_PROT_NORMAL:
1475                 default:
1476                         break;
1477
1478                 }
1479         } else {
1480                 switch (scsi_get_prot_op(sc)) {
1481                 case SCSI_PROT_READ_STRIP:
1482                 case SCSI_PROT_WRITE_INSERT:
1483                         *rxop = BG_OP_IN_CSUM_OUT_NODIF;
1484                         *txop = BG_OP_IN_NODIF_OUT_CSUM;
1485                         break;
1486
1487                 case SCSI_PROT_READ_PASS:
1488                 case SCSI_PROT_WRITE_PASS:
1489                         *rxop = BG_OP_IN_CSUM_OUT_CSUM;
1490                         *txop = BG_OP_IN_CSUM_OUT_CSUM;
1491                         break;
1492
1493                 case SCSI_PROT_READ_INSERT:
1494                 case SCSI_PROT_WRITE_STRIP:
1495                         *rxop = BG_OP_IN_NODIF_OUT_CSUM;
1496                         *txop = BG_OP_IN_CSUM_OUT_NODIF;
1497                         break;
1498
1499                 case SCSI_PROT_NORMAL:
1500                 default:
1501                         break;
1502                 }
1503         }
1504
1505         return 0;
1506 }
1507 #endif
1508
1509 /**
1510  * lpfc_bg_setup_bpl - Setup BlockGuard BPL with no protection data
1511  * @phba: The Hba for which this call is being executed.
1512  * @sc: pointer to scsi command we're working on
1513  * @bpl: pointer to buffer list for protection groups
1514  * @datasegcnt: number of segments of data that have been dma mapped
1515  *
1516  * This function sets up BPL buffer list for protection groups of
1517  * type LPFC_PG_TYPE_NO_DIF
1518  *
1519  * This is usually used when the HBA is instructed to generate
1520  * DIFs and insert them into data stream (or strip DIF from
1521  * incoming data stream)
1522  *
1523  * The buffer list consists of just one protection group described
1524  * below:
1525  *                                +-------------------------+
1526  *   start of prot group  -->     |          PDE_5          |
1527  *                                +-------------------------+
1528  *                                |          PDE_6          |
1529  *                                +-------------------------+
1530  *                                |         Data BDE        |
1531  *                                +-------------------------+
1532  *                                |more Data BDE's ... (opt)|
1533  *                                +-------------------------+
1534  *
1535  *
1536  * Note: Data s/g buffers have been dma mapped
1537  *
1538  * Returns the number of BDEs added to the BPL.
1539  **/
1540 static int
1541 lpfc_bg_setup_bpl(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1542                 struct ulp_bde64 *bpl, int datasegcnt)
1543 {
1544         struct scatterlist *sgde = NULL; /* s/g data entry */
1545         struct lpfc_pde5 *pde5 = NULL;
1546         struct lpfc_pde6 *pde6 = NULL;
1547         dma_addr_t physaddr;
1548         int i = 0, num_bde = 0, status;
1549         int datadir = sc->sc_data_direction;
1550 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1551         uint32_t rc;
1552 #endif
1553         uint32_t checking = 1;
1554         uint32_t reftag;
1555         uint8_t txop, rxop;
1556
1557         status  = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
1558         if (status)
1559                 goto out;
1560
1561         /* extract some info from the scsi command for pde*/
1562         reftag = scsi_prot_ref_tag(sc);
1563         if (reftag == LPFC_INVALID_REFTAG)
1564                 goto out;
1565
1566 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1567         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
1568         if (rc) {
1569                 if (rc & BG_ERR_SWAP)
1570                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
1571                 if (rc & BG_ERR_CHECK)
1572                         checking = 0;
1573         }
1574 #endif
1575
1576         /* setup PDE5 with what we have */
1577         pde5 = (struct lpfc_pde5 *) bpl;
1578         memset(pde5, 0, sizeof(struct lpfc_pde5));
1579         bf_set(pde5_type, pde5, LPFC_PDE5_DESCRIPTOR);
1580
1581         /* Endianness conversion if necessary for PDE5 */
1582         pde5->word0 = cpu_to_le32(pde5->word0);
1583         pde5->reftag = cpu_to_le32(reftag);
1584
1585         /* advance bpl and increment bde count */
1586         num_bde++;
1587         bpl++;
1588         pde6 = (struct lpfc_pde6 *) bpl;
1589
1590         /* setup PDE6 with the rest of the info */
1591         memset(pde6, 0, sizeof(struct lpfc_pde6));
1592         bf_set(pde6_type, pde6, LPFC_PDE6_DESCRIPTOR);
1593         bf_set(pde6_optx, pde6, txop);
1594         bf_set(pde6_oprx, pde6, rxop);
1595
1596         /*
1597          * We only need to check the data on READs, for WRITEs
1598          * protection data is automatically generated, not checked.
1599          */
1600         if (datadir == DMA_FROM_DEVICE) {
1601                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
1602                         bf_set(pde6_ce, pde6, checking);
1603                 else
1604                         bf_set(pde6_ce, pde6, 0);
1605
1606                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
1607                         bf_set(pde6_re, pde6, checking);
1608                 else
1609                         bf_set(pde6_re, pde6, 0);
1610         }
1611         bf_set(pde6_ai, pde6, 1);
1612         bf_set(pde6_ae, pde6, 0);
1613         bf_set(pde6_apptagval, pde6, 0);
1614
1615         /* Endianness conversion if necessary for PDE6 */
1616         pde6->word0 = cpu_to_le32(pde6->word0);
1617         pde6->word1 = cpu_to_le32(pde6->word1);
1618         pde6->word2 = cpu_to_le32(pde6->word2);
1619
1620         /* advance bpl and increment bde count */
1621         num_bde++;
1622         bpl++;
1623
1624         /* assumption: caller has already run dma_map_sg on command data */
1625         scsi_for_each_sg(sc, sgde, datasegcnt, i) {
1626                 physaddr = sg_dma_address(sgde);
1627                 bpl->addrLow = le32_to_cpu(putPaddrLow(physaddr));
1628                 bpl->addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
1629                 bpl->tus.f.bdeSize = sg_dma_len(sgde);
1630                 if (datadir == DMA_TO_DEVICE)
1631                         bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1632                 else
1633                         bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
1634                 bpl->tus.w = le32_to_cpu(bpl->tus.w);
1635                 bpl++;
1636                 num_bde++;
1637         }
1638
1639 out:
1640         return num_bde;
1641 }
1642
1643 /**
1644  * lpfc_bg_setup_bpl_prot - Setup BlockGuard BPL with protection data
1645  * @phba: The Hba for which this call is being executed.
1646  * @sc: pointer to scsi command we're working on
1647  * @bpl: pointer to buffer list for protection groups
1648  * @datacnt: number of segments of data that have been dma mapped
1649  * @protcnt: number of segment of protection data that have been dma mapped
1650  *
1651  * This function sets up BPL buffer list for protection groups of
1652  * type LPFC_PG_TYPE_DIF
1653  *
1654  * This is usually used when DIFs are in their own buffers,
1655  * separate from the data. The HBA can then by instructed
1656  * to place the DIFs in the outgoing stream.  For read operations,
1657  * The HBA could extract the DIFs and place it in DIF buffers.
1658  *
1659  * The buffer list for this type consists of one or more of the
1660  * protection groups described below:
1661  *                                    +-------------------------+
1662  *   start of first prot group  -->   |          PDE_5          |
1663  *                                    +-------------------------+
1664  *                                    |          PDE_6          |
1665  *                                    +-------------------------+
1666  *                                    |      PDE_7 (Prot BDE)   |
1667  *                                    +-------------------------+
1668  *                                    |        Data BDE         |
1669  *                                    +-------------------------+
1670  *                                    |more Data BDE's ... (opt)|
1671  *                                    +-------------------------+
1672  *   start of new  prot group  -->    |          PDE_5          |
1673  *                                    +-------------------------+
1674  *                                    |          ...            |
1675  *                                    +-------------------------+
1676  *
1677  * Note: It is assumed that both data and protection s/g buffers have been
1678  *       mapped for DMA
1679  *
1680  * Returns the number of BDEs added to the BPL.
1681  **/
1682 static int
1683 lpfc_bg_setup_bpl_prot(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1684                 struct ulp_bde64 *bpl, int datacnt, int protcnt)
1685 {
1686         struct scatterlist *sgde = NULL; /* s/g data entry */
1687         struct scatterlist *sgpe = NULL; /* s/g prot entry */
1688         struct lpfc_pde5 *pde5 = NULL;
1689         struct lpfc_pde6 *pde6 = NULL;
1690         struct lpfc_pde7 *pde7 = NULL;
1691         dma_addr_t dataphysaddr, protphysaddr;
1692         unsigned short curr_prot = 0;
1693         unsigned int split_offset;
1694         unsigned int protgroup_len, protgroup_offset = 0, protgroup_remainder;
1695         unsigned int protgrp_blks, protgrp_bytes;
1696         unsigned int remainder, subtotal;
1697         int status;
1698         int datadir = sc->sc_data_direction;
1699         unsigned char pgdone = 0, alldone = 0;
1700         unsigned blksize;
1701 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1702         uint32_t rc;
1703 #endif
1704         uint32_t checking = 1;
1705         uint32_t reftag;
1706         uint8_t txop, rxop;
1707         int num_bde = 0;
1708
1709         sgpe = scsi_prot_sglist(sc);
1710         sgde = scsi_sglist(sc);
1711
1712         if (!sgpe || !sgde) {
1713                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1714                                 "9020 Invalid s/g entry: data=x%px prot=x%px\n",
1715                                 sgpe, sgde);
1716                 return 0;
1717         }
1718
1719         status = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
1720         if (status)
1721                 goto out;
1722
1723         /* extract some info from the scsi command */
1724         blksize = scsi_prot_interval(sc);
1725         reftag = scsi_prot_ref_tag(sc);
1726         if (reftag == LPFC_INVALID_REFTAG)
1727                 goto out;
1728
1729 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1730         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
1731         if (rc) {
1732                 if (rc & BG_ERR_SWAP)
1733                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
1734                 if (rc & BG_ERR_CHECK)
1735                         checking = 0;
1736         }
1737 #endif
1738
1739         split_offset = 0;
1740         do {
1741                 /* Check to see if we ran out of space */
1742                 if (num_bde >= (phba->cfg_total_seg_cnt - 2))
1743                         return num_bde + 3;
1744
1745                 /* setup PDE5 with what we have */
1746                 pde5 = (struct lpfc_pde5 *) bpl;
1747                 memset(pde5, 0, sizeof(struct lpfc_pde5));
1748                 bf_set(pde5_type, pde5, LPFC_PDE5_DESCRIPTOR);
1749
1750                 /* Endianness conversion if necessary for PDE5 */
1751                 pde5->word0 = cpu_to_le32(pde5->word0);
1752                 pde5->reftag = cpu_to_le32(reftag);
1753
1754                 /* advance bpl and increment bde count */
1755                 num_bde++;
1756                 bpl++;
1757                 pde6 = (struct lpfc_pde6 *) bpl;
1758
1759                 /* setup PDE6 with the rest of the info */
1760                 memset(pde6, 0, sizeof(struct lpfc_pde6));
1761                 bf_set(pde6_type, pde6, LPFC_PDE6_DESCRIPTOR);
1762                 bf_set(pde6_optx, pde6, txop);
1763                 bf_set(pde6_oprx, pde6, rxop);
1764
1765                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
1766                         bf_set(pde6_ce, pde6, checking);
1767                 else
1768                         bf_set(pde6_ce, pde6, 0);
1769
1770                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
1771                         bf_set(pde6_re, pde6, checking);
1772                 else
1773                         bf_set(pde6_re, pde6, 0);
1774
1775                 bf_set(pde6_ai, pde6, 1);
1776                 bf_set(pde6_ae, pde6, 0);
1777                 bf_set(pde6_apptagval, pde6, 0);
1778
1779                 /* Endianness conversion if necessary for PDE6 */
1780                 pde6->word0 = cpu_to_le32(pde6->word0);
1781                 pde6->word1 = cpu_to_le32(pde6->word1);
1782                 pde6->word2 = cpu_to_le32(pde6->word2);
1783
1784                 /* advance bpl and increment bde count */
1785                 num_bde++;
1786                 bpl++;
1787
1788                 /* setup the first BDE that points to protection buffer */
1789                 protphysaddr = sg_dma_address(sgpe) + protgroup_offset;
1790                 protgroup_len = sg_dma_len(sgpe) - protgroup_offset;
1791
1792                 /* must be integer multiple of the DIF block length */
1793                 BUG_ON(protgroup_len % 8);
1794
1795                 pde7 = (struct lpfc_pde7 *) bpl;
1796                 memset(pde7, 0, sizeof(struct lpfc_pde7));
1797                 bf_set(pde7_type, pde7, LPFC_PDE7_DESCRIPTOR);
1798
1799                 pde7->addrHigh = le32_to_cpu(putPaddrHigh(protphysaddr));
1800                 pde7->addrLow = le32_to_cpu(putPaddrLow(protphysaddr));
1801
1802                 protgrp_blks = protgroup_len / 8;
1803                 protgrp_bytes = protgrp_blks * blksize;
1804
1805                 /* check if this pde is crossing the 4K boundary; if so split */
1806                 if ((pde7->addrLow & 0xfff) + protgroup_len > 0x1000) {
1807                         protgroup_remainder = 0x1000 - (pde7->addrLow & 0xfff);
1808                         protgroup_offset += protgroup_remainder;
1809                         protgrp_blks = protgroup_remainder / 8;
1810                         protgrp_bytes = protgrp_blks * blksize;
1811                 } else {
1812                         protgroup_offset = 0;
1813                         curr_prot++;
1814                 }
1815
1816                 num_bde++;
1817
1818                 /* setup BDE's for data blocks associated with DIF data */
1819                 pgdone = 0;
1820                 subtotal = 0; /* total bytes processed for current prot grp */
1821                 while (!pgdone) {
1822                         /* Check to see if we ran out of space */
1823                         if (num_bde >= phba->cfg_total_seg_cnt)
1824                                 return num_bde + 1;
1825
1826                         if (!sgde) {
1827                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1828                                         "9065 BLKGRD:%s Invalid data segment\n",
1829                                                 __func__);
1830                                 return 0;
1831                         }
1832                         bpl++;
1833                         dataphysaddr = sg_dma_address(sgde) + split_offset;
1834                         bpl->addrLow = le32_to_cpu(putPaddrLow(dataphysaddr));
1835                         bpl->addrHigh = le32_to_cpu(putPaddrHigh(dataphysaddr));
1836
1837                         remainder = sg_dma_len(sgde) - split_offset;
1838
1839                         if ((subtotal + remainder) <= protgrp_bytes) {
1840                                 /* we can use this whole buffer */
1841                                 bpl->tus.f.bdeSize = remainder;
1842                                 split_offset = 0;
1843
1844                                 if ((subtotal + remainder) == protgrp_bytes)
1845                                         pgdone = 1;
1846                         } else {
1847                                 /* must split this buffer with next prot grp */
1848                                 bpl->tus.f.bdeSize = protgrp_bytes - subtotal;
1849                                 split_offset += bpl->tus.f.bdeSize;
1850                         }
1851
1852                         subtotal += bpl->tus.f.bdeSize;
1853
1854                         if (datadir == DMA_TO_DEVICE)
1855                                 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1856                         else
1857                                 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
1858                         bpl->tus.w = le32_to_cpu(bpl->tus.w);
1859
1860                         num_bde++;
1861
1862                         if (split_offset)
1863                                 break;
1864
1865                         /* Move to the next s/g segment if possible */
1866                         sgde = sg_next(sgde);
1867
1868                 }
1869
1870                 if (protgroup_offset) {
1871                         /* update the reference tag */
1872                         reftag += protgrp_blks;
1873                         bpl++;
1874                         continue;
1875                 }
1876
1877                 /* are we done ? */
1878                 if (curr_prot == protcnt) {
1879                         alldone = 1;
1880                 } else if (curr_prot < protcnt) {
1881                         /* advance to next prot buffer */
1882                         sgpe = sg_next(sgpe);
1883                         bpl++;
1884
1885                         /* update the reference tag */
1886                         reftag += protgrp_blks;
1887                 } else {
1888                         /* if we're here, we have a bug */
1889                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1890                                         "9054 BLKGRD: bug in %s\n", __func__);
1891                 }
1892
1893         } while (!alldone);
1894 out:
1895
1896         return num_bde;
1897 }
1898
1899 /**
1900  * lpfc_bg_setup_sgl - Setup BlockGuard SGL with no protection data
1901  * @phba: The Hba for which this call is being executed.
1902  * @sc: pointer to scsi command we're working on
1903  * @sgl: pointer to buffer list for protection groups
1904  * @datasegcnt: number of segments of data that have been dma mapped
1905  * @lpfc_cmd: lpfc scsi command object pointer.
1906  *
1907  * This function sets up SGL buffer list for protection groups of
1908  * type LPFC_PG_TYPE_NO_DIF
1909  *
1910  * This is usually used when the HBA is instructed to generate
1911  * DIFs and insert them into data stream (or strip DIF from
1912  * incoming data stream)
1913  *
1914  * The buffer list consists of just one protection group described
1915  * below:
1916  *                                +-------------------------+
1917  *   start of prot group  -->     |         DI_SEED         |
1918  *                                +-------------------------+
1919  *                                |         Data SGE        |
1920  *                                +-------------------------+
1921  *                                |more Data SGE's ... (opt)|
1922  *                                +-------------------------+
1923  *
1924  *
1925  * Note: Data s/g buffers have been dma mapped
1926  *
1927  * Returns the number of SGEs added to the SGL.
1928  **/
1929 static int
1930 lpfc_bg_setup_sgl(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1931                 struct sli4_sge *sgl, int datasegcnt,
1932                 struct lpfc_io_buf *lpfc_cmd)
1933 {
1934         struct scatterlist *sgde = NULL; /* s/g data entry */
1935         struct sli4_sge_diseed *diseed = NULL;
1936         dma_addr_t physaddr;
1937         int i = 0, num_sge = 0, status;
1938         uint32_t reftag;
1939         uint8_t txop, rxop;
1940 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1941         uint32_t rc;
1942 #endif
1943         uint32_t checking = 1;
1944         uint32_t dma_len;
1945         uint32_t dma_offset = 0;
1946         struct sli4_hybrid_sgl *sgl_xtra = NULL;
1947         int j;
1948         bool lsp_just_set = false;
1949
1950         status  = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
1951         if (status)
1952                 goto out;
1953
1954         /* extract some info from the scsi command for pde*/
1955         reftag = scsi_prot_ref_tag(sc);
1956         if (reftag == LPFC_INVALID_REFTAG)
1957                 goto out;
1958
1959 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1960         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
1961         if (rc) {
1962                 if (rc & BG_ERR_SWAP)
1963                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
1964                 if (rc & BG_ERR_CHECK)
1965                         checking = 0;
1966         }
1967 #endif
1968
1969         /* setup DISEED with what we have */
1970         diseed = (struct sli4_sge_diseed *) sgl;
1971         memset(diseed, 0, sizeof(struct sli4_sge_diseed));
1972         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DISEED);
1973
1974         /* Endianness conversion if necessary */
1975         diseed->ref_tag = cpu_to_le32(reftag);
1976         diseed->ref_tag_tran = diseed->ref_tag;
1977
1978         /*
1979          * We only need to check the data on READs, for WRITEs
1980          * protection data is automatically generated, not checked.
1981          */
1982         if (sc->sc_data_direction == DMA_FROM_DEVICE) {
1983                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
1984                         bf_set(lpfc_sli4_sge_dif_ce, diseed, checking);
1985                 else
1986                         bf_set(lpfc_sli4_sge_dif_ce, diseed, 0);
1987
1988                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
1989                         bf_set(lpfc_sli4_sge_dif_re, diseed, checking);
1990                 else
1991                         bf_set(lpfc_sli4_sge_dif_re, diseed, 0);
1992         }
1993
1994         /* setup DISEED with the rest of the info */
1995         bf_set(lpfc_sli4_sge_dif_optx, diseed, txop);
1996         bf_set(lpfc_sli4_sge_dif_oprx, diseed, rxop);
1997
1998         bf_set(lpfc_sli4_sge_dif_ai, diseed, 1);
1999         bf_set(lpfc_sli4_sge_dif_me, diseed, 0);
2000
2001         /* Endianness conversion if necessary for DISEED */
2002         diseed->word2 = cpu_to_le32(diseed->word2);
2003         diseed->word3 = cpu_to_le32(diseed->word3);
2004
2005         /* advance bpl and increment sge count */
2006         num_sge++;
2007         sgl++;
2008
2009         /* assumption: caller has already run dma_map_sg on command data */
2010         sgde = scsi_sglist(sc);
2011         j = 3;
2012         for (i = 0; i < datasegcnt; i++) {
2013                 /* clear it */
2014                 sgl->word2 = 0;
2015
2016                 /* do we need to expand the segment */
2017                 if (!lsp_just_set && !((j + 1) % phba->border_sge_num) &&
2018                     ((datasegcnt - 1) != i)) {
2019                         /* set LSP type */
2020                         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_LSP);
2021
2022                         sgl_xtra = lpfc_get_sgl_per_hdwq(phba, lpfc_cmd);
2023
2024                         if (unlikely(!sgl_xtra)) {
2025                                 lpfc_cmd->seg_cnt = 0;
2026                                 return 0;
2027                         }
2028                         sgl->addr_lo = cpu_to_le32(putPaddrLow(
2029                                                 sgl_xtra->dma_phys_sgl));
2030                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(
2031                                                 sgl_xtra->dma_phys_sgl));
2032
2033                 } else {
2034                         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
2035                 }
2036
2037                 if (!(bf_get(lpfc_sli4_sge_type, sgl) & LPFC_SGE_TYPE_LSP)) {
2038                         if ((datasegcnt - 1) == i)
2039                                 bf_set(lpfc_sli4_sge_last, sgl, 1);
2040                         physaddr = sg_dma_address(sgde);
2041                         dma_len = sg_dma_len(sgde);
2042                         sgl->addr_lo = cpu_to_le32(putPaddrLow(physaddr));
2043                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(physaddr));
2044
2045                         bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
2046                         sgl->word2 = cpu_to_le32(sgl->word2);
2047                         sgl->sge_len = cpu_to_le32(dma_len);
2048
2049                         dma_offset += dma_len;
2050                         sgde = sg_next(sgde);
2051
2052                         sgl++;
2053                         num_sge++;
2054                         lsp_just_set = false;
2055
2056                 } else {
2057                         sgl->word2 = cpu_to_le32(sgl->word2);
2058                         sgl->sge_len = cpu_to_le32(phba->cfg_sg_dma_buf_size);
2059
2060                         sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
2061                         i = i - 1;
2062
2063                         lsp_just_set = true;
2064                 }
2065
2066                 j++;
2067
2068         }
2069
2070 out:
2071         return num_sge;
2072 }
2073
2074 /**
2075  * lpfc_bg_setup_sgl_prot - Setup BlockGuard SGL with protection data
2076  * @phba: The Hba for which this call is being executed.
2077  * @sc: pointer to scsi command we're working on
2078  * @sgl: pointer to buffer list for protection groups
2079  * @datacnt: number of segments of data that have been dma mapped
2080  * @protcnt: number of segment of protection data that have been dma mapped
2081  * @lpfc_cmd: lpfc scsi command object pointer.
2082  *
2083  * This function sets up SGL buffer list for protection groups of
2084  * type LPFC_PG_TYPE_DIF
2085  *
2086  * This is usually used when DIFs are in their own buffers,
2087  * separate from the data. The HBA can then by instructed
2088  * to place the DIFs in the outgoing stream.  For read operations,
2089  * The HBA could extract the DIFs and place it in DIF buffers.
2090  *
2091  * The buffer list for this type consists of one or more of the
2092  * protection groups described below:
2093  *                                    +-------------------------+
2094  *   start of first prot group  -->   |         DISEED          |
2095  *                                    +-------------------------+
2096  *                                    |      DIF (Prot SGE)     |
2097  *                                    +-------------------------+
2098  *                                    |        Data SGE         |
2099  *                                    +-------------------------+
2100  *                                    |more Data SGE's ... (opt)|
2101  *                                    +-------------------------+
2102  *   start of new  prot group  -->    |         DISEED          |
2103  *                                    +-------------------------+
2104  *                                    |          ...            |
2105  *                                    +-------------------------+
2106  *
2107  * Note: It is assumed that both data and protection s/g buffers have been
2108  *       mapped for DMA
2109  *
2110  * Returns the number of SGEs added to the SGL.
2111  **/
2112 static int
2113 lpfc_bg_setup_sgl_prot(struct lpfc_hba *phba, struct scsi_cmnd *sc,
2114                 struct sli4_sge *sgl, int datacnt, int protcnt,
2115                 struct lpfc_io_buf *lpfc_cmd)
2116 {
2117         struct scatterlist *sgde = NULL; /* s/g data entry */
2118         struct scatterlist *sgpe = NULL; /* s/g prot entry */
2119         struct sli4_sge_diseed *diseed = NULL;
2120         dma_addr_t dataphysaddr, protphysaddr;
2121         unsigned short curr_prot = 0;
2122         unsigned int split_offset;
2123         unsigned int protgroup_len, protgroup_offset = 0, protgroup_remainder;
2124         unsigned int protgrp_blks, protgrp_bytes;
2125         unsigned int remainder, subtotal;
2126         int status;
2127         unsigned char pgdone = 0, alldone = 0;
2128         unsigned blksize;
2129         uint32_t reftag;
2130         uint8_t txop, rxop;
2131         uint32_t dma_len;
2132 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2133         uint32_t rc;
2134 #endif
2135         uint32_t checking = 1;
2136         uint32_t dma_offset = 0;
2137         int num_sge = 0, j = 2;
2138         struct sli4_hybrid_sgl *sgl_xtra = NULL;
2139
2140         sgpe = scsi_prot_sglist(sc);
2141         sgde = scsi_sglist(sc);
2142
2143         if (!sgpe || !sgde) {
2144                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2145                                 "9082 Invalid s/g entry: data=x%px prot=x%px\n",
2146                                 sgpe, sgde);
2147                 return 0;
2148         }
2149
2150         status = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
2151         if (status)
2152                 goto out;
2153
2154         /* extract some info from the scsi command */
2155         blksize = scsi_prot_interval(sc);
2156         reftag = scsi_prot_ref_tag(sc);
2157         if (reftag == LPFC_INVALID_REFTAG)
2158                 goto out;
2159
2160 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2161         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
2162         if (rc) {
2163                 if (rc & BG_ERR_SWAP)
2164                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
2165                 if (rc & BG_ERR_CHECK)
2166                         checking = 0;
2167         }
2168 #endif
2169
2170         split_offset = 0;
2171         do {
2172                 /* Check to see if we ran out of space */
2173                 if ((num_sge >= (phba->cfg_total_seg_cnt - 2)) &&
2174                     !(phba->cfg_xpsgl))
2175                         return num_sge + 3;
2176
2177                 /* DISEED and DIF have to be together */
2178                 if (!((j + 1) % phba->border_sge_num) ||
2179                     !((j + 2) % phba->border_sge_num) ||
2180                     !((j + 3) % phba->border_sge_num)) {
2181                         sgl->word2 = 0;
2182
2183                         /* set LSP type */
2184                         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_LSP);
2185
2186                         sgl_xtra = lpfc_get_sgl_per_hdwq(phba, lpfc_cmd);
2187
2188                         if (unlikely(!sgl_xtra)) {
2189                                 goto out;
2190                         } else {
2191                                 sgl->addr_lo = cpu_to_le32(putPaddrLow(
2192                                                 sgl_xtra->dma_phys_sgl));
2193                                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
2194                                                        sgl_xtra->dma_phys_sgl));
2195                         }
2196
2197                         sgl->word2 = cpu_to_le32(sgl->word2);
2198                         sgl->sge_len = cpu_to_le32(phba->cfg_sg_dma_buf_size);
2199
2200                         sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
2201                         j = 0;
2202                 }
2203
2204                 /* setup DISEED with what we have */
2205                 diseed = (struct sli4_sge_diseed *) sgl;
2206                 memset(diseed, 0, sizeof(struct sli4_sge_diseed));
2207                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DISEED);
2208
2209                 /* Endianness conversion if necessary */
2210                 diseed->ref_tag = cpu_to_le32(reftag);
2211                 diseed->ref_tag_tran = diseed->ref_tag;
2212
2213                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) {
2214                         bf_set(lpfc_sli4_sge_dif_ce, diseed, checking);
2215                 } else {
2216                         bf_set(lpfc_sli4_sge_dif_ce, diseed, 0);
2217                         /*
2218                          * When in this mode, the hardware will replace
2219                          * the guard tag from the host with a
2220                          * newly generated good CRC for the wire.
2221                          * Switch to raw mode here to avoid this
2222                          * behavior. What the host sends gets put on the wire.
2223                          */
2224                         if (txop == BG_OP_IN_CRC_OUT_CRC) {
2225                                 txop = BG_OP_RAW_MODE;
2226                                 rxop = BG_OP_RAW_MODE;
2227                         }
2228                 }
2229
2230
2231                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
2232                         bf_set(lpfc_sli4_sge_dif_re, diseed, checking);
2233                 else
2234                         bf_set(lpfc_sli4_sge_dif_re, diseed, 0);
2235
2236                 /* setup DISEED with the rest of the info */
2237                 bf_set(lpfc_sli4_sge_dif_optx, diseed, txop);
2238                 bf_set(lpfc_sli4_sge_dif_oprx, diseed, rxop);
2239
2240                 bf_set(lpfc_sli4_sge_dif_ai, diseed, 1);
2241                 bf_set(lpfc_sli4_sge_dif_me, diseed, 0);
2242
2243                 /* Endianness conversion if necessary for DISEED */
2244                 diseed->word2 = cpu_to_le32(diseed->word2);
2245                 diseed->word3 = cpu_to_le32(diseed->word3);
2246
2247                 /* advance sgl and increment bde count */
2248                 num_sge++;
2249
2250                 sgl++;
2251                 j++;
2252
2253                 /* setup the first BDE that points to protection buffer */
2254                 protphysaddr = sg_dma_address(sgpe) + protgroup_offset;
2255                 protgroup_len = sg_dma_len(sgpe) - protgroup_offset;
2256
2257                 /* must be integer multiple of the DIF block length */
2258                 BUG_ON(protgroup_len % 8);
2259
2260                 /* Now setup DIF SGE */
2261                 sgl->word2 = 0;
2262                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DIF);
2263                 sgl->addr_hi = le32_to_cpu(putPaddrHigh(protphysaddr));
2264                 sgl->addr_lo = le32_to_cpu(putPaddrLow(protphysaddr));
2265                 sgl->word2 = cpu_to_le32(sgl->word2);
2266                 sgl->sge_len = 0;
2267
2268                 protgrp_blks = protgroup_len / 8;
2269                 protgrp_bytes = protgrp_blks * blksize;
2270
2271                 /* check if DIF SGE is crossing the 4K boundary; if so split */
2272                 if ((sgl->addr_lo & 0xfff) + protgroup_len > 0x1000) {
2273                         protgroup_remainder = 0x1000 - (sgl->addr_lo & 0xfff);
2274                         protgroup_offset += protgroup_remainder;
2275                         protgrp_blks = protgroup_remainder / 8;
2276                         protgrp_bytes = protgrp_blks * blksize;
2277                 } else {
2278                         protgroup_offset = 0;
2279                         curr_prot++;
2280                 }
2281
2282                 num_sge++;
2283
2284                 /* setup SGE's for data blocks associated with DIF data */
2285                 pgdone = 0;
2286                 subtotal = 0; /* total bytes processed for current prot grp */
2287
2288                 sgl++;
2289                 j++;
2290
2291                 while (!pgdone) {
2292                         /* Check to see if we ran out of space */
2293                         if ((num_sge >= phba->cfg_total_seg_cnt) &&
2294                             !phba->cfg_xpsgl)
2295                                 return num_sge + 1;
2296
2297                         if (!sgde) {
2298                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2299                                         "9086 BLKGRD:%s Invalid data segment\n",
2300                                                 __func__);
2301                                 return 0;
2302                         }
2303
2304                         if (!((j + 1) % phba->border_sge_num)) {
2305                                 sgl->word2 = 0;
2306
2307                                 /* set LSP type */
2308                                 bf_set(lpfc_sli4_sge_type, sgl,
2309                                        LPFC_SGE_TYPE_LSP);
2310
2311                                 sgl_xtra = lpfc_get_sgl_per_hdwq(phba,
2312                                                                  lpfc_cmd);
2313
2314                                 if (unlikely(!sgl_xtra)) {
2315                                         goto out;
2316                                 } else {
2317                                         sgl->addr_lo = cpu_to_le32(
2318                                           putPaddrLow(sgl_xtra->dma_phys_sgl));
2319                                         sgl->addr_hi = cpu_to_le32(
2320                                           putPaddrHigh(sgl_xtra->dma_phys_sgl));
2321                                 }
2322
2323                                 sgl->word2 = cpu_to_le32(sgl->word2);
2324                                 sgl->sge_len = cpu_to_le32(
2325                                                      phba->cfg_sg_dma_buf_size);
2326
2327                                 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
2328                         } else {
2329                                 dataphysaddr = sg_dma_address(sgde) +
2330                                                                    split_offset;
2331
2332                                 remainder = sg_dma_len(sgde) - split_offset;
2333
2334                                 if ((subtotal + remainder) <= protgrp_bytes) {
2335                                         /* we can use this whole buffer */
2336                                         dma_len = remainder;
2337                                         split_offset = 0;
2338
2339                                         if ((subtotal + remainder) ==
2340                                                                   protgrp_bytes)
2341                                                 pgdone = 1;
2342                                 } else {
2343                                         /* must split this buffer with next
2344                                          * prot grp
2345                                          */
2346                                         dma_len = protgrp_bytes - subtotal;
2347                                         split_offset += dma_len;
2348                                 }
2349
2350                                 subtotal += dma_len;
2351
2352                                 sgl->word2 = 0;
2353                                 sgl->addr_lo = cpu_to_le32(putPaddrLow(
2354                                                                  dataphysaddr));
2355                                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
2356                                                                  dataphysaddr));
2357                                 bf_set(lpfc_sli4_sge_last, sgl, 0);
2358                                 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
2359                                 bf_set(lpfc_sli4_sge_type, sgl,
2360                                        LPFC_SGE_TYPE_DATA);
2361
2362                                 sgl->sge_len = cpu_to_le32(dma_len);
2363                                 dma_offset += dma_len;
2364
2365                                 num_sge++;
2366
2367                                 if (split_offset) {
2368                                         sgl++;
2369                                         j++;
2370                                         break;
2371                                 }
2372
2373                                 /* Move to the next s/g segment if possible */
2374                                 sgde = sg_next(sgde);
2375
2376                                 sgl++;
2377                         }
2378
2379                         j++;
2380                 }
2381
2382                 if (protgroup_offset) {
2383                         /* update the reference tag */
2384                         reftag += protgrp_blks;
2385                         continue;
2386                 }
2387
2388                 /* are we done ? */
2389                 if (curr_prot == protcnt) {
2390                         /* mark the last SGL */
2391                         sgl--;
2392                         bf_set(lpfc_sli4_sge_last, sgl, 1);
2393                         alldone = 1;
2394                 } else if (curr_prot < protcnt) {
2395                         /* advance to next prot buffer */
2396                         sgpe = sg_next(sgpe);
2397
2398                         /* update the reference tag */
2399                         reftag += protgrp_blks;
2400                 } else {
2401                         /* if we're here, we have a bug */
2402                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2403                                         "9085 BLKGRD: bug in %s\n", __func__);
2404                 }
2405
2406         } while (!alldone);
2407
2408 out:
2409
2410         return num_sge;
2411 }
2412
2413 /**
2414  * lpfc_prot_group_type - Get prtotection group type of SCSI command
2415  * @phba: The Hba for which this call is being executed.
2416  * @sc: pointer to scsi command we're working on
2417  *
2418  * Given a SCSI command that supports DIF, determine composition of protection
2419  * groups involved in setting up buffer lists
2420  *
2421  * Returns: Protection group type (with or without DIF)
2422  *
2423  **/
2424 static int
2425 lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc)
2426 {
2427         int ret = LPFC_PG_TYPE_INVALID;
2428         unsigned char op = scsi_get_prot_op(sc);
2429
2430         switch (op) {
2431         case SCSI_PROT_READ_STRIP:
2432         case SCSI_PROT_WRITE_INSERT:
2433                 ret = LPFC_PG_TYPE_NO_DIF;
2434                 break;
2435         case SCSI_PROT_READ_INSERT:
2436         case SCSI_PROT_WRITE_STRIP:
2437         case SCSI_PROT_READ_PASS:
2438         case SCSI_PROT_WRITE_PASS:
2439                 ret = LPFC_PG_TYPE_DIF_BUF;
2440                 break;
2441         default:
2442                 if (phba)
2443                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2444                                         "9021 Unsupported protection op:%d\n",
2445                                         op);
2446                 break;
2447         }
2448         return ret;
2449 }
2450
2451 /**
2452  * lpfc_bg_scsi_adjust_dl - Adjust SCSI data length for BlockGuard
2453  * @phba: The Hba for which this call is being executed.
2454  * @lpfc_cmd: The scsi buffer which is going to be adjusted.
2455  *
2456  * Adjust the data length to account for how much data
2457  * is actually on the wire.
2458  *
2459  * returns the adjusted data length
2460  **/
2461 static int
2462 lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba,
2463                        struct lpfc_io_buf *lpfc_cmd)
2464 {
2465         struct scsi_cmnd *sc = lpfc_cmd->pCmd;
2466         int fcpdl;
2467
2468         fcpdl = scsi_bufflen(sc);
2469
2470         /* Check if there is protection data on the wire */
2471         if (sc->sc_data_direction == DMA_FROM_DEVICE) {
2472                 /* Read check for protection data */
2473                 if (scsi_get_prot_op(sc) ==  SCSI_PROT_READ_INSERT)
2474                         return fcpdl;
2475
2476         } else {
2477                 /* Write check for protection data */
2478                 if (scsi_get_prot_op(sc) ==  SCSI_PROT_WRITE_STRIP)
2479                         return fcpdl;
2480         }
2481
2482         /*
2483          * If we are in DIF Type 1 mode every data block has a 8 byte
2484          * DIF (trailer) attached to it. Must ajust FCP data length
2485          * to account for the protection data.
2486          */
2487         fcpdl += (fcpdl / scsi_prot_interval(sc)) * 8;
2488
2489         return fcpdl;
2490 }
2491
2492 /**
2493  * lpfc_bg_scsi_prep_dma_buf_s3 - DMA mapping for scsi buffer to SLI3 IF spec
2494  * @phba: The Hba for which this call is being executed.
2495  * @lpfc_cmd: The scsi buffer which is going to be prep'ed.
2496  *
2497  * This is the protection/DIF aware version of
2498  * lpfc_scsi_prep_dma_buf(). It may be a good idea to combine the
2499  * two functions eventually, but for now, it's here.
2500  * RETURNS 0 - SUCCESS,
2501  *         1 - Failed DMA map, retry.
2502  *         2 - Invalid scsi cmd or prot-type. Do not rety.
2503  **/
2504 static int
2505 lpfc_bg_scsi_prep_dma_buf_s3(struct lpfc_hba *phba,
2506                 struct lpfc_io_buf *lpfc_cmd)
2507 {
2508         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
2509         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
2510         struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl;
2511         IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb;
2512         uint32_t num_bde = 0;
2513         int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction;
2514         int prot_group_type = 0;
2515         int fcpdl;
2516         int ret = 1;
2517         struct lpfc_vport *vport = phba->pport;
2518
2519         /*
2520          * Start the lpfc command prep by bumping the bpl beyond fcp_cmnd
2521          *  fcp_rsp regions to the first data bde entry
2522          */
2523         bpl += 2;
2524         if (scsi_sg_count(scsi_cmnd)) {
2525                 /*
2526                  * The driver stores the segment count returned from dma_map_sg
2527                  * because this a count of dma-mappings used to map the use_sg
2528                  * pages.  They are not guaranteed to be the same for those
2529                  * architectures that implement an IOMMU.
2530                  */
2531                 datasegcnt = dma_map_sg(&phba->pcidev->dev,
2532                                         scsi_sglist(scsi_cmnd),
2533                                         scsi_sg_count(scsi_cmnd), datadir);
2534                 if (unlikely(!datasegcnt))
2535                         return 1;
2536
2537                 lpfc_cmd->seg_cnt = datasegcnt;
2538
2539                 /* First check if data segment count from SCSI Layer is good */
2540                 if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
2541                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
2542                         ret = 2;
2543                         goto err;
2544                 }
2545
2546                 prot_group_type = lpfc_prot_group_type(phba, scsi_cmnd);
2547
2548                 switch (prot_group_type) {
2549                 case LPFC_PG_TYPE_NO_DIF:
2550
2551                         /* Here we need to add a PDE5 and PDE6 to the count */
2552                         if ((lpfc_cmd->seg_cnt + 2) > phba->cfg_total_seg_cnt) {
2553                                 ret = 2;
2554                                 goto err;
2555                         }
2556
2557                         num_bde = lpfc_bg_setup_bpl(phba, scsi_cmnd, bpl,
2558                                         datasegcnt);
2559                         /* we should have 2 or more entries in buffer list */
2560                         if (num_bde < 2) {
2561                                 ret = 2;
2562                                 goto err;
2563                         }
2564                         break;
2565
2566                 case LPFC_PG_TYPE_DIF_BUF:
2567                         /*
2568                          * This type indicates that protection buffers are
2569                          * passed to the driver, so that needs to be prepared
2570                          * for DMA
2571                          */
2572                         protsegcnt = dma_map_sg(&phba->pcidev->dev,
2573                                         scsi_prot_sglist(scsi_cmnd),
2574                                         scsi_prot_sg_count(scsi_cmnd), datadir);
2575                         if (unlikely(!protsegcnt)) {
2576                                 scsi_dma_unmap(scsi_cmnd);
2577                                 return 1;
2578                         }
2579
2580                         lpfc_cmd->prot_seg_cnt = protsegcnt;
2581
2582                         /*
2583                          * There is a minimun of 4 BPLs used for every
2584                          * protection data segment.
2585                          */
2586                         if ((lpfc_cmd->prot_seg_cnt * 4) >
2587                             (phba->cfg_total_seg_cnt - 2)) {
2588                                 ret = 2;
2589                                 goto err;
2590                         }
2591
2592                         num_bde = lpfc_bg_setup_bpl_prot(phba, scsi_cmnd, bpl,
2593                                         datasegcnt, protsegcnt);
2594                         /* we should have 3 or more entries in buffer list */
2595                         if ((num_bde < 3) ||
2596                             (num_bde > phba->cfg_total_seg_cnt)) {
2597                                 ret = 2;
2598                                 goto err;
2599                         }
2600                         break;
2601
2602                 case LPFC_PG_TYPE_INVALID:
2603                 default:
2604                         scsi_dma_unmap(scsi_cmnd);
2605                         lpfc_cmd->seg_cnt = 0;
2606
2607                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2608                                         "9022 Unexpected protection group %i\n",
2609                                         prot_group_type);
2610                         return 2;
2611                 }
2612         }
2613
2614         /*
2615          * Finish initializing those IOCB fields that are dependent on the
2616          * scsi_cmnd request_buffer.  Note that the bdeSize is explicitly
2617          * reinitialized since all iocb memory resources are used many times
2618          * for transmit, receive, and continuation bpl's.
2619          */
2620         iocb_cmd->un.fcpi64.bdl.bdeSize = (2 * sizeof(struct ulp_bde64));
2621         iocb_cmd->un.fcpi64.bdl.bdeSize += (num_bde * sizeof(struct ulp_bde64));
2622         iocb_cmd->ulpBdeCount = 1;
2623         iocb_cmd->ulpLe = 1;
2624
2625         fcpdl = lpfc_bg_scsi_adjust_dl(phba, lpfc_cmd);
2626         fcp_cmnd->fcpDl = be32_to_cpu(fcpdl);
2627
2628         /*
2629          * Due to difference in data length between DIF/non-DIF paths,
2630          * we need to set word 4 of IOCB here
2631          */
2632         iocb_cmd->un.fcpi.fcpi_parm = fcpdl;
2633
2634         /*
2635          * For First burst, we may need to adjust the initial transfer
2636          * length for DIF
2637          */
2638         if (iocb_cmd->un.fcpi.fcpi_XRdy &&
2639             (fcpdl < vport->cfg_first_burst_size))
2640                 iocb_cmd->un.fcpi.fcpi_XRdy = fcpdl;
2641
2642         return 0;
2643 err:
2644         if (lpfc_cmd->seg_cnt)
2645                 scsi_dma_unmap(scsi_cmnd);
2646         if (lpfc_cmd->prot_seg_cnt)
2647                 dma_unmap_sg(&phba->pcidev->dev, scsi_prot_sglist(scsi_cmnd),
2648                              scsi_prot_sg_count(scsi_cmnd),
2649                              scsi_cmnd->sc_data_direction);
2650
2651         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2652                         "9023 Cannot setup S/G List for HBA"
2653                         "IO segs %d/%d BPL %d SCSI %d: %d %d\n",
2654                         lpfc_cmd->seg_cnt, lpfc_cmd->prot_seg_cnt,
2655                         phba->cfg_total_seg_cnt, phba->cfg_sg_seg_cnt,
2656                         prot_group_type, num_bde);
2657
2658         lpfc_cmd->seg_cnt = 0;
2659         lpfc_cmd->prot_seg_cnt = 0;
2660         return ret;
2661 }
2662
2663 /*
2664  * This function calcuates the T10 DIF guard tag
2665  * on the specified data using a CRC algorithmn
2666  * using crc_t10dif.
2667  */
2668 static uint16_t
2669 lpfc_bg_crc(uint8_t *data, int count)
2670 {
2671         uint16_t crc = 0;
2672         uint16_t x;
2673
2674         crc = crc_t10dif(data, count);
2675         x = cpu_to_be16(crc);
2676         return x;
2677 }
2678
2679 /*
2680  * This function calcuates the T10 DIF guard tag
2681  * on the specified data using a CSUM algorithmn
2682  * using ip_compute_csum.
2683  */
2684 static uint16_t
2685 lpfc_bg_csum(uint8_t *data, int count)
2686 {
2687         uint16_t ret;
2688
2689         ret = ip_compute_csum(data, count);
2690         return ret;
2691 }
2692
2693 /*
2694  * This function examines the protection data to try to determine
2695  * what type of T10-DIF error occurred.
2696  */
2697 static void
2698 lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
2699 {
2700         struct scatterlist *sgpe; /* s/g prot entry */
2701         struct scatterlist *sgde; /* s/g data entry */
2702         struct scsi_cmnd *cmd = lpfc_cmd->pCmd;
2703         struct scsi_dif_tuple *src = NULL;
2704         uint8_t *data_src = NULL;
2705         uint16_t guard_tag;
2706         uint16_t start_app_tag, app_tag;
2707         uint32_t start_ref_tag, ref_tag;
2708         int prot, protsegcnt;
2709         int err_type, len, data_len;
2710         int chk_ref, chk_app, chk_guard;
2711         uint16_t sum;
2712         unsigned blksize;
2713
2714         err_type = BGS_GUARD_ERR_MASK;
2715         sum = 0;
2716         guard_tag = 0;
2717
2718         /* First check to see if there is protection data to examine */
2719         prot = scsi_get_prot_op(cmd);
2720         if ((prot == SCSI_PROT_READ_STRIP) ||
2721             (prot == SCSI_PROT_WRITE_INSERT) ||
2722             (prot == SCSI_PROT_NORMAL))
2723                 goto out;
2724
2725         /* Currently the driver just supports ref_tag and guard_tag checking */
2726         chk_ref = 1;
2727         chk_app = 0;
2728         chk_guard = 0;
2729
2730         /* Setup a ptr to the protection data provided by the SCSI host */
2731         sgpe = scsi_prot_sglist(cmd);
2732         protsegcnt = lpfc_cmd->prot_seg_cnt;
2733
2734         if (sgpe && protsegcnt) {
2735
2736                 /*
2737                  * We will only try to verify guard tag if the segment
2738                  * data length is a multiple of the blksize.
2739                  */
2740                 sgde = scsi_sglist(cmd);
2741                 blksize = scsi_prot_interval(cmd);
2742                 data_src = (uint8_t *)sg_virt(sgde);
2743                 data_len = sgde->length;
2744                 if ((data_len & (blksize - 1)) == 0)
2745                         chk_guard = 1;
2746
2747                 src = (struct scsi_dif_tuple *)sg_virt(sgpe);
2748                 start_ref_tag = scsi_prot_ref_tag(cmd);
2749                 if (start_ref_tag == LPFC_INVALID_REFTAG)
2750                         goto out;
2751                 start_app_tag = src->app_tag;
2752                 len = sgpe->length;
2753                 while (src && protsegcnt) {
2754                         while (len) {
2755
2756                                 /*
2757                                  * First check to see if a protection data
2758                                  * check is valid
2759                                  */
2760                                 if ((src->ref_tag == T10_PI_REF_ESCAPE) ||
2761                                     (src->app_tag == T10_PI_APP_ESCAPE)) {
2762                                         start_ref_tag++;
2763                                         goto skipit;
2764                                 }
2765
2766                                 /* First Guard Tag checking */
2767                                 if (chk_guard) {
2768                                         guard_tag = src->guard_tag;
2769                                         if (cmd->prot_flags
2770                                             & SCSI_PROT_IP_CHECKSUM)
2771                                                 sum = lpfc_bg_csum(data_src,
2772                                                                    blksize);
2773                                         else
2774                                                 sum = lpfc_bg_crc(data_src,
2775                                                                   blksize);
2776                                         if ((guard_tag != sum)) {
2777                                                 err_type = BGS_GUARD_ERR_MASK;
2778                                                 goto out;
2779                                         }
2780                                 }
2781
2782                                 /* Reference Tag checking */
2783                                 ref_tag = be32_to_cpu(src->ref_tag);
2784                                 if (chk_ref && (ref_tag != start_ref_tag)) {
2785                                         err_type = BGS_REFTAG_ERR_MASK;
2786                                         goto out;
2787                                 }
2788                                 start_ref_tag++;
2789
2790                                 /* App Tag checking */
2791                                 app_tag = src->app_tag;
2792                                 if (chk_app && (app_tag != start_app_tag)) {
2793                                         err_type = BGS_APPTAG_ERR_MASK;
2794                                         goto out;
2795                                 }
2796 skipit:
2797                                 len -= sizeof(struct scsi_dif_tuple);
2798                                 if (len < 0)
2799                                         len = 0;
2800                                 src++;
2801
2802                                 data_src += blksize;
2803                                 data_len -= blksize;
2804
2805                                 /*
2806                                  * Are we at the end of the Data segment?
2807                                  * The data segment is only used for Guard
2808                                  * tag checking.
2809                                  */
2810                                 if (chk_guard && (data_len == 0)) {
2811                                         chk_guard = 0;
2812                                         sgde = sg_next(sgde);
2813                                         if (!sgde)
2814                                                 goto out;
2815
2816                                         data_src = (uint8_t *)sg_virt(sgde);
2817                                         data_len = sgde->length;
2818                                         if ((data_len & (blksize - 1)) == 0)
2819                                                 chk_guard = 1;
2820                                 }
2821                         }
2822
2823                         /* Goto the next Protection data segment */
2824                         sgpe = sg_next(sgpe);
2825                         if (sgpe) {
2826                                 src = (struct scsi_dif_tuple *)sg_virt(sgpe);
2827                                 len = sgpe->length;
2828                         } else {
2829                                 src = NULL;
2830                         }
2831                         protsegcnt--;
2832                 }
2833         }
2834 out:
2835         if (err_type == BGS_GUARD_ERR_MASK) {
2836                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
2837                 set_host_byte(cmd, DID_ABORT);
2838                 phba->bg_guard_err_cnt++;
2839                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2840                                 "9069 BLKGRD: reftag %x grd_tag err %x != %x\n",
2841                                 scsi_prot_ref_tag(cmd),
2842                                 sum, guard_tag);
2843
2844         } else if (err_type == BGS_REFTAG_ERR_MASK) {
2845                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
2846                 set_host_byte(cmd, DID_ABORT);
2847
2848                 phba->bg_reftag_err_cnt++;
2849                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2850                                 "9066 BLKGRD: reftag %x ref_tag err %x != %x\n",
2851                                 scsi_prot_ref_tag(cmd),
2852                                 ref_tag, start_ref_tag);
2853
2854         } else if (err_type == BGS_APPTAG_ERR_MASK) {
2855                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
2856                 set_host_byte(cmd, DID_ABORT);
2857
2858                 phba->bg_apptag_err_cnt++;
2859                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2860                                 "9041 BLKGRD: reftag %x app_tag err %x != %x\n",
2861                                 scsi_prot_ref_tag(cmd),
2862                                 app_tag, start_app_tag);
2863         }
2864 }
2865
2866 /*
2867  * This function checks for BlockGuard errors detected by
2868  * the HBA.  In case of errors, the ASC/ASCQ fields in the
2869  * sense buffer will be set accordingly, paired with
2870  * ILLEGAL_REQUEST to signal to the kernel that the HBA
2871  * detected corruption.
2872  *
2873  * Returns:
2874  *  0 - No error found
2875  *  1 - BlockGuard error found
2876  * -1 - Internal error (bad profile, ...etc)
2877  */
2878 static int
2879 lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd,
2880                   struct lpfc_iocbq *pIocbOut)
2881 {
2882         struct scsi_cmnd *cmd = lpfc_cmd->pCmd;
2883         struct sli3_bg_fields *bgf;
2884         int ret = 0;
2885         struct lpfc_wcqe_complete *wcqe;
2886         u32 status;
2887         u32 bghm = 0;
2888         u32 bgstat = 0;
2889         u64 failing_sector = 0;
2890
2891         if (phba->sli_rev == LPFC_SLI_REV4) {
2892                 wcqe = &pIocbOut->wcqe_cmpl;
2893                 status = bf_get(lpfc_wcqe_c_status, wcqe);
2894
2895                 if (status == CQE_STATUS_DI_ERROR) {
2896                         /* Guard Check failed */
2897                         if (bf_get(lpfc_wcqe_c_bg_ge, wcqe))
2898                                 bgstat |= BGS_GUARD_ERR_MASK;
2899
2900                         /* AppTag Check failed */
2901                         if (bf_get(lpfc_wcqe_c_bg_ae, wcqe))
2902                                 bgstat |= BGS_APPTAG_ERR_MASK;
2903
2904                         /* RefTag Check failed */
2905                         if (bf_get(lpfc_wcqe_c_bg_re, wcqe))
2906                                 bgstat |= BGS_REFTAG_ERR_MASK;
2907
2908                         /* Check to see if there was any good data before the
2909                          * error
2910                          */
2911                         if (bf_get(lpfc_wcqe_c_bg_tdpv, wcqe)) {
2912                                 bgstat |= BGS_HI_WATER_MARK_PRESENT_MASK;
2913                                 bghm = wcqe->total_data_placed;
2914                         }
2915
2916                         /*
2917                          * Set ALL the error bits to indicate we don't know what
2918                          * type of error it is.
2919                          */
2920                         if (!bgstat)
2921                                 bgstat |= (BGS_REFTAG_ERR_MASK |
2922                                            BGS_APPTAG_ERR_MASK |
2923                                            BGS_GUARD_ERR_MASK);
2924                 }
2925
2926         } else {
2927                 bgf = &pIocbOut->iocb.unsli3.sli3_bg;
2928                 bghm = bgf->bghm;
2929                 bgstat = bgf->bgstat;
2930         }
2931
2932         if (lpfc_bgs_get_invalid_prof(bgstat)) {
2933                 cmd->result = DID_ERROR << 16;
2934                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2935                                 "9072 BLKGRD: Invalid BG Profile in cmd "
2936                                 "0x%x reftag 0x%x blk cnt 0x%x "
2937                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2938                                 scsi_prot_ref_tag(cmd),
2939                                 scsi_logical_block_count(cmd), bgstat, bghm);
2940                 ret = (-1);
2941                 goto out;
2942         }
2943
2944         if (lpfc_bgs_get_uninit_dif_block(bgstat)) {
2945                 cmd->result = DID_ERROR << 16;
2946                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2947                                 "9073 BLKGRD: Invalid BG PDIF Block in cmd "
2948                                 "0x%x reftag 0x%x blk cnt 0x%x "
2949                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2950                                 scsi_prot_ref_tag(cmd),
2951                                 scsi_logical_block_count(cmd), bgstat, bghm);
2952                 ret = (-1);
2953                 goto out;
2954         }
2955
2956         if (lpfc_bgs_get_guard_err(bgstat)) {
2957                 ret = 1;
2958                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
2959                 set_host_byte(cmd, DID_ABORT);
2960                 phba->bg_guard_err_cnt++;
2961                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2962                                 "9055 BLKGRD: Guard Tag error in cmd "
2963                                 "0x%x reftag 0x%x blk cnt 0x%x "
2964                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2965                                 scsi_prot_ref_tag(cmd),
2966                                 scsi_logical_block_count(cmd), bgstat, bghm);
2967         }
2968
2969         if (lpfc_bgs_get_reftag_err(bgstat)) {
2970                 ret = 1;
2971                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
2972                 set_host_byte(cmd, DID_ABORT);
2973                 phba->bg_reftag_err_cnt++;
2974                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2975                                 "9056 BLKGRD: Ref Tag error in cmd "
2976                                 "0x%x reftag 0x%x blk cnt 0x%x "
2977                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2978                                 scsi_prot_ref_tag(cmd),
2979                                 scsi_logical_block_count(cmd), bgstat, bghm);
2980         }
2981
2982         if (lpfc_bgs_get_apptag_err(bgstat)) {
2983                 ret = 1;
2984                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
2985                 set_host_byte(cmd, DID_ABORT);
2986                 phba->bg_apptag_err_cnt++;
2987                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2988                                 "9061 BLKGRD: App Tag error in cmd "
2989                                 "0x%x reftag 0x%x blk cnt 0x%x "
2990                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2991                                 scsi_prot_ref_tag(cmd),
2992                                 scsi_logical_block_count(cmd), bgstat, bghm);
2993         }
2994
2995         if (lpfc_bgs_get_hi_water_mark_present(bgstat)) {
2996                 /*
2997                  * setup sense data descriptor 0 per SPC-4 as an information
2998                  * field, and put the failing LBA in it.
2999                  * This code assumes there was also a guard/app/ref tag error
3000                  * indication.
3001                  */
3002                 cmd->sense_buffer[7] = 0xc;   /* Additional sense length */
3003                 cmd->sense_buffer[8] = 0;     /* Information descriptor type */
3004                 cmd->sense_buffer[9] = 0xa;   /* Additional descriptor length */
3005                 cmd->sense_buffer[10] = 0x80; /* Validity bit */
3006
3007                 /* bghm is a "on the wire" FC frame based count */
3008                 switch (scsi_get_prot_op(cmd)) {
3009                 case SCSI_PROT_READ_INSERT:
3010                 case SCSI_PROT_WRITE_STRIP:
3011                         bghm /= cmd->device->sector_size;
3012                         break;
3013                 case SCSI_PROT_READ_STRIP:
3014                 case SCSI_PROT_WRITE_INSERT:
3015                 case SCSI_PROT_READ_PASS:
3016                 case SCSI_PROT_WRITE_PASS:
3017                         bghm /= (cmd->device->sector_size +
3018                                 sizeof(struct scsi_dif_tuple));
3019                         break;
3020                 }
3021
3022                 failing_sector = scsi_get_lba(cmd);
3023                 failing_sector += bghm;
3024
3025                 /* Descriptor Information */
3026                 put_unaligned_be64(failing_sector, &cmd->sense_buffer[12]);
3027         }
3028
3029         if (!ret) {
3030                 /* No error was reported - problem in FW? */
3031                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
3032                                 "9057 BLKGRD: Unknown error in cmd "
3033                                 "0x%x reftag 0x%x blk cnt 0x%x "
3034                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
3035                                 scsi_prot_ref_tag(cmd),
3036                                 scsi_logical_block_count(cmd), bgstat, bghm);
3037
3038                 /* Calculate what type of error it was */
3039                 lpfc_calc_bg_err(phba, lpfc_cmd);
3040         }
3041 out:
3042         return ret;
3043 }
3044
3045 /**
3046  * lpfc_scsi_prep_dma_buf_s4 - DMA mapping for scsi buffer to SLI4 IF spec
3047  * @phba: The Hba for which this call is being executed.
3048  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3049  *
3050  * This routine does the pci dma mapping for scatter-gather list of scsi cmnd
3051  * field of @lpfc_cmd for device with SLI-4 interface spec.
3052  *
3053  * Return codes:
3054  *      2 - Error - Do not retry
3055  *      1 - Error - Retry
3056  *      0 - Success
3057  **/
3058 static int
3059 lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
3060 {
3061         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
3062         struct scatterlist *sgel = NULL;
3063         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
3064         struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
3065         struct sli4_sge *first_data_sgl;
3066         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
3067         struct lpfc_vport *vport = phba->pport;
3068         union lpfc_wqe128 *wqe = &pwqeq->wqe;
3069         dma_addr_t physaddr;
3070         uint32_t dma_len;
3071         uint32_t dma_offset = 0;
3072         int nseg, i, j;
3073         struct ulp_bde64 *bde;
3074         bool lsp_just_set = false;
3075         struct sli4_hybrid_sgl *sgl_xtra = NULL;
3076
3077         /*
3078          * There are three possibilities here - use scatter-gather segment, use
3079          * the single mapping, or neither.  Start the lpfc command prep by
3080          * bumping the bpl beyond the fcp_cmnd and fcp_rsp regions to the first
3081          * data bde entry.
3082          */
3083         if (scsi_sg_count(scsi_cmnd)) {
3084                 /*
3085                  * The driver stores the segment count returned from dma_map_sg
3086                  * because this a count of dma-mappings used to map the use_sg
3087                  * pages.  They are not guaranteed to be the same for those
3088                  * architectures that implement an IOMMU.
3089                  */
3090
3091                 nseg = scsi_dma_map(scsi_cmnd);
3092                 if (unlikely(nseg <= 0))
3093                         return 1;
3094                 sgl += 1;
3095                 /* clear the last flag in the fcp_rsp map entry */
3096                 sgl->word2 = le32_to_cpu(sgl->word2);
3097                 bf_set(lpfc_sli4_sge_last, sgl, 0);
3098                 sgl->word2 = cpu_to_le32(sgl->word2);
3099                 sgl += 1;
3100                 first_data_sgl = sgl;
3101                 lpfc_cmd->seg_cnt = nseg;
3102                 if (!phba->cfg_xpsgl &&
3103                     lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
3104                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3105                                         "9074 BLKGRD:"
3106                                         " %s: Too many sg segments from "
3107                                         "dma_map_sg.  Config %d, seg_cnt %d\n",
3108                                         __func__, phba->cfg_sg_seg_cnt,
3109                                         lpfc_cmd->seg_cnt);
3110                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
3111                         lpfc_cmd->seg_cnt = 0;
3112                         scsi_dma_unmap(scsi_cmnd);
3113                         return 2;
3114                 }
3115
3116                 /*
3117                  * The driver established a maximum scatter-gather segment count
3118                  * during probe that limits the number of sg elements in any
3119                  * single scsi command.  Just run through the seg_cnt and format
3120                  * the sge's.
3121                  * When using SLI-3 the driver will try to fit all the BDEs into
3122                  * the IOCB. If it can't then the BDEs get added to a BPL as it
3123                  * does for SLI-2 mode.
3124                  */
3125
3126                 /* for tracking segment boundaries */
3127                 sgel = scsi_sglist(scsi_cmnd);
3128                 j = 2;
3129                 for (i = 0; i < nseg; i++) {
3130                         sgl->word2 = 0;
3131                         if (nseg == 1) {
3132                                 bf_set(lpfc_sli4_sge_last, sgl, 1);
3133                                 bf_set(lpfc_sli4_sge_type, sgl,
3134                                        LPFC_SGE_TYPE_DATA);
3135                         } else {
3136                                 bf_set(lpfc_sli4_sge_last, sgl, 0);
3137
3138                                 /* do we need to expand the segment */
3139                                 if (!lsp_just_set &&
3140                                     !((j + 1) % phba->border_sge_num) &&
3141                                     ((nseg - 1) != i)) {
3142                                         /* set LSP type */
3143                                         bf_set(lpfc_sli4_sge_type, sgl,
3144                                                LPFC_SGE_TYPE_LSP);
3145
3146                                         sgl_xtra = lpfc_get_sgl_per_hdwq(
3147                                                         phba, lpfc_cmd);
3148
3149                                         if (unlikely(!sgl_xtra)) {
3150                                                 lpfc_cmd->seg_cnt = 0;
3151                                                 scsi_dma_unmap(scsi_cmnd);
3152                                                 return 1;
3153                                         }
3154                                         sgl->addr_lo = cpu_to_le32(putPaddrLow(
3155                                                        sgl_xtra->dma_phys_sgl));
3156                                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(
3157                                                        sgl_xtra->dma_phys_sgl));
3158
3159                                 } else {
3160                                         bf_set(lpfc_sli4_sge_type, sgl,
3161                                                LPFC_SGE_TYPE_DATA);
3162                                 }
3163                         }
3164
3165                         if (!(bf_get(lpfc_sli4_sge_type, sgl) &
3166                                      LPFC_SGE_TYPE_LSP)) {
3167                                 if ((nseg - 1) == i)
3168                                         bf_set(lpfc_sli4_sge_last, sgl, 1);
3169
3170                                 physaddr = sg_dma_address(sgel);
3171                                 dma_len = sg_dma_len(sgel);
3172                                 sgl->addr_lo = cpu_to_le32(putPaddrLow(
3173                                                            physaddr));
3174                                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
3175                                                            physaddr));
3176
3177                                 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
3178                                 sgl->word2 = cpu_to_le32(sgl->word2);
3179                                 sgl->sge_len = cpu_to_le32(dma_len);
3180
3181                                 dma_offset += dma_len;
3182                                 sgel = sg_next(sgel);
3183
3184                                 sgl++;
3185                                 lsp_just_set = false;
3186
3187                         } else {
3188                                 sgl->word2 = cpu_to_le32(sgl->word2);
3189                                 sgl->sge_len = cpu_to_le32(
3190                                                      phba->cfg_sg_dma_buf_size);
3191
3192                                 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
3193                                 i = i - 1;
3194
3195                                 lsp_just_set = true;
3196                         }
3197
3198                         j++;
3199                 }
3200
3201                 /* PBDE support for first data SGE only.
3202                  * For FCoE, we key off Performance Hints.
3203                  * For FC, we key off lpfc_enable_pbde.
3204                  */
3205                 if (nseg == 1 &&
3206                     ((phba->sli3_options & LPFC_SLI4_PERFH_ENABLED) ||
3207                      phba->cfg_enable_pbde)) {
3208                         /* Words 13-15 */
3209                         bde = (struct ulp_bde64 *)
3210                                 &wqe->words[13];
3211                         bde->addrLow = first_data_sgl->addr_lo;
3212                         bde->addrHigh = first_data_sgl->addr_hi;
3213                         bde->tus.f.bdeSize =
3214                                         le32_to_cpu(first_data_sgl->sge_len);
3215                         bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
3216                         bde->tus.w = cpu_to_le32(bde->tus.w);
3217
3218                         /* Word 11 - set PBDE bit */
3219                         bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
3220                 } else {
3221                         memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
3222                         /* Word 11 - PBDE bit disabled by default template */
3223                 }
3224         } else {
3225                 sgl += 1;
3226                 /* set the last flag in the fcp_rsp map entry */
3227                 sgl->word2 = le32_to_cpu(sgl->word2);
3228                 bf_set(lpfc_sli4_sge_last, sgl, 1);
3229                 sgl->word2 = cpu_to_le32(sgl->word2);
3230
3231                 if ((phba->sli3_options & LPFC_SLI4_PERFH_ENABLED) ||
3232                     phba->cfg_enable_pbde) {
3233                         bde = (struct ulp_bde64 *)
3234                                 &wqe->words[13];
3235                         memset(bde, 0, (sizeof(uint32_t) * 3));
3236                 }
3237         }
3238
3239         /*
3240          * Finish initializing those IOCB fields that are dependent on the
3241          * scsi_cmnd request_buffer.  Note that for SLI-2 the bdeSize is
3242          * explicitly reinitialized.
3243          * all iocb memory resources are reused.
3244          */
3245         fcp_cmnd->fcpDl = cpu_to_be32(scsi_bufflen(scsi_cmnd));
3246         /* Set first-burst provided it was successfully negotiated */
3247         if (!(phba->hba_flag & HBA_FCOE_MODE) &&
3248             vport->cfg_first_burst_size &&
3249             scsi_cmnd->sc_data_direction == DMA_TO_DEVICE) {
3250                 u32 init_len, total_len;
3251
3252                 total_len = be32_to_cpu(fcp_cmnd->fcpDl);
3253                 init_len = min(total_len, vport->cfg_first_burst_size);
3254
3255                 /* Word 4 & 5 */
3256                 wqe->fcp_iwrite.initial_xfer_len = init_len;
3257                 wqe->fcp_iwrite.total_xfer_len = total_len;
3258         } else {
3259                 /* Word 4 */
3260                 wqe->fcp_iwrite.total_xfer_len =
3261                         be32_to_cpu(fcp_cmnd->fcpDl);
3262         }
3263
3264         /*
3265          * If the OAS driver feature is enabled and the lun is enabled for
3266          * OAS, set the oas iocb related flags.
3267          */
3268         if ((phba->cfg_fof) && ((struct lpfc_device_data *)
3269                 scsi_cmnd->device->hostdata)->oas_enabled) {
3270                 lpfc_cmd->cur_iocbq.cmd_flag |= (LPFC_IO_OAS | LPFC_IO_FOF);
3271                 lpfc_cmd->cur_iocbq.priority = ((struct lpfc_device_data *)
3272                         scsi_cmnd->device->hostdata)->priority;
3273
3274                 /* Word 10 */
3275                 bf_set(wqe_oas, &wqe->generic.wqe_com, 1);
3276                 bf_set(wqe_ccpe, &wqe->generic.wqe_com, 1);
3277
3278                 if (lpfc_cmd->cur_iocbq.priority)
3279                         bf_set(wqe_ccp, &wqe->generic.wqe_com,
3280                                (lpfc_cmd->cur_iocbq.priority << 1));
3281                 else
3282                         bf_set(wqe_ccp, &wqe->generic.wqe_com,
3283                                (phba->cfg_XLanePriority << 1));
3284         }
3285
3286         return 0;
3287 }
3288
3289 /**
3290  * lpfc_bg_scsi_prep_dma_buf_s4 - DMA mapping for scsi buffer to SLI4 IF spec
3291  * @phba: The Hba for which this call is being executed.
3292  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3293  *
3294  * This is the protection/DIF aware version of
3295  * lpfc_scsi_prep_dma_buf(). It may be a good idea to combine the
3296  * two functions eventually, but for now, it's here
3297  * Return codes:
3298  *      2 - Error - Do not retry
3299  *      1 - Error - Retry
3300  *      0 - Success
3301  **/
3302 static int
3303 lpfc_bg_scsi_prep_dma_buf_s4(struct lpfc_hba *phba,
3304                 struct lpfc_io_buf *lpfc_cmd)
3305 {
3306         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
3307         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
3308         struct sli4_sge *sgl = (struct sli4_sge *)(lpfc_cmd->dma_sgl);
3309         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
3310         union lpfc_wqe128 *wqe = &pwqeq->wqe;
3311         uint32_t num_sge = 0;
3312         int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction;
3313         int prot_group_type = 0;
3314         int fcpdl;
3315         int ret = 1;
3316         struct lpfc_vport *vport = phba->pport;
3317
3318         /*
3319          * Start the lpfc command prep by bumping the sgl beyond fcp_cmnd
3320          *  fcp_rsp regions to the first data sge entry
3321          */
3322         if (scsi_sg_count(scsi_cmnd)) {
3323                 /*
3324                  * The driver stores the segment count returned from dma_map_sg
3325                  * because this a count of dma-mappings used to map the use_sg
3326                  * pages.  They are not guaranteed to be the same for those
3327                  * architectures that implement an IOMMU.
3328                  */
3329                 datasegcnt = dma_map_sg(&phba->pcidev->dev,
3330                                         scsi_sglist(scsi_cmnd),
3331                                         scsi_sg_count(scsi_cmnd), datadir);
3332                 if (unlikely(!datasegcnt))
3333                         return 1;
3334
3335                 sgl += 1;
3336                 /* clear the last flag in the fcp_rsp map entry */
3337                 sgl->word2 = le32_to_cpu(sgl->word2);
3338                 bf_set(lpfc_sli4_sge_last, sgl, 0);
3339                 sgl->word2 = cpu_to_le32(sgl->word2);
3340
3341                 sgl += 1;
3342                 lpfc_cmd->seg_cnt = datasegcnt;
3343
3344                 /* First check if data segment count from SCSI Layer is good */
3345                 if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt &&
3346                     !phba->cfg_xpsgl) {
3347                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
3348                         ret = 2;
3349                         goto err;
3350                 }
3351
3352                 prot_group_type = lpfc_prot_group_type(phba, scsi_cmnd);
3353
3354                 switch (prot_group_type) {
3355                 case LPFC_PG_TYPE_NO_DIF:
3356                         /* Here we need to add a DISEED to the count */
3357                         if (((lpfc_cmd->seg_cnt + 1) >
3358                                         phba->cfg_total_seg_cnt) &&
3359                             !phba->cfg_xpsgl) {
3360                                 ret = 2;
3361                                 goto err;
3362                         }
3363
3364                         num_sge = lpfc_bg_setup_sgl(phba, scsi_cmnd, sgl,
3365                                         datasegcnt, lpfc_cmd);
3366
3367                         /* we should have 2 or more entries in buffer list */
3368                         if (num_sge < 2) {
3369                                 ret = 2;
3370                                 goto err;
3371                         }
3372                         break;
3373
3374                 case LPFC_PG_TYPE_DIF_BUF:
3375                         /*
3376                          * This type indicates that protection buffers are
3377                          * passed to the driver, so that needs to be prepared
3378                          * for DMA
3379                          */
3380                         protsegcnt = dma_map_sg(&phba->pcidev->dev,
3381                                         scsi_prot_sglist(scsi_cmnd),
3382                                         scsi_prot_sg_count(scsi_cmnd), datadir);
3383                         if (unlikely(!protsegcnt)) {
3384                                 scsi_dma_unmap(scsi_cmnd);
3385                                 return 1;
3386                         }
3387
3388                         lpfc_cmd->prot_seg_cnt = protsegcnt;
3389                         /*
3390                          * There is a minimun of 3 SGEs used for every
3391                          * protection data segment.
3392                          */
3393                         if (((lpfc_cmd->prot_seg_cnt * 3) >
3394                                         (phba->cfg_total_seg_cnt - 2)) &&
3395                             !phba->cfg_xpsgl) {
3396                                 ret = 2;
3397                                 goto err;
3398                         }
3399
3400                         num_sge = lpfc_bg_setup_sgl_prot(phba, scsi_cmnd, sgl,
3401                                         datasegcnt, protsegcnt, lpfc_cmd);
3402
3403                         /* we should have 3 or more entries in buffer list */
3404                         if (num_sge < 3 ||
3405                             (num_sge > phba->cfg_total_seg_cnt &&
3406                              !phba->cfg_xpsgl)) {
3407                                 ret = 2;
3408                                 goto err;
3409                         }
3410                         break;
3411
3412                 case LPFC_PG_TYPE_INVALID:
3413                 default:
3414                         scsi_dma_unmap(scsi_cmnd);
3415                         lpfc_cmd->seg_cnt = 0;
3416
3417                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3418                                         "9083 Unexpected protection group %i\n",
3419                                         prot_group_type);
3420                         return 2;
3421                 }
3422         }
3423
3424         switch (scsi_get_prot_op(scsi_cmnd)) {
3425         case SCSI_PROT_WRITE_STRIP:
3426         case SCSI_PROT_READ_STRIP:
3427                 lpfc_cmd->cur_iocbq.cmd_flag |= LPFC_IO_DIF_STRIP;
3428                 break;
3429         case SCSI_PROT_WRITE_INSERT:
3430         case SCSI_PROT_READ_INSERT:
3431                 lpfc_cmd->cur_iocbq.cmd_flag |= LPFC_IO_DIF_INSERT;
3432                 break;
3433         case SCSI_PROT_WRITE_PASS:
3434         case SCSI_PROT_READ_PASS:
3435                 lpfc_cmd->cur_iocbq.cmd_flag |= LPFC_IO_DIF_PASS;
3436                 break;
3437         }
3438
3439         fcpdl = lpfc_bg_scsi_adjust_dl(phba, lpfc_cmd);
3440         fcp_cmnd->fcpDl = be32_to_cpu(fcpdl);
3441
3442         /* Set first-burst provided it was successfully negotiated */
3443         if (!(phba->hba_flag & HBA_FCOE_MODE) &&
3444             vport->cfg_first_burst_size &&
3445             scsi_cmnd->sc_data_direction == DMA_TO_DEVICE) {
3446                 u32 init_len, total_len;
3447
3448                 total_len = be32_to_cpu(fcp_cmnd->fcpDl);
3449                 init_len = min(total_len, vport->cfg_first_burst_size);
3450
3451                 /* Word 4 & 5 */
3452                 wqe->fcp_iwrite.initial_xfer_len = init_len;
3453                 wqe->fcp_iwrite.total_xfer_len = total_len;
3454         } else {
3455                 /* Word 4 */
3456                 wqe->fcp_iwrite.total_xfer_len =
3457                         be32_to_cpu(fcp_cmnd->fcpDl);
3458         }
3459
3460         /*
3461          * If the OAS driver feature is enabled and the lun is enabled for
3462          * OAS, set the oas iocb related flags.
3463          */
3464         if ((phba->cfg_fof) && ((struct lpfc_device_data *)
3465                 scsi_cmnd->device->hostdata)->oas_enabled) {
3466                 lpfc_cmd->cur_iocbq.cmd_flag |= (LPFC_IO_OAS | LPFC_IO_FOF);
3467
3468                 /* Word 10 */
3469                 bf_set(wqe_oas, &wqe->generic.wqe_com, 1);
3470                 bf_set(wqe_ccpe, &wqe->generic.wqe_com, 1);
3471                 bf_set(wqe_ccp, &wqe->generic.wqe_com,
3472                        (phba->cfg_XLanePriority << 1));
3473         }
3474
3475         /* Word 7. DIF Flags */
3476         if (lpfc_cmd->cur_iocbq.cmd_flag & LPFC_IO_DIF_PASS)
3477                 bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_PASSTHRU);
3478         else if (lpfc_cmd->cur_iocbq.cmd_flag & LPFC_IO_DIF_STRIP)
3479                 bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_STRIP);
3480         else if (lpfc_cmd->cur_iocbq.cmd_flag & LPFC_IO_DIF_INSERT)
3481                 bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_INSERT);
3482
3483         lpfc_cmd->cur_iocbq.cmd_flag &= ~(LPFC_IO_DIF_PASS |
3484                                  LPFC_IO_DIF_STRIP | LPFC_IO_DIF_INSERT);
3485
3486         return 0;
3487 err:
3488         if (lpfc_cmd->seg_cnt)
3489                 scsi_dma_unmap(scsi_cmnd);
3490         if (lpfc_cmd->prot_seg_cnt)
3491                 dma_unmap_sg(&phba->pcidev->dev, scsi_prot_sglist(scsi_cmnd),
3492                              scsi_prot_sg_count(scsi_cmnd),
3493                              scsi_cmnd->sc_data_direction);
3494
3495         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3496                         "9084 Cannot setup S/G List for HBA"
3497                         "IO segs %d/%d SGL %d SCSI %d: %d %d\n",
3498                         lpfc_cmd->seg_cnt, lpfc_cmd->prot_seg_cnt,
3499                         phba->cfg_total_seg_cnt, phba->cfg_sg_seg_cnt,
3500                         prot_group_type, num_sge);
3501
3502         lpfc_cmd->seg_cnt = 0;
3503         lpfc_cmd->prot_seg_cnt = 0;
3504         return ret;
3505 }
3506
3507 /**
3508  * lpfc_scsi_prep_dma_buf - Wrapper function for DMA mapping of scsi buffer
3509  * @phba: The Hba for which this call is being executed.
3510  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3511  *
3512  * This routine wraps the actual DMA mapping function pointer from the
3513  * lpfc_hba struct.
3514  *
3515  * Return codes:
3516  *      1 - Error
3517  *      0 - Success
3518  **/
3519 static inline int
3520 lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
3521 {
3522         return phba->lpfc_scsi_prep_dma_buf(phba, lpfc_cmd);
3523 }
3524
3525 /**
3526  * lpfc_bg_scsi_prep_dma_buf - Wrapper function for DMA mapping of scsi buffer
3527  * using BlockGuard.
3528  * @phba: The Hba for which this call is being executed.
3529  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3530  *
3531  * This routine wraps the actual DMA mapping function pointer from the
3532  * lpfc_hba struct.
3533  *
3534  * Return codes:
3535  *      1 - Error
3536  *      0 - Success
3537  **/
3538 static inline int
3539 lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
3540 {
3541         return phba->lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd);
3542 }
3543
3544 /**
3545  * lpfc_scsi_prep_cmnd_buf - Wrapper function for IOCB/WQE mapping of scsi
3546  * buffer
3547  * @vport: Pointer to vport object.
3548  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3549  * @tmo: Timeout value for IO
3550  *
3551  * This routine initializes IOCB/WQE data structure from scsi command
3552  *
3553  * Return codes:
3554  *      1 - Error
3555  *      0 - Success
3556  **/
3557 static inline int
3558 lpfc_scsi_prep_cmnd_buf(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd,
3559                         uint8_t tmo)
3560 {
3561         return vport->phba->lpfc_scsi_prep_cmnd_buf(vport, lpfc_cmd, tmo);
3562 }
3563
3564 /**
3565  * lpfc_send_scsi_error_event - Posts an event when there is SCSI error
3566  * @phba: Pointer to hba context object.
3567  * @vport: Pointer to vport object.
3568  * @lpfc_cmd: Pointer to lpfc scsi command which reported the error.
3569  * @fcpi_parm: FCP Initiator parameter.
3570  *
3571  * This function posts an event when there is a SCSI command reporting
3572  * error from the scsi device.
3573  **/
3574 static void
3575 lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport,
3576                 struct lpfc_io_buf *lpfc_cmd, uint32_t fcpi_parm) {
3577         struct scsi_cmnd *cmnd = lpfc_cmd->pCmd;
3578         struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp;
3579         uint32_t resp_info = fcprsp->rspStatus2;
3580         uint32_t scsi_status = fcprsp->rspStatus3;
3581         struct lpfc_fast_path_event *fast_path_evt = NULL;
3582         struct lpfc_nodelist *pnode = lpfc_cmd->rdata->pnode;
3583         unsigned long flags;
3584
3585         if (!pnode)
3586                 return;
3587
3588         /* If there is queuefull or busy condition send a scsi event */
3589         if ((cmnd->result == SAM_STAT_TASK_SET_FULL) ||
3590                 (cmnd->result == SAM_STAT_BUSY)) {
3591                 fast_path_evt = lpfc_alloc_fast_evt(phba);
3592                 if (!fast_path_evt)
3593                         return;
3594                 fast_path_evt->un.scsi_evt.event_type =
3595                         FC_REG_SCSI_EVENT;
3596                 fast_path_evt->un.scsi_evt.subcategory =
3597                 (cmnd->result == SAM_STAT_TASK_SET_FULL) ?
3598                 LPFC_EVENT_QFULL : LPFC_EVENT_DEVBSY;
3599                 fast_path_evt->un.scsi_evt.lun = cmnd->device->lun;
3600                 memcpy(&fast_path_evt->un.scsi_evt.wwpn,
3601                         &pnode->nlp_portname, sizeof(struct lpfc_name));
3602                 memcpy(&fast_path_evt->un.scsi_evt.wwnn,
3603                         &pnode->nlp_nodename, sizeof(struct lpfc_name));
3604         } else if ((resp_info & SNS_LEN_VALID) && fcprsp->rspSnsLen &&
3605                 ((cmnd->cmnd[0] == READ_10) || (cmnd->cmnd[0] == WRITE_10))) {
3606                 fast_path_evt = lpfc_alloc_fast_evt(phba);
3607                 if (!fast_path_evt)
3608                         return;
3609                 fast_path_evt->un.check_cond_evt.scsi_event.event_type =
3610                         FC_REG_SCSI_EVENT;
3611                 fast_path_evt->un.check_cond_evt.scsi_event.subcategory =
3612                         LPFC_EVENT_CHECK_COND;
3613                 fast_path_evt->un.check_cond_evt.scsi_event.lun =
3614                         cmnd->device->lun;
3615                 memcpy(&fast_path_evt->un.check_cond_evt.scsi_event.wwpn,
3616                         &pnode->nlp_portname, sizeof(struct lpfc_name));
3617                 memcpy(&fast_path_evt->un.check_cond_evt.scsi_event.wwnn,
3618                         &pnode->nlp_nodename, sizeof(struct lpfc_name));
3619                 fast_path_evt->un.check_cond_evt.sense_key =
3620                         cmnd->sense_buffer[2] & 0xf;
3621                 fast_path_evt->un.check_cond_evt.asc = cmnd->sense_buffer[12];
3622                 fast_path_evt->un.check_cond_evt.ascq = cmnd->sense_buffer[13];
3623         } else if ((cmnd->sc_data_direction == DMA_FROM_DEVICE) &&
3624                      fcpi_parm &&
3625                      ((be32_to_cpu(fcprsp->rspResId) != fcpi_parm) ||
3626                         ((scsi_status == SAM_STAT_GOOD) &&
3627                         !(resp_info & (RESID_UNDER | RESID_OVER))))) {
3628                 /*
3629                  * If status is good or resid does not match with fcp_param and
3630                  * there is valid fcpi_parm, then there is a read_check error
3631                  */
3632                 fast_path_evt = lpfc_alloc_fast_evt(phba);
3633                 if (!fast_path_evt)
3634                         return;
3635                 fast_path_evt->un.read_check_error.header.event_type =
3636                         FC_REG_FABRIC_EVENT;
3637                 fast_path_evt->un.read_check_error.header.subcategory =
3638                         LPFC_EVENT_FCPRDCHKERR;
3639                 memcpy(&fast_path_evt->un.read_check_error.header.wwpn,
3640                         &pnode->nlp_portname, sizeof(struct lpfc_name));
3641                 memcpy(&fast_path_evt->un.read_check_error.header.wwnn,
3642                         &pnode->nlp_nodename, sizeof(struct lpfc_name));
3643                 fast_path_evt->un.read_check_error.lun = cmnd->device->lun;
3644                 fast_path_evt->un.read_check_error.opcode = cmnd->cmnd[0];
3645                 fast_path_evt->un.read_check_error.fcpiparam =
3646                         fcpi_parm;
3647         } else
3648                 return;
3649
3650         fast_path_evt->vport = vport;
3651         spin_lock_irqsave(&phba->hbalock, flags);
3652         list_add_tail(&fast_path_evt->work_evt.evt_listp, &phba->work_list);
3653         spin_unlock_irqrestore(&phba->hbalock, flags);
3654         lpfc_worker_wake_up(phba);
3655         return;
3656 }
3657
3658 /**
3659  * lpfc_scsi_unprep_dma_buf - Un-map DMA mapping of SG-list for dev
3660  * @phba: The HBA for which this call is being executed.
3661  * @psb: The scsi buffer which is going to be un-mapped.
3662  *
3663  * This routine does DMA un-mapping of scatter gather list of scsi command
3664  * field of @lpfc_cmd for device with SLI-3 interface spec.
3665  **/
3666 static void
3667 lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
3668 {
3669         /*
3670          * There are only two special cases to consider.  (1) the scsi command
3671          * requested scatter-gather usage or (2) the scsi command allocated
3672          * a request buffer, but did not request use_sg.  There is a third
3673          * case, but it does not require resource deallocation.
3674          */
3675         if (psb->seg_cnt > 0)
3676                 scsi_dma_unmap(psb->pCmd);
3677         if (psb->prot_seg_cnt > 0)
3678                 dma_unmap_sg(&phba->pcidev->dev, scsi_prot_sglist(psb->pCmd),
3679                                 scsi_prot_sg_count(psb->pCmd),
3680                                 psb->pCmd->sc_data_direction);
3681 }
3682
3683 /**
3684  * lpfc_unblock_requests - allow further commands to be queued.
3685  * @phba: pointer to phba object
3686  *
3687  * For single vport, just call scsi_unblock_requests on physical port.
3688  * For multiple vports, send scsi_unblock_requests for all the vports.
3689  */
3690 void
3691 lpfc_unblock_requests(struct lpfc_hba *phba)
3692 {
3693         struct lpfc_vport **vports;
3694         struct Scsi_Host  *shost;
3695         int i;
3696
3697         if (phba->sli_rev == LPFC_SLI_REV4 &&
3698             !phba->sli4_hba.max_cfg_param.vpi_used) {
3699                 shost = lpfc_shost_from_vport(phba->pport);
3700                 scsi_unblock_requests(shost);
3701                 return;
3702         }
3703
3704         vports = lpfc_create_vport_work_array(phba);
3705         if (vports != NULL)
3706                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
3707                         shost = lpfc_shost_from_vport(vports[i]);
3708                         scsi_unblock_requests(shost);
3709                 }
3710         lpfc_destroy_vport_work_array(phba, vports);
3711 }
3712
3713 /**
3714  * lpfc_block_requests - prevent further commands from being queued.
3715  * @phba: pointer to phba object
3716  *
3717  * For single vport, just call scsi_block_requests on physical port.
3718  * For multiple vports, send scsi_block_requests for all the vports.
3719  */
3720 void
3721 lpfc_block_requests(struct lpfc_hba *phba)
3722 {
3723         struct lpfc_vport **vports;
3724         struct Scsi_Host  *shost;
3725         int i;
3726
3727         if (atomic_read(&phba->cmf_stop_io))
3728                 return;
3729
3730         if (phba->sli_rev == LPFC_SLI_REV4 &&
3731             !phba->sli4_hba.max_cfg_param.vpi_used) {
3732                 shost = lpfc_shost_from_vport(phba->pport);
3733                 scsi_block_requests(shost);
3734                 return;
3735         }
3736
3737         vports = lpfc_create_vport_work_array(phba);
3738         if (vports != NULL)
3739                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
3740                         shost = lpfc_shost_from_vport(vports[i]);
3741                         scsi_block_requests(shost);
3742                 }
3743         lpfc_destroy_vport_work_array(phba, vports);
3744 }
3745
3746 /**
3747  * lpfc_update_cmf_cmpl - Adjust CMF counters for IO completion
3748  * @phba: The HBA for which this call is being executed.
3749  * @time: The latency of the IO that completed (in ns)
3750  * @size: The size of the IO that completed
3751  * @shost: SCSI host the IO completed on (NULL for a NVME IO)
3752  *
3753  * The routine adjusts the various Burst and Bandwidth counters used in
3754  * Congestion management and E2E. If time is set to LPFC_CGN_NOT_SENT,
3755  * that means the IO was never issued to the HBA, so this routine is
3756  * just being called to cleanup the counter from a previous
3757  * lpfc_update_cmf_cmd call.
3758  */
3759 int
3760 lpfc_update_cmf_cmpl(struct lpfc_hba *phba,
3761                      uint64_t time, uint32_t size, struct Scsi_Host *shost)
3762 {
3763         struct lpfc_cgn_stat *cgs;
3764
3765         if (time != LPFC_CGN_NOT_SENT) {
3766                 /* lat is ns coming in, save latency in us */
3767                 if (time < 1000)
3768                         time = 1;
3769                 else
3770                         time = div_u64(time + 500, 1000); /* round it */
3771
3772                 cgs = per_cpu_ptr(phba->cmf_stat, raw_smp_processor_id());
3773                 atomic64_add(size, &cgs->rcv_bytes);
3774                 atomic64_add(time, &cgs->rx_latency);
3775                 atomic_inc(&cgs->rx_io_cnt);
3776         }
3777         return 0;
3778 }
3779
3780 /**
3781  * lpfc_update_cmf_cmd - Adjust CMF counters for IO submission
3782  * @phba: The HBA for which this call is being executed.
3783  * @size: The size of the IO that will be issued
3784  *
3785  * The routine adjusts the various Burst and Bandwidth counters used in
3786  * Congestion management and E2E.
3787  */
3788 int
3789 lpfc_update_cmf_cmd(struct lpfc_hba *phba, uint32_t size)
3790 {
3791         uint64_t total;
3792         struct lpfc_cgn_stat *cgs;
3793         int cpu;
3794
3795         /* At this point we are either LPFC_CFG_MANAGED or LPFC_CFG_MONITOR */
3796         if (phba->cmf_active_mode == LPFC_CFG_MANAGED &&
3797             phba->cmf_max_bytes_per_interval) {
3798                 total = 0;
3799                 for_each_present_cpu(cpu) {
3800                         cgs = per_cpu_ptr(phba->cmf_stat, cpu);
3801                         total += atomic64_read(&cgs->total_bytes);
3802                 }
3803                 if (total >= phba->cmf_max_bytes_per_interval) {
3804                         if (!atomic_xchg(&phba->cmf_bw_wait, 1)) {
3805                                 lpfc_block_requests(phba);
3806                                 phba->cmf_last_ts =
3807                                         lpfc_calc_cmf_latency(phba);
3808                         }
3809                         atomic_inc(&phba->cmf_busy);
3810                         return -EBUSY;
3811                 }
3812                 if (size > atomic_read(&phba->rx_max_read_cnt))
3813                         atomic_set(&phba->rx_max_read_cnt, size);
3814         }
3815
3816         cgs = per_cpu_ptr(phba->cmf_stat, raw_smp_processor_id());
3817         atomic64_add(size, &cgs->total_bytes);
3818         return 0;
3819 }
3820
3821 /**
3822  * lpfc_handle_fcp_err - FCP response handler
3823  * @vport: The virtual port for which this call is being executed.
3824  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
3825  * @fcpi_parm: FCP Initiator parameter.
3826  *
3827  * This routine is called to process response IOCB with status field
3828  * IOSTAT_FCP_RSP_ERROR. This routine sets result field of scsi command
3829  * based upon SCSI and FCP error.
3830  **/
3831 static void
3832 lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd,
3833                     uint32_t fcpi_parm)
3834 {
3835         struct scsi_cmnd *cmnd = lpfc_cmd->pCmd;
3836         struct fcp_cmnd *fcpcmd = lpfc_cmd->fcp_cmnd;
3837         struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp;
3838         uint32_t resp_info = fcprsp->rspStatus2;
3839         uint32_t scsi_status = fcprsp->rspStatus3;
3840         uint32_t *lp;
3841         uint32_t host_status = DID_OK;
3842         uint32_t rsplen = 0;
3843         uint32_t fcpDl;
3844         uint32_t logit = LOG_FCP | LOG_FCP_ERROR;
3845
3846
3847         /*
3848          *  If this is a task management command, there is no
3849          *  scsi packet associated with this lpfc_cmd.  The driver
3850          *  consumes it.
3851          */
3852         if (fcpcmd->fcpCntl2) {
3853                 scsi_status = 0;
3854                 goto out;
3855         }
3856
3857         if (resp_info & RSP_LEN_VALID) {
3858                 rsplen = be32_to_cpu(fcprsp->rspRspLen);
3859                 if (rsplen != 0 && rsplen != 4 && rsplen != 8) {
3860                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
3861                                          "2719 Invalid response length: "
3862                                          "tgt x%x lun x%llx cmnd x%x rsplen "
3863                                          "x%x\n", cmnd->device->id,
3864                                          cmnd->device->lun, cmnd->cmnd[0],
3865                                          rsplen);
3866                         host_status = DID_ERROR;
3867                         goto out;
3868                 }
3869                 if (fcprsp->rspInfo3 != RSP_NO_FAILURE) {
3870                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
3871                                  "2757 Protocol failure detected during "
3872                                  "processing of FCP I/O op: "
3873                                  "tgt x%x lun x%llx cmnd x%x rspInfo3 x%x\n",
3874                                  cmnd->device->id,
3875                                  cmnd->device->lun, cmnd->cmnd[0],
3876                                  fcprsp->rspInfo3);
3877                         host_status = DID_ERROR;
3878                         goto out;
3879                 }
3880         }
3881
3882         if ((resp_info & SNS_LEN_VALID) && fcprsp->rspSnsLen) {
3883                 uint32_t snslen = be32_to_cpu(fcprsp->rspSnsLen);
3884                 if (snslen > SCSI_SENSE_BUFFERSIZE)
3885                         snslen = SCSI_SENSE_BUFFERSIZE;
3886
3887                 if (resp_info & RSP_LEN_VALID)
3888                   rsplen = be32_to_cpu(fcprsp->rspRspLen);
3889                 memcpy(cmnd->sense_buffer, &fcprsp->rspInfo0 + rsplen, snslen);
3890         }
3891         lp = (uint32_t *)cmnd->sense_buffer;
3892
3893         /* special handling for under run conditions */
3894         if (!scsi_status && (resp_info & RESID_UNDER)) {
3895                 /* don't log under runs if fcp set... */
3896                 if (vport->cfg_log_verbose & LOG_FCP)
3897                         logit = LOG_FCP_ERROR;
3898                 /* unless operator says so */
3899                 if (vport->cfg_log_verbose & LOG_FCP_UNDER)
3900                         logit = LOG_FCP_UNDER;
3901         }
3902
3903         lpfc_printf_vlog(vport, KERN_WARNING, logit,
3904                          "9024 FCP command x%x failed: x%x SNS x%x x%x "
3905                          "Data: x%x x%x x%x x%x x%x\n",
3906                          cmnd->cmnd[0], scsi_status,
3907                          be32_to_cpu(*lp), be32_to_cpu(*(lp + 3)), resp_info,
3908                          be32_to_cpu(fcprsp->rspResId),
3909                          be32_to_cpu(fcprsp->rspSnsLen),
3910                          be32_to_cpu(fcprsp->rspRspLen),
3911                          fcprsp->rspInfo3);
3912
3913         scsi_set_resid(cmnd, 0);
3914         fcpDl = be32_to_cpu(fcpcmd->fcpDl);
3915         if (resp_info & RESID_UNDER) {
3916                 scsi_set_resid(cmnd, be32_to_cpu(fcprsp->rspResId));
3917
3918                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_UNDER,
3919                                  "9025 FCP Underrun, expected %d, "
3920                                  "residual %d Data: x%x x%x x%x\n",
3921                                  fcpDl,
3922                                  scsi_get_resid(cmnd), fcpi_parm, cmnd->cmnd[0],
3923                                  cmnd->underflow);
3924
3925                 /*
3926                  * If there is an under run, check if under run reported by
3927                  * storage array is same as the under run reported by HBA.
3928                  * If this is not same, there is a dropped frame.
3929                  */
3930                 if (fcpi_parm && (scsi_get_resid(cmnd) != fcpi_parm)) {
3931                         lpfc_printf_vlog(vport, KERN_WARNING,
3932                                          LOG_FCP | LOG_FCP_ERROR,
3933                                          "9026 FCP Read Check Error "
3934                                          "and Underrun Data: x%x x%x x%x x%x\n",
3935                                          fcpDl,
3936                                          scsi_get_resid(cmnd), fcpi_parm,
3937                                          cmnd->cmnd[0]);
3938                         scsi_set_resid(cmnd, scsi_bufflen(cmnd));
3939                         host_status = DID_ERROR;
3940                 }
3941                 /*
3942                  * The cmnd->underflow is the minimum number of bytes that must
3943                  * be transferred for this command.  Provided a sense condition
3944                  * is not present, make sure the actual amount transferred is at
3945                  * least the underflow value or fail.
3946                  */
3947                 if (!(resp_info & SNS_LEN_VALID) &&
3948                     (scsi_status == SAM_STAT_GOOD) &&
3949                     (scsi_bufflen(cmnd) - scsi_get_resid(cmnd)
3950                      < cmnd->underflow)) {
3951                         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
3952                                          "9027 FCP command x%x residual "
3953                                          "underrun converted to error "
3954                                          "Data: x%x x%x x%x\n",
3955                                          cmnd->cmnd[0], scsi_bufflen(cmnd),
3956                                          scsi_get_resid(cmnd), cmnd->underflow);
3957                         host_status = DID_ERROR;
3958                 }
3959         } else if (resp_info & RESID_OVER) {
3960                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
3961                                  "9028 FCP command x%x residual overrun error. "
3962                                  "Data: x%x x%x\n", cmnd->cmnd[0],
3963                                  scsi_bufflen(cmnd), scsi_get_resid(cmnd));
3964                 host_status = DID_ERROR;
3965
3966         /*
3967          * Check SLI validation that all the transfer was actually done
3968          * (fcpi_parm should be zero). Apply check only to reads.
3969          */
3970         } else if (fcpi_parm) {
3971                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP | LOG_FCP_ERROR,
3972                                  "9029 FCP %s Check Error Data: "
3973                                  "x%x x%x x%x x%x x%x\n",
3974                                  ((cmnd->sc_data_direction == DMA_FROM_DEVICE) ?
3975                                  "Read" : "Write"),
3976                                  fcpDl, be32_to_cpu(fcprsp->rspResId),
3977                                  fcpi_parm, cmnd->cmnd[0], scsi_status);
3978
3979                 /* There is some issue with the LPe12000 that causes it
3980                  * to miscalculate the fcpi_parm and falsely trip this
3981                  * recovery logic.  Detect this case and don't error when true.
3982                  */
3983                 if (fcpi_parm > fcpDl)
3984                         goto out;
3985
3986                 switch (scsi_status) {
3987                 case SAM_STAT_GOOD:
3988                 case SAM_STAT_CHECK_CONDITION:
3989                         /* Fabric dropped a data frame. Fail any successful
3990                          * command in which we detected dropped frames.
3991                          * A status of good or some check conditions could
3992                          * be considered a successful command.
3993                          */
3994                         host_status = DID_ERROR;
3995                         break;
3996                 }
3997                 scsi_set_resid(cmnd, scsi_bufflen(cmnd));
3998         }
3999
4000  out:
4001         cmnd->result = host_status << 16 | scsi_status;
4002         lpfc_send_scsi_error_event(vport->phba, vport, lpfc_cmd, fcpi_parm);
4003 }
4004
4005 /**
4006  * lpfc_fcp_io_cmd_wqe_cmpl - Complete a FCP IO
4007  * @phba: The hba for which this call is being executed.
4008  * @pwqeIn: The command WQE for the scsi cmnd.
4009  * @pwqeOut: Pointer to driver response WQE object.
4010  *
4011  * This routine assigns scsi command result by looking into response WQE
4012  * status field appropriately. This routine handles QUEUE FULL condition as
4013  * well by ramping down device queue depth.
4014  **/
4015 static void
4016 lpfc_fcp_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
4017                          struct lpfc_iocbq *pwqeOut)
4018 {
4019         struct lpfc_io_buf *lpfc_cmd = pwqeIn->io_buf;
4020         struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
4021         struct lpfc_vport *vport = pwqeIn->vport;
4022         struct lpfc_rport_data *rdata;
4023         struct lpfc_nodelist *ndlp;
4024         struct scsi_cmnd *cmd;
4025         unsigned long flags;
4026         struct lpfc_fast_path_event *fast_path_evt;
4027         struct Scsi_Host *shost;
4028         u32 logit = LOG_FCP;
4029         u32 idx;
4030         u32 lat;
4031         u8 wait_xb_clr = 0;
4032
4033         /* Sanity check on return of outstanding command */
4034         if (!lpfc_cmd) {
4035                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
4036                                  "9032 Null lpfc_cmd pointer. No "
4037                                  "release, skip completion\n");
4038                 return;
4039         }
4040
4041         rdata = lpfc_cmd->rdata;
4042         ndlp = rdata->pnode;
4043
4044         /* Sanity check on return of outstanding command */
4045         cmd = lpfc_cmd->pCmd;
4046         if (!cmd) {
4047                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
4048                                  "9042 I/O completion: Not an active IO\n");
4049                 lpfc_release_scsi_buf(phba, lpfc_cmd);
4050                 return;
4051         }
4052         /* Guard against abort handler being called at same time */
4053         spin_lock(&lpfc_cmd->buf_lock);
4054         idx = lpfc_cmd->cur_iocbq.hba_wqidx;
4055         if (phba->sli4_hba.hdwq)
4056                 phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++;
4057
4058 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4059         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_SCSI_IO))
4060                 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
4061 #endif
4062         shost = cmd->device->host;
4063
4064         lpfc_cmd->status = bf_get(lpfc_wcqe_c_status, wcqe);
4065         lpfc_cmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
4066
4067         lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY;
4068         if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
4069                 lpfc_cmd->flags |= LPFC_SBUF_XBUSY;
4070                 if (phba->cfg_fcp_wait_abts_rsp)
4071                         wait_xb_clr = 1;
4072         }
4073
4074 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4075         if (lpfc_cmd->prot_data_type) {
4076                 struct scsi_dif_tuple *src = NULL;
4077
4078                 src =  (struct scsi_dif_tuple *)lpfc_cmd->prot_data_segment;
4079                 /*
4080                  * Used to restore any changes to protection
4081                  * data for error injection.
4082                  */
4083                 switch (lpfc_cmd->prot_data_type) {
4084                 case LPFC_INJERR_REFTAG:
4085                         src->ref_tag =
4086                                 lpfc_cmd->prot_data;
4087                         break;
4088                 case LPFC_INJERR_APPTAG:
4089                         src->app_tag =
4090                                 (uint16_t)lpfc_cmd->prot_data;
4091                         break;
4092                 case LPFC_INJERR_GUARD:
4093                         src->guard_tag =
4094                                 (uint16_t)lpfc_cmd->prot_data;
4095                         break;
4096                 default:
4097                         break;
4098                 }
4099
4100                 lpfc_cmd->prot_data = 0;
4101                 lpfc_cmd->prot_data_type = 0;
4102                 lpfc_cmd->prot_data_segment = NULL;
4103         }
4104 #endif
4105         if (unlikely(lpfc_cmd->status)) {
4106                 if (lpfc_cmd->status == IOSTAT_FCP_RSP_ERROR &&
4107                     !lpfc_cmd->fcp_rsp->rspStatus3 &&
4108                     (lpfc_cmd->fcp_rsp->rspStatus2 & RESID_UNDER) &&
4109                     !(vport->cfg_log_verbose & LOG_FCP_UNDER))
4110                         logit = 0;
4111                 else
4112                         logit = LOG_FCP | LOG_FCP_UNDER;
4113                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4114                                  "9034 FCP cmd x%x failed <%d/%lld> "
4115                                  "status: x%x result: x%x "
4116                                  "sid: x%x did: x%x oxid: x%x "
4117                                  "Data: x%x x%x x%x\n",
4118                                  cmd->cmnd[0],
4119                                  cmd->device ? cmd->device->id : 0xffff,
4120                                  cmd->device ? cmd->device->lun : 0xffff,
4121                                  lpfc_cmd->status, lpfc_cmd->result,
4122                                  vport->fc_myDID,
4123                                  (ndlp) ? ndlp->nlp_DID : 0,
4124                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4125                                  wcqe->parameter, wcqe->total_data_placed,
4126                                  lpfc_cmd->cur_iocbq.iotag);
4127         }
4128
4129         switch (lpfc_cmd->status) {
4130         case CQE_STATUS_SUCCESS:
4131                 cmd->result = DID_OK << 16;
4132                 break;
4133         case CQE_STATUS_FCP_RSP_FAILURE:
4134                 lpfc_handle_fcp_err(vport, lpfc_cmd,
4135                                     pwqeIn->wqe.fcp_iread.total_xfer_len -
4136                                     wcqe->total_data_placed);
4137                 break;
4138         case CQE_STATUS_NPORT_BSY:
4139         case CQE_STATUS_FABRIC_BSY:
4140                 cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4141                 fast_path_evt = lpfc_alloc_fast_evt(phba);
4142                 if (!fast_path_evt)
4143                         break;
4144                 fast_path_evt->un.fabric_evt.event_type =
4145                         FC_REG_FABRIC_EVENT;
4146                 fast_path_evt->un.fabric_evt.subcategory =
4147                         (lpfc_cmd->status == IOSTAT_NPORT_BSY) ?
4148                         LPFC_EVENT_PORT_BUSY : LPFC_EVENT_FABRIC_BUSY;
4149                 if (ndlp) {
4150                         memcpy(&fast_path_evt->un.fabric_evt.wwpn,
4151                                &ndlp->nlp_portname,
4152                                 sizeof(struct lpfc_name));
4153                         memcpy(&fast_path_evt->un.fabric_evt.wwnn,
4154                                &ndlp->nlp_nodename,
4155                                 sizeof(struct lpfc_name));
4156                 }
4157                 fast_path_evt->vport = vport;
4158                 fast_path_evt->work_evt.evt =
4159                         LPFC_EVT_FASTPATH_MGMT_EVT;
4160                 spin_lock_irqsave(&phba->hbalock, flags);
4161                 list_add_tail(&fast_path_evt->work_evt.evt_listp,
4162                               &phba->work_list);
4163                 spin_unlock_irqrestore(&phba->hbalock, flags);
4164                 lpfc_worker_wake_up(phba);
4165                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4166                                  "9035 Fabric/Node busy FCP cmd x%x failed"
4167                                  " <%d/%lld> "
4168                                  "status: x%x result: x%x "
4169                                  "sid: x%x did: x%x oxid: x%x "
4170                                  "Data: x%x x%x x%x\n",
4171                                  cmd->cmnd[0],
4172                                  cmd->device ? cmd->device->id : 0xffff,
4173                                  cmd->device ? cmd->device->lun : 0xffff,
4174                                  lpfc_cmd->status, lpfc_cmd->result,
4175                                  vport->fc_myDID,
4176                                  (ndlp) ? ndlp->nlp_DID : 0,
4177                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4178                                  wcqe->parameter,
4179                                  wcqe->total_data_placed,
4180                                  lpfc_cmd->cur_iocbq.iocb.ulpIoTag);
4181                 break;
4182         case CQE_STATUS_DI_ERROR:
4183                 if (bf_get(lpfc_wcqe_c_bg_edir, wcqe))
4184                         lpfc_cmd->result = IOERR_RX_DMA_FAILED;
4185                 else
4186                         lpfc_cmd->result = IOERR_TX_DMA_FAILED;
4187                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP | LOG_BG,
4188                                  "9048 DI Error xri x%x status x%x DI ext "
4189                                  "status x%x data placed x%x\n",
4190                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4191                                  lpfc_cmd->status, wcqe->parameter,
4192                                  wcqe->total_data_placed);
4193                 if (scsi_get_prot_op(cmd) != SCSI_PROT_NORMAL) {
4194                         /* BG enabled cmd. Parse BG error */
4195                         lpfc_parse_bg_err(phba, lpfc_cmd, pwqeOut);
4196                         break;
4197                 }
4198                 cmd->result = DID_ERROR << 16;
4199                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
4200                                  "9040 DI Error on unprotected cmd\n");
4201                 break;
4202         case CQE_STATUS_REMOTE_STOP:
4203                 if (ndlp) {
4204                         /* This I/O was aborted by the target, we don't
4205                          * know the rxid and because we did not send the
4206                          * ABTS we cannot generate and RRQ.
4207                          */
4208                         lpfc_set_rrq_active(phba, ndlp,
4209                                             lpfc_cmd->cur_iocbq.sli4_lxritag,
4210                                             0, 0);
4211                 }
4212                 fallthrough;
4213         case CQE_STATUS_LOCAL_REJECT:
4214                 if (lpfc_cmd->result & IOERR_DRVR_MASK)
4215                         lpfc_cmd->status = IOSTAT_DRIVER_REJECT;
4216                 if (lpfc_cmd->result == IOERR_ELXSEC_KEY_UNWRAP_ERROR ||
4217                     lpfc_cmd->result ==
4218                     IOERR_ELXSEC_KEY_UNWRAP_COMPARE_ERROR ||
4219                     lpfc_cmd->result == IOERR_ELXSEC_CRYPTO_ERROR ||
4220                     lpfc_cmd->result ==
4221                     IOERR_ELXSEC_CRYPTO_COMPARE_ERROR) {
4222                         cmd->result = DID_NO_CONNECT << 16;
4223                         break;
4224                 }
4225                 if (lpfc_cmd->result == IOERR_INVALID_RPI ||
4226                     lpfc_cmd->result == IOERR_LINK_DOWN ||
4227                     lpfc_cmd->result == IOERR_NO_RESOURCES ||
4228                     lpfc_cmd->result == IOERR_ABORT_REQUESTED ||
4229                     lpfc_cmd->result == IOERR_RPI_SUSPENDED ||
4230                     lpfc_cmd->result == IOERR_SLER_CMD_RCV_FAILURE) {
4231                         cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4232                         break;
4233                 }
4234                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4235                                  "9036 Local Reject FCP cmd x%x failed"
4236                                  " <%d/%lld> "
4237                                  "status: x%x result: x%x "
4238                                  "sid: x%x did: x%x oxid: x%x "
4239                                  "Data: x%x x%x x%x\n",
4240                                  cmd->cmnd[0],
4241                                  cmd->device ? cmd->device->id : 0xffff,
4242                                  cmd->device ? cmd->device->lun : 0xffff,
4243                                  lpfc_cmd->status, lpfc_cmd->result,
4244                                  vport->fc_myDID,
4245                                  (ndlp) ? ndlp->nlp_DID : 0,
4246                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4247                                  wcqe->parameter,
4248                                  wcqe->total_data_placed,
4249                                  lpfc_cmd->cur_iocbq.iocb.ulpIoTag);
4250                 fallthrough;
4251         default:
4252                 cmd->result = DID_ERROR << 16;
4253                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
4254                                  "9037 FCP Completion Error: xri %x "
4255                                  "status x%x result x%x [x%x] "
4256                                  "placed x%x\n",
4257                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4258                                  lpfc_cmd->status, lpfc_cmd->result,
4259                                  wcqe->parameter,
4260                                  wcqe->total_data_placed);
4261         }
4262         if (cmd->result || lpfc_cmd->fcp_rsp->rspSnsLen) {
4263                 u32 *lp = (u32 *)cmd->sense_buffer;
4264
4265                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
4266                                  "9039 Iodone <%d/%llu> cmd x%px, error "
4267                                  "x%x SNS x%x x%x LBA x%llx Data: x%x x%x\n",
4268                                  cmd->device->id, cmd->device->lun, cmd,
4269                                  cmd->result, *lp, *(lp + 3),
4270                                  (cmd->device->sector_size) ?
4271                                  (u64)scsi_get_lba(cmd) : 0,
4272                                  cmd->retries, scsi_get_resid(cmd));
4273         }
4274
4275         if (vport->cfg_max_scsicmpl_time &&
4276             time_after(jiffies, lpfc_cmd->start_time +
4277             msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
4278                 spin_lock_irqsave(shost->host_lock, flags);
4279                 if (ndlp) {
4280                         if (ndlp->cmd_qdepth >
4281                                 atomic_read(&ndlp->cmd_pending) &&
4282                                 (atomic_read(&ndlp->cmd_pending) >
4283                                 LPFC_MIN_TGT_QDEPTH) &&
4284                                 (cmd->cmnd[0] == READ_10 ||
4285                                 cmd->cmnd[0] == WRITE_10))
4286                                 ndlp->cmd_qdepth =
4287                                         atomic_read(&ndlp->cmd_pending);
4288
4289                         ndlp->last_change_time = jiffies;
4290                 }
4291                 spin_unlock_irqrestore(shost->host_lock, flags);
4292         }
4293         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
4294
4295 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4296         if (lpfc_cmd->ts_cmd_start) {
4297                 lpfc_cmd->ts_isr_cmpl = lpfc_cmd->cur_iocbq.isr_timestamp;
4298                 lpfc_cmd->ts_data_io = ktime_get_ns();
4299                 phba->ktime_last_cmd = lpfc_cmd->ts_data_io;
4300                 lpfc_io_ktime(phba, lpfc_cmd);
4301         }
4302 #endif
4303         if (likely(!wait_xb_clr))
4304                 lpfc_cmd->pCmd = NULL;
4305         spin_unlock(&lpfc_cmd->buf_lock);
4306
4307         /* Check if IO qualified for CMF */
4308         if (phba->cmf_active_mode != LPFC_CFG_OFF &&
4309             cmd->sc_data_direction == DMA_FROM_DEVICE &&
4310             (scsi_sg_count(cmd))) {
4311                 /* Used when calculating average latency */
4312                 lat = ktime_get_ns() - lpfc_cmd->rx_cmd_start;
4313                 lpfc_update_cmf_cmpl(phba, lat, scsi_bufflen(cmd), shost);
4314         }
4315
4316         if (wait_xb_clr)
4317                 goto out;
4318
4319         /* The sdev is not guaranteed to be valid post scsi_done upcall. */
4320         scsi_done(cmd);
4321
4322         /*
4323          * If there is an abort thread waiting for command completion
4324          * wake up the thread.
4325          */
4326         spin_lock(&lpfc_cmd->buf_lock);
4327         lpfc_cmd->cur_iocbq.cmd_flag &= ~LPFC_DRIVER_ABORTED;
4328         if (lpfc_cmd->waitq)
4329                 wake_up(lpfc_cmd->waitq);
4330         spin_unlock(&lpfc_cmd->buf_lock);
4331 out:
4332         lpfc_release_scsi_buf(phba, lpfc_cmd);
4333 }
4334
4335 /**
4336  * lpfc_scsi_cmd_iocb_cmpl - Scsi cmnd IOCB completion routine
4337  * @phba: The Hba for which this call is being executed.
4338  * @pIocbIn: The command IOCBQ for the scsi cmnd.
4339  * @pIocbOut: The response IOCBQ for the scsi cmnd.
4340  *
4341  * This routine assigns scsi command result by looking into response IOCB
4342  * status field appropriately. This routine handles QUEUE FULL condition as
4343  * well by ramping down device queue depth.
4344  **/
4345 static void
4346 lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
4347                         struct lpfc_iocbq *pIocbOut)
4348 {
4349         struct lpfc_io_buf *lpfc_cmd =
4350                 (struct lpfc_io_buf *) pIocbIn->io_buf;
4351         struct lpfc_vport      *vport = pIocbIn->vport;
4352         struct lpfc_rport_data *rdata = lpfc_cmd->rdata;
4353         struct lpfc_nodelist *pnode = rdata->pnode;
4354         struct scsi_cmnd *cmd;
4355         unsigned long flags;
4356         struct lpfc_fast_path_event *fast_path_evt;
4357         struct Scsi_Host *shost;
4358         int idx;
4359         uint32_t logit = LOG_FCP;
4360
4361         /* Guard against abort handler being called at same time */
4362         spin_lock(&lpfc_cmd->buf_lock);
4363
4364         /* Sanity check on return of outstanding command */
4365         cmd = lpfc_cmd->pCmd;
4366         if (!cmd || !phba) {
4367                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
4368                                  "2621 IO completion: Not an active IO\n");
4369                 spin_unlock(&lpfc_cmd->buf_lock);
4370                 return;
4371         }
4372
4373         idx = lpfc_cmd->cur_iocbq.hba_wqidx;
4374         if (phba->sli4_hba.hdwq)
4375                 phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++;
4376
4377 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4378         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_SCSI_IO))
4379                 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
4380 #endif
4381         shost = cmd->device->host;
4382
4383         lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK);
4384         lpfc_cmd->status = pIocbOut->iocb.ulpStatus;
4385         /* pick up SLI4 exchange busy status from HBA */
4386         lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY;
4387         if (pIocbOut->cmd_flag & LPFC_EXCHANGE_BUSY)
4388                 lpfc_cmd->flags |= LPFC_SBUF_XBUSY;
4389
4390 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4391         if (lpfc_cmd->prot_data_type) {
4392                 struct scsi_dif_tuple *src = NULL;
4393
4394                 src =  (struct scsi_dif_tuple *)lpfc_cmd->prot_data_segment;
4395                 /*
4396                  * Used to restore any changes to protection
4397                  * data for error injection.
4398                  */
4399                 switch (lpfc_cmd->prot_data_type) {
4400                 case LPFC_INJERR_REFTAG:
4401                         src->ref_tag =
4402                                 lpfc_cmd->prot_data;
4403                         break;
4404                 case LPFC_INJERR_APPTAG:
4405                         src->app_tag =
4406                                 (uint16_t)lpfc_cmd->prot_data;
4407                         break;
4408                 case LPFC_INJERR_GUARD:
4409                         src->guard_tag =
4410                                 (uint16_t)lpfc_cmd->prot_data;
4411                         break;
4412                 default:
4413                         break;
4414                 }
4415
4416                 lpfc_cmd->prot_data = 0;
4417                 lpfc_cmd->prot_data_type = 0;
4418                 lpfc_cmd->prot_data_segment = NULL;
4419         }
4420 #endif
4421
4422         if (unlikely(lpfc_cmd->status)) {
4423                 if (lpfc_cmd->status == IOSTAT_LOCAL_REJECT &&
4424                     (lpfc_cmd->result & IOERR_DRVR_MASK))
4425                         lpfc_cmd->status = IOSTAT_DRIVER_REJECT;
4426                 else if (lpfc_cmd->status >= IOSTAT_CNT)
4427                         lpfc_cmd->status = IOSTAT_DEFAULT;
4428                 if (lpfc_cmd->status == IOSTAT_FCP_RSP_ERROR &&
4429                     !lpfc_cmd->fcp_rsp->rspStatus3 &&
4430                     (lpfc_cmd->fcp_rsp->rspStatus2 & RESID_UNDER) &&
4431                     !(vport->cfg_log_verbose & LOG_FCP_UNDER))
4432                         logit = 0;
4433                 else
4434                         logit = LOG_FCP | LOG_FCP_UNDER;
4435                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4436                          "9030 FCP cmd x%x failed <%d/%lld> "
4437                          "status: x%x result: x%x "
4438                          "sid: x%x did: x%x oxid: x%x "
4439                          "Data: x%x x%x\n",
4440                          cmd->cmnd[0],
4441                          cmd->device ? cmd->device->id : 0xffff,
4442                          cmd->device ? cmd->device->lun : 0xffff,
4443                          lpfc_cmd->status, lpfc_cmd->result,
4444                          vport->fc_myDID,
4445                          (pnode) ? pnode->nlp_DID : 0,
4446                          phba->sli_rev == LPFC_SLI_REV4 ?
4447                              lpfc_cmd->cur_iocbq.sli4_xritag : 0xffff,
4448                          pIocbOut->iocb.ulpContext,
4449                          lpfc_cmd->cur_iocbq.iocb.ulpIoTag);
4450
4451                 switch (lpfc_cmd->status) {
4452                 case IOSTAT_FCP_RSP_ERROR:
4453                         /* Call FCP RSP handler to determine result */
4454                         lpfc_handle_fcp_err(vport, lpfc_cmd,
4455                                             pIocbOut->iocb.un.fcpi.fcpi_parm);
4456                         break;
4457                 case IOSTAT_NPORT_BSY:
4458                 case IOSTAT_FABRIC_BSY:
4459                         cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4460                         fast_path_evt = lpfc_alloc_fast_evt(phba);
4461                         if (!fast_path_evt)
4462                                 break;
4463                         fast_path_evt->un.fabric_evt.event_type =
4464                                 FC_REG_FABRIC_EVENT;
4465                         fast_path_evt->un.fabric_evt.subcategory =
4466                                 (lpfc_cmd->status == IOSTAT_NPORT_BSY) ?
4467                                 LPFC_EVENT_PORT_BUSY : LPFC_EVENT_FABRIC_BUSY;
4468                         if (pnode) {
4469                                 memcpy(&fast_path_evt->un.fabric_evt.wwpn,
4470                                         &pnode->nlp_portname,
4471                                         sizeof(struct lpfc_name));
4472                                 memcpy(&fast_path_evt->un.fabric_evt.wwnn,
4473                                         &pnode->nlp_nodename,
4474                                         sizeof(struct lpfc_name));
4475                         }
4476                         fast_path_evt->vport = vport;
4477                         fast_path_evt->work_evt.evt =
4478                                 LPFC_EVT_FASTPATH_MGMT_EVT;
4479                         spin_lock_irqsave(&phba->hbalock, flags);
4480                         list_add_tail(&fast_path_evt->work_evt.evt_listp,
4481                                 &phba->work_list);
4482                         spin_unlock_irqrestore(&phba->hbalock, flags);
4483                         lpfc_worker_wake_up(phba);
4484                         break;
4485                 case IOSTAT_LOCAL_REJECT:
4486                 case IOSTAT_REMOTE_STOP:
4487                         if (lpfc_cmd->result == IOERR_ELXSEC_KEY_UNWRAP_ERROR ||
4488                             lpfc_cmd->result ==
4489                                         IOERR_ELXSEC_KEY_UNWRAP_COMPARE_ERROR ||
4490                             lpfc_cmd->result == IOERR_ELXSEC_CRYPTO_ERROR ||
4491                             lpfc_cmd->result ==
4492                                         IOERR_ELXSEC_CRYPTO_COMPARE_ERROR) {
4493                                 cmd->result = DID_NO_CONNECT << 16;
4494                                 break;
4495                         }
4496                         if (lpfc_cmd->result == IOERR_INVALID_RPI ||
4497                             lpfc_cmd->result == IOERR_NO_RESOURCES ||
4498                             lpfc_cmd->result == IOERR_ABORT_REQUESTED ||
4499                             lpfc_cmd->result == IOERR_SLER_CMD_RCV_FAILURE) {
4500                                 cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4501                                 break;
4502                         }
4503                         if ((lpfc_cmd->result == IOERR_RX_DMA_FAILED ||
4504                              lpfc_cmd->result == IOERR_TX_DMA_FAILED) &&
4505                              pIocbOut->iocb.unsli3.sli3_bg.bgstat) {
4506                                 if (scsi_get_prot_op(cmd) != SCSI_PROT_NORMAL) {
4507                                         /*
4508                                          * This is a response for a BG enabled
4509                                          * cmd. Parse BG error
4510                                          */
4511                                         lpfc_parse_bg_err(phba, lpfc_cmd,
4512                                                         pIocbOut);
4513                                         break;
4514                                 } else {
4515                                         lpfc_printf_vlog(vport, KERN_WARNING,
4516                                                         LOG_BG,
4517                                                         "9031 non-zero BGSTAT "
4518                                                         "on unprotected cmd\n");
4519                                 }
4520                         }
4521                         if ((lpfc_cmd->status == IOSTAT_REMOTE_STOP)
4522                                 && (phba->sli_rev == LPFC_SLI_REV4)
4523                                 && pnode) {
4524                                 /* This IO was aborted by the target, we don't
4525                                  * know the rxid and because we did not send the
4526                                  * ABTS we cannot generate and RRQ.
4527                                  */
4528                                 lpfc_set_rrq_active(phba, pnode,
4529                                         lpfc_cmd->cur_iocbq.sli4_lxritag,
4530                                         0, 0);
4531                         }
4532                         fallthrough;
4533                 default:
4534                         cmd->result = DID_ERROR << 16;
4535                         break;
4536                 }
4537
4538                 if (!pnode || (pnode->nlp_state != NLP_STE_MAPPED_NODE))
4539                         cmd->result = DID_TRANSPORT_DISRUPTED << 16 |
4540                                       SAM_STAT_BUSY;
4541         } else
4542                 cmd->result = DID_OK << 16;
4543
4544         if (cmd->result || lpfc_cmd->fcp_rsp->rspSnsLen) {
4545                 uint32_t *lp = (uint32_t *)cmd->sense_buffer;
4546
4547                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
4548                                  "0710 Iodone <%d/%llu> cmd x%px, error "
4549                                  "x%x SNS x%x x%x Data: x%x x%x\n",
4550                                  cmd->device->id, cmd->device->lun, cmd,
4551                                  cmd->result, *lp, *(lp + 3), cmd->retries,
4552                                  scsi_get_resid(cmd));
4553         }
4554
4555         if (vport->cfg_max_scsicmpl_time &&
4556            time_after(jiffies, lpfc_cmd->start_time +
4557                 msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
4558                 spin_lock_irqsave(shost->host_lock, flags);
4559                 if (pnode) {
4560                         if (pnode->cmd_qdepth >
4561                                 atomic_read(&pnode->cmd_pending) &&
4562                                 (atomic_read(&pnode->cmd_pending) >
4563                                 LPFC_MIN_TGT_QDEPTH) &&
4564                                 ((cmd->cmnd[0] == READ_10) ||
4565                                 (cmd->cmnd[0] == WRITE_10)))
4566                                 pnode->cmd_qdepth =
4567                                         atomic_read(&pnode->cmd_pending);
4568
4569                         pnode->last_change_time = jiffies;
4570                 }
4571                 spin_unlock_irqrestore(shost->host_lock, flags);
4572         }
4573         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
4574
4575         lpfc_cmd->pCmd = NULL;
4576         spin_unlock(&lpfc_cmd->buf_lock);
4577
4578 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4579         if (lpfc_cmd->ts_cmd_start) {
4580                 lpfc_cmd->ts_isr_cmpl = pIocbIn->isr_timestamp;
4581                 lpfc_cmd->ts_data_io = ktime_get_ns();
4582                 phba->ktime_last_cmd = lpfc_cmd->ts_data_io;
4583                 lpfc_io_ktime(phba, lpfc_cmd);
4584         }
4585 #endif
4586
4587         /* The sdev is not guaranteed to be valid post scsi_done upcall. */
4588         scsi_done(cmd);
4589
4590         /*
4591          * If there is an abort thread waiting for command completion
4592          * wake up the thread.
4593          */
4594         spin_lock(&lpfc_cmd->buf_lock);
4595         lpfc_cmd->cur_iocbq.cmd_flag &= ~LPFC_DRIVER_ABORTED;
4596         if (lpfc_cmd->waitq)
4597                 wake_up(lpfc_cmd->waitq);
4598         spin_unlock(&lpfc_cmd->buf_lock);
4599
4600         lpfc_release_scsi_buf(phba, lpfc_cmd);
4601 }
4602
4603 /**
4604  * lpfc_scsi_prep_cmnd_buf_s3 - SLI-3 IOCB init for the IO
4605  * @vport: Pointer to vport object.
4606  * @lpfc_cmd: The scsi buffer which is going to be prep'ed.
4607  * @tmo: timeout value for the IO
4608  *
4609  * Based on the data-direction of the command, initialize IOCB
4610  * in the I/O buffer. Fill in the IOCB fields which are independent
4611  * of the scsi buffer
4612  *
4613  * RETURNS 0 - SUCCESS,
4614  **/
4615 static int lpfc_scsi_prep_cmnd_buf_s3(struct lpfc_vport *vport,
4616                                       struct lpfc_io_buf *lpfc_cmd,
4617                                       uint8_t tmo)
4618 {
4619         IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb;
4620         struct lpfc_iocbq *piocbq = &lpfc_cmd->cur_iocbq;
4621         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
4622         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
4623         struct lpfc_nodelist *pnode = lpfc_cmd->ndlp;
4624         int datadir = scsi_cmnd->sc_data_direction;
4625         u32 fcpdl;
4626
4627         piocbq->iocb.un.fcpi.fcpi_XRdy = 0;
4628
4629         /*
4630          * There are three possibilities here - use scatter-gather segment, use
4631          * the single mapping, or neither.  Start the lpfc command prep by
4632          * bumping the bpl beyond the fcp_cmnd and fcp_rsp regions to the first
4633          * data bde entry.
4634          */
4635         if (scsi_sg_count(scsi_cmnd)) {
4636                 if (datadir == DMA_TO_DEVICE) {
4637                         iocb_cmd->ulpCommand = CMD_FCP_IWRITE64_CR;
4638                         iocb_cmd->ulpPU = PARM_READ_CHECK;
4639                         if (vport->cfg_first_burst_size &&
4640                             (pnode->nlp_flag & NLP_FIRSTBURST)) {
4641                                 u32 xrdy_len;
4642
4643                                 fcpdl = scsi_bufflen(scsi_cmnd);
4644                                 xrdy_len = min(fcpdl,
4645                                                vport->cfg_first_burst_size);
4646                                 piocbq->iocb.un.fcpi.fcpi_XRdy = xrdy_len;
4647                         }
4648                         fcp_cmnd->fcpCntl3 = WRITE_DATA;
4649                 } else {
4650                         iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR;
4651                         iocb_cmd->ulpPU = PARM_READ_CHECK;
4652                         fcp_cmnd->fcpCntl3 = READ_DATA;
4653                 }
4654         } else {
4655                 iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR;
4656                 iocb_cmd->un.fcpi.fcpi_parm = 0;
4657                 iocb_cmd->ulpPU = 0;
4658                 fcp_cmnd->fcpCntl3 = 0;
4659         }
4660
4661         /*
4662          * Finish initializing those IOCB fields that are independent
4663          * of the scsi_cmnd request_buffer
4664          */
4665         piocbq->iocb.ulpContext = pnode->nlp_rpi;
4666         if (pnode->nlp_fcp_info & NLP_FCP_2_DEVICE)
4667                 piocbq->iocb.ulpFCP2Rcvy = 1;
4668         else
4669                 piocbq->iocb.ulpFCP2Rcvy = 0;
4670
4671         piocbq->iocb.ulpClass = (pnode->nlp_fcp_info & 0x0f);
4672         piocbq->io_buf  = lpfc_cmd;
4673         if (!piocbq->cmd_cmpl)
4674                 piocbq->cmd_cmpl = lpfc_scsi_cmd_iocb_cmpl;
4675         piocbq->iocb.ulpTimeout = tmo;
4676         piocbq->vport = vport;
4677         return 0;
4678 }
4679
4680 /**
4681  * lpfc_scsi_prep_cmnd_buf_s4 - SLI-4 WQE init for the IO
4682  * @vport: Pointer to vport object.
4683  * @lpfc_cmd: The scsi buffer which is going to be prep'ed.
4684  * @tmo: timeout value for the IO
4685  *
4686  * Based on the data-direction of the command copy WQE template
4687  * to I/O buffer WQE. Fill in the WQE fields which are independent
4688  * of the scsi buffer
4689  *
4690  * RETURNS 0 - SUCCESS,
4691  **/
4692 static int lpfc_scsi_prep_cmnd_buf_s4(struct lpfc_vport *vport,
4693                                       struct lpfc_io_buf *lpfc_cmd,
4694                                       uint8_t tmo)
4695 {
4696         struct lpfc_hba *phba = vport->phba;
4697         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
4698         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
4699         struct lpfc_sli4_hdw_queue *hdwq = NULL;
4700         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
4701         struct lpfc_nodelist *pnode = lpfc_cmd->ndlp;
4702         union lpfc_wqe128 *wqe = &pwqeq->wqe;
4703         u16 idx = lpfc_cmd->hdwq_no;
4704         int datadir = scsi_cmnd->sc_data_direction;
4705
4706         hdwq = &phba->sli4_hba.hdwq[idx];
4707
4708         /* Initialize 64 bytes only */
4709         memset(wqe, 0, sizeof(union lpfc_wqe128));
4710
4711         /*
4712          * There are three possibilities here - use scatter-gather segment, use
4713          * the single mapping, or neither.
4714          */
4715         if (scsi_sg_count(scsi_cmnd)) {
4716                 if (datadir == DMA_TO_DEVICE) {
4717                         /* From the iwrite template, initialize words 7 -  11 */
4718                         memcpy(&wqe->words[7],
4719                                &lpfc_iwrite_cmd_template.words[7],
4720                                sizeof(uint32_t) * 5);
4721
4722                         fcp_cmnd->fcpCntl3 = WRITE_DATA;
4723                         if (hdwq)
4724                                 hdwq->scsi_cstat.output_requests++;
4725                 } else {
4726                         /* From the iread template, initialize words 7 - 11 */
4727                         memcpy(&wqe->words[7],
4728                                &lpfc_iread_cmd_template.words[7],
4729                                sizeof(uint32_t) * 5);
4730
4731                         /* Word 7 */
4732                         bf_set(wqe_tmo, &wqe->fcp_iread.wqe_com, tmo);
4733
4734                         fcp_cmnd->fcpCntl3 = READ_DATA;
4735                         if (hdwq)
4736                                 hdwq->scsi_cstat.input_requests++;
4737
4738                         /* For a CMF Managed port, iod must be zero'ed */
4739                         if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
4740                                 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
4741                                        LPFC_WQE_IOD_NONE);
4742                 }
4743         } else {
4744                 /* From the icmnd template, initialize words 4 - 11 */
4745                 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
4746                        sizeof(uint32_t) * 8);
4747
4748                 /* Word 7 */
4749                 bf_set(wqe_tmo, &wqe->fcp_icmd.wqe_com, tmo);
4750
4751                 fcp_cmnd->fcpCntl3 = 0;
4752                 if (hdwq)
4753                         hdwq->scsi_cstat.control_requests++;
4754         }
4755
4756         /*
4757          * Finish initializing those WQE fields that are independent
4758          * of the request_buffer
4759          */
4760
4761          /* Word 3 */
4762         bf_set(payload_offset_len, &wqe->fcp_icmd,
4763                sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp));
4764
4765         /* Word 6 */
4766         bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
4767                phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
4768         bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
4769
4770         /* Word 7*/
4771         if (pnode->nlp_fcp_info & NLP_FCP_2_DEVICE)
4772                 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
4773
4774         bf_set(wqe_class, &wqe->generic.wqe_com,
4775                (pnode->nlp_fcp_info & 0x0f));
4776
4777          /* Word 8 */
4778         wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
4779
4780         /* Word 9 */
4781         bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
4782
4783         pwqeq->vport = vport;
4784         pwqeq->io_buf = lpfc_cmd;
4785         pwqeq->hba_wqidx = lpfc_cmd->hdwq_no;
4786         pwqeq->cmd_cmpl = lpfc_fcp_io_cmd_wqe_cmpl;
4787
4788         return 0;
4789 }
4790
4791 /**
4792  * lpfc_scsi_prep_cmnd - Wrapper func for convert scsi cmnd to FCP info unit
4793  * @vport: The virtual port for which this call is being executed.
4794  * @lpfc_cmd: The scsi command which needs to send.
4795  * @pnode: Pointer to lpfc_nodelist.
4796  *
4797  * This routine initializes fcp_cmnd and iocb data structure from scsi command
4798  * to transfer for device with SLI3 interface spec.
4799  **/
4800 static int
4801 lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd,
4802                     struct lpfc_nodelist *pnode)
4803 {
4804         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
4805         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
4806         u8 *ptr;
4807
4808         if (!pnode)
4809                 return 0;
4810
4811         lpfc_cmd->fcp_rsp->rspSnsLen = 0;
4812         /* clear task management bits */
4813         lpfc_cmd->fcp_cmnd->fcpCntl2 = 0;
4814
4815         int_to_scsilun(lpfc_cmd->pCmd->device->lun,
4816                        &lpfc_cmd->fcp_cmnd->fcp_lun);
4817
4818         ptr = &fcp_cmnd->fcpCdb[0];
4819         memcpy(ptr, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
4820         if (scsi_cmnd->cmd_len < LPFC_FCP_CDB_LEN) {
4821                 ptr += scsi_cmnd->cmd_len;
4822                 memset(ptr, 0, (LPFC_FCP_CDB_LEN - scsi_cmnd->cmd_len));
4823         }
4824
4825         fcp_cmnd->fcpCntl1 = SIMPLE_Q;
4826
4827         lpfc_scsi_prep_cmnd_buf(vport, lpfc_cmd, lpfc_cmd->timeout);
4828
4829         return 0;
4830 }
4831
4832 /**
4833  * lpfc_scsi_prep_task_mgmt_cmd_s3 - Convert SLI3 scsi TM cmd to FCP info unit
4834  * @vport: The virtual port for which this call is being executed.
4835  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
4836  * @lun: Logical unit number.
4837  * @task_mgmt_cmd: SCSI task management command.
4838  *
4839  * This routine creates FCP information unit corresponding to @task_mgmt_cmd
4840  * for device with SLI-3 interface spec.
4841  *
4842  * Return codes:
4843  *   0 - Error
4844  *   1 - Success
4845  **/
4846 static int
4847 lpfc_scsi_prep_task_mgmt_cmd_s3(struct lpfc_vport *vport,
4848                                 struct lpfc_io_buf *lpfc_cmd,
4849                                 u64 lun, u8 task_mgmt_cmd)
4850 {
4851         struct lpfc_iocbq *piocbq;
4852         IOCB_t *piocb;
4853         struct fcp_cmnd *fcp_cmnd;
4854         struct lpfc_rport_data *rdata = lpfc_cmd->rdata;
4855         struct lpfc_nodelist *ndlp = rdata->pnode;
4856
4857         if (!ndlp || ndlp->nlp_state != NLP_STE_MAPPED_NODE)
4858                 return 0;
4859
4860         piocbq = &(lpfc_cmd->cur_iocbq);
4861         piocbq->vport = vport;
4862
4863         piocb = &piocbq->iocb;
4864
4865         fcp_cmnd = lpfc_cmd->fcp_cmnd;
4866         /* Clear out any old data in the FCP command area */
4867         memset(fcp_cmnd, 0, sizeof(struct fcp_cmnd));
4868         int_to_scsilun(lun, &fcp_cmnd->fcp_lun);
4869         fcp_cmnd->fcpCntl2 = task_mgmt_cmd;
4870         if (!(vport->phba->sli3_options & LPFC_SLI3_BG_ENABLED))
4871                 lpfc_fcpcmd_to_iocb(piocb->unsli3.fcp_ext.icd, fcp_cmnd);
4872         piocb->ulpCommand = CMD_FCP_ICMND64_CR;
4873         piocb->ulpContext = ndlp->nlp_rpi;
4874         piocb->ulpFCP2Rcvy = (ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE) ? 1 : 0;
4875         piocb->ulpClass = (ndlp->nlp_fcp_info & 0x0f);
4876         piocb->ulpPU = 0;
4877         piocb->un.fcpi.fcpi_parm = 0;
4878
4879         /* ulpTimeout is only one byte */
4880         if (lpfc_cmd->timeout > 0xff) {
4881                 /*
4882                  * Do not timeout the command at the firmware level.
4883                  * The driver will provide the timeout mechanism.
4884                  */
4885                 piocb->ulpTimeout = 0;
4886         } else
4887                 piocb->ulpTimeout = lpfc_cmd->timeout;
4888
4889         return 1;
4890 }
4891
4892 /**
4893  * lpfc_scsi_prep_task_mgmt_cmd_s4 - Convert SLI4 scsi TM cmd to FCP info unit
4894  * @vport: The virtual port for which this call is being executed.
4895  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
4896  * @lun: Logical unit number.
4897  * @task_mgmt_cmd: SCSI task management command.
4898  *
4899  * This routine creates FCP information unit corresponding to @task_mgmt_cmd
4900  * for device with SLI-4 interface spec.
4901  *
4902  * Return codes:
4903  *   0 - Error
4904  *   1 - Success
4905  **/
4906 static int
4907 lpfc_scsi_prep_task_mgmt_cmd_s4(struct lpfc_vport *vport,
4908                                 struct lpfc_io_buf *lpfc_cmd,
4909                                 u64 lun, u8 task_mgmt_cmd)
4910 {
4911         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
4912         union lpfc_wqe128 *wqe = &pwqeq->wqe;
4913         struct fcp_cmnd *fcp_cmnd;
4914         struct lpfc_rport_data *rdata = lpfc_cmd->rdata;
4915         struct lpfc_nodelist *ndlp = rdata->pnode;
4916
4917         if (!ndlp || ndlp->nlp_state != NLP_STE_MAPPED_NODE)
4918                 return 0;
4919
4920         pwqeq->vport = vport;
4921         /* Initialize 64 bytes only */
4922         memset(wqe, 0, sizeof(union lpfc_wqe128));
4923
4924         /* From the icmnd template, initialize words 4 - 11 */
4925         memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
4926                sizeof(uint32_t) * 8);
4927
4928         fcp_cmnd = lpfc_cmd->fcp_cmnd;
4929         /* Clear out any old data in the FCP command area */
4930         memset(fcp_cmnd, 0, sizeof(struct fcp_cmnd));
4931         int_to_scsilun(lun, &fcp_cmnd->fcp_lun);
4932         fcp_cmnd->fcpCntl3 = 0;
4933         fcp_cmnd->fcpCntl2 = task_mgmt_cmd;
4934
4935         bf_set(payload_offset_len, &wqe->fcp_icmd,
4936                sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp));
4937         bf_set(cmd_buff_len, &wqe->fcp_icmd, 0);
4938         bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,  /* ulpContext */
4939                vport->phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
4940         bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com,
4941                ((ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE) ? 1 : 0));
4942         bf_set(wqe_class, &wqe->fcp_icmd.wqe_com,
4943                (ndlp->nlp_fcp_info & 0x0f));
4944
4945         /* ulpTimeout is only one byte */
4946         if (lpfc_cmd->timeout > 0xff) {
4947                 /*
4948                  * Do not timeout the command at the firmware level.
4949                  * The driver will provide the timeout mechanism.
4950                  */
4951                 bf_set(wqe_tmo, &wqe->fcp_icmd.wqe_com, 0);
4952         } else {
4953                 bf_set(wqe_tmo, &wqe->fcp_icmd.wqe_com, lpfc_cmd->timeout);
4954         }
4955
4956         lpfc_prep_embed_io(vport->phba, lpfc_cmd);
4957         bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
4958         wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
4959         bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
4960
4961         lpfc_sli4_set_rsp_sgl_last(vport->phba, lpfc_cmd);
4962
4963         return 1;
4964 }
4965
4966 /**
4967  * lpfc_scsi_api_table_setup - Set up scsi api function jump table
4968  * @phba: The hba struct for which this call is being executed.
4969  * @dev_grp: The HBA PCI-Device group number.
4970  *
4971  * This routine sets up the SCSI interface API function jump table in @phba
4972  * struct.
4973  * Returns: 0 - success, -ENODEV - failure.
4974  **/
4975 int
4976 lpfc_scsi_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
4977 {
4978
4979         phba->lpfc_scsi_unprep_dma_buf = lpfc_scsi_unprep_dma_buf;
4980
4981         switch (dev_grp) {
4982         case LPFC_PCI_DEV_LP:
4983                 phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s3;
4984                 phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s3;
4985                 phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s3;
4986                 phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s3;
4987                 phba->lpfc_scsi_prep_cmnd_buf = lpfc_scsi_prep_cmnd_buf_s3;
4988                 phba->lpfc_scsi_prep_task_mgmt_cmd =
4989                                         lpfc_scsi_prep_task_mgmt_cmd_s3;
4990                 break;
4991         case LPFC_PCI_DEV_OC:
4992                 phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s4;
4993                 phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s4;
4994                 phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s4;
4995                 phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s4;
4996                 phba->lpfc_scsi_prep_cmnd_buf = lpfc_scsi_prep_cmnd_buf_s4;
4997                 phba->lpfc_scsi_prep_task_mgmt_cmd =
4998                                         lpfc_scsi_prep_task_mgmt_cmd_s4;
4999                 break;
5000         default:
5001                 lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
5002                                 "1418 Invalid HBA PCI-device group: 0x%x\n",
5003                                 dev_grp);
5004                 return -ENODEV;
5005         }
5006         phba->lpfc_rampdown_queue_depth = lpfc_rampdown_queue_depth;
5007         return 0;
5008 }
5009
5010 /**
5011  * lpfc_tskmgmt_def_cmpl - IOCB completion routine for task management command
5012  * @phba: The Hba for which this call is being executed.
5013  * @cmdiocbq: Pointer to lpfc_iocbq data structure.
5014  * @rspiocbq: Pointer to lpfc_iocbq data structure.
5015  *
5016  * This routine is IOCB completion routine for device reset and target reset
5017  * routine. This routine release scsi buffer associated with lpfc_cmd.
5018  **/
5019 static void
5020 lpfc_tskmgmt_def_cmpl(struct lpfc_hba *phba,
5021                         struct lpfc_iocbq *cmdiocbq,
5022                         struct lpfc_iocbq *rspiocbq)
5023 {
5024         struct lpfc_io_buf *lpfc_cmd = cmdiocbq->io_buf;
5025         if (lpfc_cmd)
5026                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5027         return;
5028 }
5029
5030 /**
5031  * lpfc_check_pci_resettable - Walks list of devices on pci_dev's bus to check
5032  *                             if issuing a pci_bus_reset is possibly unsafe
5033  * @phba: lpfc_hba pointer.
5034  *
5035  * Description:
5036  * Walks the bus_list to ensure only PCI devices with Emulex
5037  * vendor id, device ids that support hot reset, and only one occurrence
5038  * of function 0.
5039  *
5040  * Returns:
5041  * -EBADSLT,  detected invalid device
5042  *      0,    successful
5043  */
5044 int
5045 lpfc_check_pci_resettable(struct lpfc_hba *phba)
5046 {
5047         const struct pci_dev *pdev = phba->pcidev;
5048         struct pci_dev *ptr = NULL;
5049         u8 counter = 0;
5050
5051         /* Walk the list of devices on the pci_dev's bus */
5052         list_for_each_entry(ptr, &pdev->bus->devices, bus_list) {
5053                 /* Check for Emulex Vendor ID */
5054                 if (ptr->vendor != PCI_VENDOR_ID_EMULEX) {
5055                         lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5056                                         "8346 Non-Emulex vendor found: "
5057                                         "0x%04x\n", ptr->vendor);
5058                         return -EBADSLT;
5059                 }
5060
5061                 /* Check for valid Emulex Device ID */
5062                 if (phba->sli_rev != LPFC_SLI_REV4 ||
5063                     phba->hba_flag & HBA_FCOE_MODE) {
5064                         lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5065                                         "8347 Incapable PCI reset device: "
5066                                         "0x%04x\n", ptr->device);
5067                         return -EBADSLT;
5068                 }
5069
5070                 /* Check for only one function 0 ID to ensure only one HBA on
5071                  * secondary bus
5072                  */
5073                 if (ptr->devfn == 0) {
5074                         if (++counter > 1) {
5075                                 lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5076                                                 "8348 More than one device on "
5077                                                 "secondary bus found\n");
5078                                 return -EBADSLT;
5079                         }
5080                 }
5081         }
5082
5083         return 0;
5084 }
5085
5086 /**
5087  * lpfc_info - Info entry point of scsi_host_template data structure
5088  * @host: The scsi host for which this call is being executed.
5089  *
5090  * This routine provides module information about hba.
5091  *
5092  * Reutrn code:
5093  *   Pointer to char - Success.
5094  **/
5095 const char *
5096 lpfc_info(struct Scsi_Host *host)
5097 {
5098         struct lpfc_vport *vport = (struct lpfc_vport *) host->hostdata;
5099         struct lpfc_hba   *phba = vport->phba;
5100         int link_speed = 0;
5101         static char lpfcinfobuf[384];
5102         char tmp[384] = {0};
5103
5104         memset(lpfcinfobuf, 0, sizeof(lpfcinfobuf));
5105         if (phba && phba->pcidev){
5106                 /* Model Description */
5107                 scnprintf(tmp, sizeof(tmp), phba->ModelDesc);
5108                 if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5109                     sizeof(lpfcinfobuf))
5110                         goto buffer_done;
5111
5112                 /* PCI Info */
5113                 scnprintf(tmp, sizeof(tmp),
5114                           " on PCI bus %02x device %02x irq %d",
5115                           phba->pcidev->bus->number, phba->pcidev->devfn,
5116                           phba->pcidev->irq);
5117                 if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5118                     sizeof(lpfcinfobuf))
5119                         goto buffer_done;
5120
5121                 /* Port Number */
5122                 if (phba->Port[0]) {
5123                         scnprintf(tmp, sizeof(tmp), " port %s", phba->Port);
5124                         if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5125                             sizeof(lpfcinfobuf))
5126                                 goto buffer_done;
5127                 }
5128
5129                 /* Link Speed */
5130                 link_speed = lpfc_sli_port_speed_get(phba);
5131                 if (link_speed != 0) {
5132                         scnprintf(tmp, sizeof(tmp),
5133                                   " Logical Link Speed: %d Mbps", link_speed);
5134                         if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5135                             sizeof(lpfcinfobuf))
5136                                 goto buffer_done;
5137                 }
5138
5139                 /* PCI resettable */
5140                 if (!lpfc_check_pci_resettable(phba)) {
5141                         scnprintf(tmp, sizeof(tmp), " PCI resettable");
5142                         strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf));
5143                 }
5144         }
5145
5146 buffer_done:
5147         return lpfcinfobuf;
5148 }
5149
5150 /**
5151  * lpfc_poll_rearm_timer - Routine to modify fcp_poll timer of hba
5152  * @phba: The Hba for which this call is being executed.
5153  *
5154  * This routine modifies fcp_poll_timer  field of @phba by cfg_poll_tmo.
5155  * The default value of cfg_poll_tmo is 10 milliseconds.
5156  **/
5157 static __inline__ void lpfc_poll_rearm_timer(struct lpfc_hba * phba)
5158 {
5159         unsigned long  poll_tmo_expires =
5160                 (jiffies + msecs_to_jiffies(phba->cfg_poll_tmo));
5161
5162         if (!list_empty(&phba->sli.sli3_ring[LPFC_FCP_RING].txcmplq))
5163                 mod_timer(&phba->fcp_poll_timer,
5164                           poll_tmo_expires);
5165 }
5166
5167 /**
5168  * lpfc_poll_start_timer - Routine to start fcp_poll_timer of HBA
5169  * @phba: The Hba for which this call is being executed.
5170  *
5171  * This routine starts the fcp_poll_timer of @phba.
5172  **/
5173 void lpfc_poll_start_timer(struct lpfc_hba * phba)
5174 {
5175         lpfc_poll_rearm_timer(phba);
5176 }
5177
5178 /**
5179  * lpfc_poll_timeout - Restart polling timer
5180  * @t: Timer construct where lpfc_hba data structure pointer is obtained.
5181  *
5182  * This routine restarts fcp_poll timer, when FCP ring  polling is enable
5183  * and FCP Ring interrupt is disable.
5184  **/
5185 void lpfc_poll_timeout(struct timer_list *t)
5186 {
5187         struct lpfc_hba *phba = from_timer(phba, t, fcp_poll_timer);
5188
5189         if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
5190                 lpfc_sli_handle_fast_ring_event(phba,
5191                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
5192
5193                 if (phba->cfg_poll & DISABLE_FCP_RING_INT)
5194                         lpfc_poll_rearm_timer(phba);
5195         }
5196 }
5197
5198 /*
5199  * lpfc_is_command_vm_io - get the UUID from blk cgroup
5200  * @cmd: Pointer to scsi_cmnd data structure
5201  * Returns UUID if present, otherwise NULL
5202  */
5203 static char *lpfc_is_command_vm_io(struct scsi_cmnd *cmd)
5204 {
5205         struct bio *bio = scsi_cmd_to_rq(cmd)->bio;
5206
5207         if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !bio)
5208                 return NULL;
5209         return blkcg_get_fc_appid(bio);
5210 }
5211
5212 /**
5213  * lpfc_queuecommand - scsi_host_template queuecommand entry point
5214  * @shost: kernel scsi host pointer.
5215  * @cmnd: Pointer to scsi_cmnd data structure.
5216  *
5217  * Driver registers this routine to scsi midlayer to submit a @cmd to process.
5218  * This routine prepares an IOCB from scsi command and provides to firmware.
5219  * The @done callback is invoked after driver finished processing the command.
5220  *
5221  * Return value :
5222  *   0 - Success
5223  *   SCSI_MLQUEUE_HOST_BUSY - Block all devices served by this host temporarily.
5224  **/
5225 static int
5226 lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
5227 {
5228         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
5229         struct lpfc_hba   *phba = vport->phba;
5230         struct lpfc_iocbq *cur_iocbq = NULL;
5231         struct lpfc_rport_data *rdata;
5232         struct lpfc_nodelist *ndlp;
5233         struct lpfc_io_buf *lpfc_cmd;
5234         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
5235         int err, idx;
5236         u8 *uuid = NULL;
5237         uint64_t start;
5238
5239         start = ktime_get_ns();
5240         rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
5241
5242         /* sanity check on references */
5243         if (unlikely(!rdata) || unlikely(!rport))
5244                 goto out_fail_command;
5245
5246         err = fc_remote_port_chkready(rport);
5247         if (err) {
5248                 cmnd->result = err;
5249                 goto out_fail_command;
5250         }
5251         ndlp = rdata->pnode;
5252
5253         if ((scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) &&
5254                 (!(phba->sli3_options & LPFC_SLI3_BG_ENABLED))) {
5255
5256                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5257                                 "9058 BLKGRD: ERROR: rcvd protected cmd:%02x"
5258                                 " op:%02x str=%s without registering for"
5259                                 " BlockGuard - Rejecting command\n",
5260                                 cmnd->cmnd[0], scsi_get_prot_op(cmnd),
5261                                 dif_op_str[scsi_get_prot_op(cmnd)]);
5262                 goto out_fail_command;
5263         }
5264
5265         /*
5266          * Catch race where our node has transitioned, but the
5267          * transport is still transitioning.
5268          */
5269         if (!ndlp)
5270                 goto out_tgt_busy1;
5271
5272         /* Check if IO qualifies for CMF */
5273         if (phba->cmf_active_mode != LPFC_CFG_OFF &&
5274             cmnd->sc_data_direction == DMA_FROM_DEVICE &&
5275             (scsi_sg_count(cmnd))) {
5276                 /* Latency start time saved in rx_cmd_start later in routine */
5277                 err = lpfc_update_cmf_cmd(phba, scsi_bufflen(cmnd));
5278                 if (err)
5279                         goto out_tgt_busy1;
5280         }
5281
5282         if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
5283                 if (atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) {
5284                         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_ERROR,
5285                                          "3377 Target Queue Full, scsi Id:%d "
5286                                          "Qdepth:%d Pending command:%d"
5287                                          " WWNN:%02x:%02x:%02x:%02x:"
5288                                          "%02x:%02x:%02x:%02x, "
5289                                          " WWPN:%02x:%02x:%02x:%02x:"
5290                                          "%02x:%02x:%02x:%02x",
5291                                          ndlp->nlp_sid, ndlp->cmd_qdepth,
5292                                          atomic_read(&ndlp->cmd_pending),
5293                                          ndlp->nlp_nodename.u.wwn[0],
5294                                          ndlp->nlp_nodename.u.wwn[1],
5295                                          ndlp->nlp_nodename.u.wwn[2],
5296                                          ndlp->nlp_nodename.u.wwn[3],
5297                                          ndlp->nlp_nodename.u.wwn[4],
5298                                          ndlp->nlp_nodename.u.wwn[5],
5299                                          ndlp->nlp_nodename.u.wwn[6],
5300                                          ndlp->nlp_nodename.u.wwn[7],
5301                                          ndlp->nlp_portname.u.wwn[0],
5302                                          ndlp->nlp_portname.u.wwn[1],
5303                                          ndlp->nlp_portname.u.wwn[2],
5304                                          ndlp->nlp_portname.u.wwn[3],
5305                                          ndlp->nlp_portname.u.wwn[4],
5306                                          ndlp->nlp_portname.u.wwn[5],
5307                                          ndlp->nlp_portname.u.wwn[6],
5308                                          ndlp->nlp_portname.u.wwn[7]);
5309                         goto out_tgt_busy2;
5310                 }
5311         }
5312
5313         lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp, cmnd);
5314         if (lpfc_cmd == NULL) {
5315                 lpfc_rampdown_queue_depth(phba);
5316
5317                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_ERROR,
5318                                  "0707 driver's buffer pool is empty, "
5319                                  "IO busied\n");
5320                 goto out_host_busy;
5321         }
5322         lpfc_cmd->rx_cmd_start = start;
5323
5324         cur_iocbq = &lpfc_cmd->cur_iocbq;
5325         /*
5326          * Store the midlayer's command structure for the completion phase
5327          * and complete the command initialization.
5328          */
5329         lpfc_cmd->pCmd  = cmnd;
5330         lpfc_cmd->rdata = rdata;
5331         lpfc_cmd->ndlp = ndlp;
5332         cur_iocbq->cmd_cmpl = NULL;
5333         cmnd->host_scribble = (unsigned char *)lpfc_cmd;
5334
5335         err = lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp);
5336         if (err)
5337                 goto out_host_busy_release_buf;
5338
5339         if (scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) {
5340                 if (vport->phba->cfg_enable_bg) {
5341                         lpfc_printf_vlog(vport,
5342                                          KERN_INFO, LOG_SCSI_CMD,
5343                                          "9033 BLKGRD: rcvd %s cmd:x%x "
5344                                          "reftag x%x cnt %u pt %x\n",
5345                                          dif_op_str[scsi_get_prot_op(cmnd)],
5346                                          cmnd->cmnd[0],
5347                                          scsi_prot_ref_tag(cmnd),
5348                                          scsi_logical_block_count(cmnd),
5349                                          (cmnd->cmnd[1]>>5));
5350                 }
5351                 err = lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd);
5352         } else {
5353                 if (vport->phba->cfg_enable_bg) {
5354                         lpfc_printf_vlog(vport,
5355                                          KERN_INFO, LOG_SCSI_CMD,
5356                                          "9038 BLKGRD: rcvd PROT_NORMAL cmd: "
5357                                          "x%x reftag x%x cnt %u pt %x\n",
5358                                          cmnd->cmnd[0],
5359                                          scsi_prot_ref_tag(cmnd),
5360                                          scsi_logical_block_count(cmnd),
5361                                          (cmnd->cmnd[1]>>5));
5362                 }
5363                 err = lpfc_scsi_prep_dma_buf(phba, lpfc_cmd);
5364         }
5365
5366         if (unlikely(err)) {
5367                 if (err == 2) {
5368                         cmnd->result = DID_ERROR << 16;
5369                         goto out_fail_command_release_buf;
5370                 }
5371                 goto out_host_busy_free_buf;
5372         }
5373
5374         /* check the necessary and sufficient condition to support VMID */
5375         if (lpfc_is_vmid_enabled(phba) &&
5376             (ndlp->vmid_support ||
5377              phba->pport->vmid_priority_tagging ==
5378              LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
5379                 /* is the I/O generated by a VM, get the associated virtual */
5380                 /* entity id */
5381                 uuid = lpfc_is_command_vm_io(cmnd);
5382
5383                 if (uuid) {
5384                         err = lpfc_vmid_get_appid(vport, uuid,
5385                                         cmnd->sc_data_direction,
5386                                         (union lpfc_vmid_io_tag *)
5387                                                 &cur_iocbq->vmid_tag);
5388                         if (!err)
5389                                 cur_iocbq->cmd_flag |= LPFC_IO_VMID;
5390                 }
5391         }
5392
5393 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
5394         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_SCSI_IO))
5395                 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
5396 #endif
5397         /* Issue I/O to adapter */
5398         err = lpfc_sli_issue_fcp_io(phba, LPFC_FCP_RING, cur_iocbq,
5399                                     SLI_IOCB_RET_IOCB);
5400 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
5401         if (start) {
5402                 lpfc_cmd->ts_cmd_start = start;
5403                 lpfc_cmd->ts_last_cmd = phba->ktime_last_cmd;
5404                 lpfc_cmd->ts_cmd_wqput = ktime_get_ns();
5405         } else {
5406                 lpfc_cmd->ts_cmd_start = 0;
5407         }
5408 #endif
5409         if (err) {
5410                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5411                                  "3376 FCP could not issue iocb err %x "
5412                                  "FCP cmd x%x <%d/%llu> "
5413                                  "sid: x%x did: x%x oxid: x%x "
5414                                  "Data: x%x x%x x%x x%x\n",
5415                                  err, cmnd->cmnd[0],
5416                                  cmnd->device ? cmnd->device->id : 0xffff,
5417                                  cmnd->device ? cmnd->device->lun : (u64)-1,
5418                                  vport->fc_myDID, ndlp->nlp_DID,
5419                                  phba->sli_rev == LPFC_SLI_REV4 ?
5420                                  cur_iocbq->sli4_xritag : 0xffff,
5421                                  phba->sli_rev == LPFC_SLI_REV4 ?
5422                                  phba->sli4_hba.rpi_ids[ndlp->nlp_rpi] :
5423                                  cur_iocbq->iocb.ulpContext,
5424                                  cur_iocbq->iotag,
5425                                  phba->sli_rev == LPFC_SLI_REV4 ?
5426                                  bf_get(wqe_tmo,
5427                                         &cur_iocbq->wqe.generic.wqe_com) :
5428                                  cur_iocbq->iocb.ulpTimeout,
5429                                  (uint32_t)(scsi_cmd_to_rq(cmnd)->timeout / 1000));
5430
5431                 goto out_host_busy_free_buf;
5432         }
5433
5434         if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
5435                 lpfc_sli_handle_fast_ring_event(phba,
5436                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
5437
5438                 if (phba->cfg_poll & DISABLE_FCP_RING_INT)
5439                         lpfc_poll_rearm_timer(phba);
5440         }
5441
5442         if (phba->cfg_xri_rebalancing)
5443                 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_cmd->hdwq_no);
5444
5445         return 0;
5446
5447  out_host_busy_free_buf:
5448         idx = lpfc_cmd->hdwq_no;
5449         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
5450         if (phba->sli4_hba.hdwq) {
5451                 switch (lpfc_cmd->fcp_cmnd->fcpCntl3) {
5452                 case WRITE_DATA:
5453                         phba->sli4_hba.hdwq[idx].scsi_cstat.output_requests--;
5454                         break;
5455                 case READ_DATA:
5456                         phba->sli4_hba.hdwq[idx].scsi_cstat.input_requests--;
5457                         break;
5458                 default:
5459                         phba->sli4_hba.hdwq[idx].scsi_cstat.control_requests--;
5460                 }
5461         }
5462  out_host_busy_release_buf:
5463         lpfc_release_scsi_buf(phba, lpfc_cmd);
5464  out_host_busy:
5465         lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, scsi_bufflen(cmnd),
5466                              shost);
5467         return SCSI_MLQUEUE_HOST_BUSY;
5468
5469  out_tgt_busy2:
5470         lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, scsi_bufflen(cmnd),
5471                              shost);
5472  out_tgt_busy1:
5473         return SCSI_MLQUEUE_TARGET_BUSY;
5474
5475  out_fail_command_release_buf:
5476         lpfc_release_scsi_buf(phba, lpfc_cmd);
5477         lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, scsi_bufflen(cmnd),
5478                              shost);
5479
5480  out_fail_command:
5481         scsi_done(cmnd);
5482         return 0;
5483 }
5484
5485 /*
5486  * lpfc_vmid_vport_cleanup - cleans up the resources associated with a vport
5487  * @vport: The virtual port for which this call is being executed.
5488  */
5489 void lpfc_vmid_vport_cleanup(struct lpfc_vport *vport)
5490 {
5491         u32 bucket;
5492         struct lpfc_vmid *cur;
5493
5494         if (vport->port_type == LPFC_PHYSICAL_PORT)
5495                 del_timer_sync(&vport->phba->inactive_vmid_poll);
5496
5497         kfree(vport->qfpa_res);
5498         kfree(vport->vmid_priority.vmid_range);
5499         kfree(vport->vmid);
5500
5501         if (!hash_empty(vport->hash_table))
5502                 hash_for_each(vport->hash_table, bucket, cur, hnode)
5503                         hash_del(&cur->hnode);
5504
5505         vport->qfpa_res = NULL;
5506         vport->vmid_priority.vmid_range = NULL;
5507         vport->vmid = NULL;
5508         vport->cur_vmid_cnt = 0;
5509 }
5510
5511 /**
5512  * lpfc_abort_handler - scsi_host_template eh_abort_handler entry point
5513  * @cmnd: Pointer to scsi_cmnd data structure.
5514  *
5515  * This routine aborts @cmnd pending in base driver.
5516  *
5517  * Return code :
5518  *   0x2003 - Error
5519  *   0x2002 - Success
5520  **/
5521 static int
5522 lpfc_abort_handler(struct scsi_cmnd *cmnd)
5523 {
5524         struct Scsi_Host  *shost = cmnd->device->host;
5525         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
5526         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
5527         struct lpfc_hba   *phba = vport->phba;
5528         struct lpfc_iocbq *iocb;
5529         struct lpfc_io_buf *lpfc_cmd;
5530         int ret = SUCCESS, status = 0;
5531         struct lpfc_sli_ring *pring_s4 = NULL;
5532         struct lpfc_sli_ring *pring = NULL;
5533         int ret_val;
5534         unsigned long flags;
5535         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
5536
5537         status = fc_block_rport(rport);
5538         if (status != 0 && status != SUCCESS)
5539                 return status;
5540
5541         lpfc_cmd = (struct lpfc_io_buf *)cmnd->host_scribble;
5542         if (!lpfc_cmd)
5543                 return ret;
5544
5545         /* Guard against IO completion being called at same time */
5546         spin_lock_irqsave(&lpfc_cmd->buf_lock, flags);
5547
5548         spin_lock(&phba->hbalock);
5549         /* driver queued commands are in process of being flushed */
5550         if (phba->hba_flag & HBA_IOQ_FLUSH) {
5551                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5552                         "3168 SCSI Layer abort requested I/O has been "
5553                         "flushed by LLD.\n");
5554                 ret = FAILED;
5555                 goto out_unlock_hba;
5556         }
5557
5558         if (!lpfc_cmd->pCmd) {
5559                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5560                          "2873 SCSI Layer I/O Abort Request IO CMPL Status "
5561                          "x%x ID %d LUN %llu\n",
5562                          SUCCESS, cmnd->device->id, cmnd->device->lun);
5563                 goto out_unlock_hba;
5564         }
5565
5566         iocb = &lpfc_cmd->cur_iocbq;
5567         if (phba->sli_rev == LPFC_SLI_REV4) {
5568                 pring_s4 = phba->sli4_hba.hdwq[iocb->hba_wqidx].io_wq->pring;
5569                 if (!pring_s4) {
5570                         ret = FAILED;
5571                         goto out_unlock_hba;
5572                 }
5573                 spin_lock(&pring_s4->ring_lock);
5574         }
5575         /* the command is in process of being cancelled */
5576         if (!(iocb->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
5577                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5578                         "3169 SCSI Layer abort requested I/O has been "
5579                         "cancelled by LLD.\n");
5580                 ret = FAILED;
5581                 goto out_unlock_ring;
5582         }
5583         /*
5584          * If pCmd field of the corresponding lpfc_io_buf structure
5585          * points to a different SCSI command, then the driver has
5586          * already completed this command, but the midlayer did not
5587          * see the completion before the eh fired. Just return SUCCESS.
5588          */
5589         if (lpfc_cmd->pCmd != cmnd) {
5590                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5591                         "3170 SCSI Layer abort requested I/O has been "
5592                         "completed by LLD.\n");
5593                 goto out_unlock_ring;
5594         }
5595
5596         WARN_ON(iocb->io_buf != lpfc_cmd);
5597
5598         /* abort issued in recovery is still in progress */
5599         if (iocb->cmd_flag & LPFC_DRIVER_ABORTED) {
5600                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5601                          "3389 SCSI Layer I/O Abort Request is pending\n");
5602                 if (phba->sli_rev == LPFC_SLI_REV4)
5603                         spin_unlock(&pring_s4->ring_lock);
5604                 spin_unlock(&phba->hbalock);
5605                 spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);
5606                 goto wait_for_cmpl;
5607         }
5608
5609         lpfc_cmd->waitq = &waitq;
5610         if (phba->sli_rev == LPFC_SLI_REV4) {
5611                 spin_unlock(&pring_s4->ring_lock);
5612                 ret_val = lpfc_sli4_issue_abort_iotag(phba, iocb,
5613                                                       lpfc_sli_abort_fcp_cmpl);
5614         } else {
5615                 pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
5616                 ret_val = lpfc_sli_issue_abort_iotag(phba, pring, iocb,
5617                                                      lpfc_sli_abort_fcp_cmpl);
5618         }
5619
5620         /* Make sure HBA is alive */
5621         lpfc_issue_hb_tmo(phba);
5622
5623         if (ret_val != IOCB_SUCCESS) {
5624                 /* Indicate the IO is not being aborted by the driver. */
5625                 lpfc_cmd->waitq = NULL;
5626                 ret = FAILED;
5627                 goto out_unlock_hba;
5628         }
5629
5630         /* no longer need the lock after this point */
5631         spin_unlock(&phba->hbalock);
5632         spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);
5633
5634         if (phba->cfg_poll & DISABLE_FCP_RING_INT)
5635                 lpfc_sli_handle_fast_ring_event(phba,
5636                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
5637
5638 wait_for_cmpl:
5639         /*
5640          * cmd_flag is set to LPFC_DRIVER_ABORTED before we wait
5641          * for abort to complete.
5642          */
5643         wait_event_timeout(waitq,
5644                           (lpfc_cmd->pCmd != cmnd),
5645                            msecs_to_jiffies(2*vport->cfg_devloss_tmo*1000));
5646
5647         spin_lock(&lpfc_cmd->buf_lock);
5648
5649         if (lpfc_cmd->pCmd == cmnd) {
5650                 ret = FAILED;
5651                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5652                                  "0748 abort handler timed out waiting "
5653                                  "for aborting I/O (xri:x%x) to complete: "
5654                                  "ret %#x, ID %d, LUN %llu\n",
5655                                  iocb->sli4_xritag, ret,
5656                                  cmnd->device->id, cmnd->device->lun);
5657         }
5658
5659         lpfc_cmd->waitq = NULL;
5660
5661         spin_unlock(&lpfc_cmd->buf_lock);
5662         goto out;
5663
5664 out_unlock_ring:
5665         if (phba->sli_rev == LPFC_SLI_REV4)
5666                 spin_unlock(&pring_s4->ring_lock);
5667 out_unlock_hba:
5668         spin_unlock(&phba->hbalock);
5669         spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);
5670 out:
5671         lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5672                          "0749 SCSI Layer I/O Abort Request Status x%x ID %d "
5673                          "LUN %llu\n", ret, cmnd->device->id,
5674                          cmnd->device->lun);
5675         return ret;
5676 }
5677
5678 static char *
5679 lpfc_taskmgmt_name(uint8_t task_mgmt_cmd)
5680 {
5681         switch (task_mgmt_cmd) {
5682         case FCP_ABORT_TASK_SET:
5683                 return "ABORT_TASK_SET";
5684         case FCP_CLEAR_TASK_SET:
5685                 return "FCP_CLEAR_TASK_SET";
5686         case FCP_BUS_RESET:
5687                 return "FCP_BUS_RESET";
5688         case FCP_LUN_RESET:
5689                 return "FCP_LUN_RESET";
5690         case FCP_TARGET_RESET:
5691                 return "FCP_TARGET_RESET";
5692         case FCP_CLEAR_ACA:
5693                 return "FCP_CLEAR_ACA";
5694         case FCP_TERMINATE_TASK:
5695                 return "FCP_TERMINATE_TASK";
5696         default:
5697                 return "unknown";
5698         }
5699 }
5700
5701
5702 /**
5703  * lpfc_check_fcp_rsp - check the returned fcp_rsp to see if task failed
5704  * @vport: The virtual port for which this call is being executed.
5705  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
5706  *
5707  * This routine checks the FCP RSP INFO to see if the tsk mgmt command succeded
5708  *
5709  * Return code :
5710  *   0x2003 - Error
5711  *   0x2002 - Success
5712  **/
5713 static int
5714 lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd)
5715 {
5716         struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp;
5717         uint32_t rsp_info;
5718         uint32_t rsp_len;
5719         uint8_t  rsp_info_code;
5720         int ret = FAILED;
5721
5722
5723         if (fcprsp == NULL)
5724                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5725                                  "0703 fcp_rsp is missing\n");
5726         else {
5727                 rsp_info = fcprsp->rspStatus2;
5728                 rsp_len = be32_to_cpu(fcprsp->rspRspLen);
5729                 rsp_info_code = fcprsp->rspInfo3;
5730
5731
5732                 lpfc_printf_vlog(vport, KERN_INFO,
5733                                  LOG_FCP,
5734                                  "0706 fcp_rsp valid 0x%x,"
5735                                  " rsp len=%d code 0x%x\n",
5736                                  rsp_info,
5737                                  rsp_len, rsp_info_code);
5738
5739                 /* If FCP_RSP_LEN_VALID bit is one, then the FCP_RSP_LEN
5740                  * field specifies the number of valid bytes of FCP_RSP_INFO.
5741                  * The FCP_RSP_LEN field shall be set to 0x04 or 0x08
5742                  */
5743                 if ((fcprsp->rspStatus2 & RSP_LEN_VALID) &&
5744                     ((rsp_len == 8) || (rsp_len == 4))) {
5745                         switch (rsp_info_code) {
5746                         case RSP_NO_FAILURE:
5747                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5748                                                  "0715 Task Mgmt No Failure\n");
5749                                 ret = SUCCESS;
5750                                 break;
5751                         case RSP_TM_NOT_SUPPORTED: /* TM rejected */
5752                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5753                                                  "0716 Task Mgmt Target "
5754                                                 "reject\n");
5755                                 break;
5756                         case RSP_TM_NOT_COMPLETED: /* TM failed */
5757                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5758                                                  "0717 Task Mgmt Target "
5759                                                 "failed TM\n");
5760                                 break;
5761                         case RSP_TM_INVALID_LU: /* TM to invalid LU! */
5762                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5763                                                  "0718 Task Mgmt to invalid "
5764                                                 "LUN\n");
5765                                 break;
5766                         }
5767                 }
5768         }
5769         return ret;
5770 }
5771
5772
5773 /**
5774  * lpfc_send_taskmgmt - Generic SCSI Task Mgmt Handler
5775  * @vport: The virtual port for which this call is being executed.
5776  * @rport: Pointer to remote port
5777  * @tgt_id: Target ID of remote device.
5778  * @lun_id: Lun number for the TMF
5779  * @task_mgmt_cmd: type of TMF to send
5780  *
5781  * This routine builds and sends a TMF (SCSI Task Mgmt Function) to
5782  * a remote port.
5783  *
5784  * Return Code:
5785  *   0x2003 - Error
5786  *   0x2002 - Success.
5787  **/
5788 static int
5789 lpfc_send_taskmgmt(struct lpfc_vport *vport, struct fc_rport *rport,
5790                    unsigned int tgt_id, uint64_t lun_id,
5791                    uint8_t task_mgmt_cmd)
5792 {
5793         struct lpfc_hba   *phba = vport->phba;
5794         struct lpfc_io_buf *lpfc_cmd;
5795         struct lpfc_iocbq *iocbq;
5796         struct lpfc_iocbq *iocbqrsp;
5797         struct lpfc_rport_data *rdata;
5798         struct lpfc_nodelist *pnode;
5799         int ret;
5800         int status;
5801
5802         rdata = rport->dd_data;
5803         if (!rdata || !rdata->pnode)
5804                 return FAILED;
5805         pnode = rdata->pnode;
5806
5807         lpfc_cmd = lpfc_get_scsi_buf(phba, rdata->pnode, NULL);
5808         if (lpfc_cmd == NULL)
5809                 return FAILED;
5810         lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo;
5811         lpfc_cmd->rdata = rdata;
5812         lpfc_cmd->pCmd = NULL;
5813         lpfc_cmd->ndlp = pnode;
5814
5815         status = phba->lpfc_scsi_prep_task_mgmt_cmd(vport, lpfc_cmd, lun_id,
5816                                                     task_mgmt_cmd);
5817         if (!status) {
5818                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5819                 return FAILED;
5820         }
5821
5822         iocbq = &lpfc_cmd->cur_iocbq;
5823         iocbqrsp = lpfc_sli_get_iocbq(phba);
5824         if (iocbqrsp == NULL) {
5825                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5826                 return FAILED;
5827         }
5828         iocbq->cmd_cmpl = lpfc_tskmgmt_def_cmpl;
5829         iocbq->vport = vport;
5830
5831         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5832                          "0702 Issue %s to TGT %d LUN %llu "
5833                          "rpi x%x nlp_flag x%x Data: x%x x%x\n",
5834                          lpfc_taskmgmt_name(task_mgmt_cmd), tgt_id, lun_id,
5835                          pnode->nlp_rpi, pnode->nlp_flag, iocbq->sli4_xritag,
5836                          iocbq->cmd_flag);
5837
5838         status = lpfc_sli_issue_iocb_wait(phba, LPFC_FCP_RING,
5839                                           iocbq, iocbqrsp, lpfc_cmd->timeout);
5840         if ((status != IOCB_SUCCESS) ||
5841             (get_job_ulpstatus(phba, iocbqrsp) != IOSTAT_SUCCESS)) {
5842                 if (status != IOCB_SUCCESS ||
5843                     get_job_ulpstatus(phba, iocbqrsp) != IOSTAT_FCP_RSP_ERROR)
5844                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5845                                          "0727 TMF %s to TGT %d LUN %llu "
5846                                          "failed (%d, %d) cmd_flag x%x\n",
5847                                          lpfc_taskmgmt_name(task_mgmt_cmd),
5848                                          tgt_id, lun_id,
5849                                          get_job_ulpstatus(phba, iocbqrsp),
5850                                          get_job_word4(phba, iocbqrsp),
5851                                          iocbq->cmd_flag);
5852                 /* if ulpStatus != IOCB_SUCCESS, then status == IOCB_SUCCESS */
5853                 if (status == IOCB_SUCCESS) {
5854                         if (get_job_ulpstatus(phba, iocbqrsp) ==
5855                             IOSTAT_FCP_RSP_ERROR)
5856                                 /* Something in the FCP_RSP was invalid.
5857                                  * Check conditions */
5858                                 ret = lpfc_check_fcp_rsp(vport, lpfc_cmd);
5859                         else
5860                                 ret = FAILED;
5861                 } else if ((status == IOCB_TIMEDOUT) ||
5862                            (status == IOCB_ABORTED)) {
5863                         ret = TIMEOUT_ERROR;
5864                 } else {
5865                         ret = FAILED;
5866                 }
5867         } else
5868                 ret = SUCCESS;
5869
5870         lpfc_sli_release_iocbq(phba, iocbqrsp);
5871
5872         if (status != IOCB_TIMEDOUT)
5873                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5874
5875         return ret;
5876 }
5877
5878 /**
5879  * lpfc_chk_tgt_mapped -
5880  * @vport: The virtual port to check on
5881  * @rport: Pointer to fc_rport data structure.
5882  *
5883  * This routine delays until the scsi target (aka rport) for the
5884  * command exists (is present and logged in) or we declare it non-existent.
5885  *
5886  * Return code :
5887  *  0x2003 - Error
5888  *  0x2002 - Success
5889  **/
5890 static int
5891 lpfc_chk_tgt_mapped(struct lpfc_vport *vport, struct fc_rport *rport)
5892 {
5893         struct lpfc_rport_data *rdata;
5894         struct lpfc_nodelist *pnode = NULL;
5895         unsigned long later;
5896
5897         rdata = rport->dd_data;
5898         if (!rdata) {
5899                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5900                         "0797 Tgt Map rport failure: rdata x%px\n", rdata);
5901                 return FAILED;
5902         }
5903         pnode = rdata->pnode;
5904
5905         /*
5906          * If target is not in a MAPPED state, delay until
5907          * target is rediscovered or devloss timeout expires.
5908          */
5909         later = msecs_to_jiffies(2 * vport->cfg_devloss_tmo * 1000) + jiffies;
5910         while (time_after(later, jiffies)) {
5911                 if (!pnode)
5912                         return FAILED;
5913                 if (pnode->nlp_state == NLP_STE_MAPPED_NODE)
5914                         return SUCCESS;
5915                 schedule_timeout_uninterruptible(msecs_to_jiffies(500));
5916                 rdata = rport->dd_data;
5917                 if (!rdata)
5918                         return FAILED;
5919                 pnode = rdata->pnode;
5920         }
5921         if (!pnode || (pnode->nlp_state != NLP_STE_MAPPED_NODE))
5922                 return FAILED;
5923         return SUCCESS;
5924 }
5925
5926 /**
5927  * lpfc_reset_flush_io_context -
5928  * @vport: The virtual port (scsi_host) for the flush context
5929  * @tgt_id: If aborting by Target contect - specifies the target id
5930  * @lun_id: If aborting by Lun context - specifies the lun id
5931  * @context: specifies the context level to flush at.
5932  *
5933  * After a reset condition via TMF, we need to flush orphaned i/o
5934  * contexts from the adapter. This routine aborts any contexts
5935  * outstanding, then waits for their completions. The wait is
5936  * bounded by devloss_tmo though.
5937  *
5938  * Return code :
5939  *  0x2003 - Error
5940  *  0x2002 - Success
5941  **/
5942 static int
5943 lpfc_reset_flush_io_context(struct lpfc_vport *vport, uint16_t tgt_id,
5944                         uint64_t lun_id, lpfc_ctx_cmd context)
5945 {
5946         struct lpfc_hba   *phba = vport->phba;
5947         unsigned long later;
5948         int cnt;
5949
5950         cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
5951         if (cnt)
5952                 lpfc_sli_abort_taskmgmt(vport,
5953                                         &phba->sli.sli3_ring[LPFC_FCP_RING],
5954                                         tgt_id, lun_id, context);
5955         later = msecs_to_jiffies(2 * vport->cfg_devloss_tmo * 1000) + jiffies;
5956         while (time_after(later, jiffies) && cnt) {
5957                 schedule_timeout_uninterruptible(msecs_to_jiffies(20));
5958                 cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
5959         }
5960         if (cnt) {
5961                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5962                         "0724 I/O flush failure for context %s : cnt x%x\n",
5963                         ((context == LPFC_CTX_LUN) ? "LUN" :
5964                          ((context == LPFC_CTX_TGT) ? "TGT" :
5965                           ((context == LPFC_CTX_HOST) ? "HOST" : "Unknown"))),
5966                         cnt);
5967                 return FAILED;
5968         }
5969         return SUCCESS;
5970 }
5971
5972 /**
5973  * lpfc_device_reset_handler - scsi_host_template eh_device_reset entry point
5974  * @cmnd: Pointer to scsi_cmnd data structure.
5975  *
5976  * This routine does a device reset by sending a LUN_RESET task management
5977  * command.
5978  *
5979  * Return code :
5980  *  0x2003 - Error
5981  *  0x2002 - Success
5982  **/
5983 static int
5984 lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
5985 {
5986         struct Scsi_Host  *shost = cmnd->device->host;
5987         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
5988         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
5989         struct lpfc_rport_data *rdata;
5990         struct lpfc_nodelist *pnode;
5991         unsigned tgt_id = cmnd->device->id;
5992         uint64_t lun_id = cmnd->device->lun;
5993         struct lpfc_scsi_event_header scsi_event;
5994         int status;
5995         u32 logit = LOG_FCP;
5996
5997         if (!rport)
5998                 return FAILED;
5999
6000         rdata = rport->dd_data;
6001         if (!rdata || !rdata->pnode) {
6002                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6003                                  "0798 Device Reset rdata failure: rdata x%px\n",
6004                                  rdata);
6005                 return FAILED;
6006         }
6007         pnode = rdata->pnode;
6008         status = fc_block_rport(rport);
6009         if (status != 0 && status != SUCCESS)
6010                 return status;
6011
6012         status = lpfc_chk_tgt_mapped(vport, rport);
6013         if (status == FAILED) {
6014                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6015                         "0721 Device Reset rport failure: rdata x%px\n", rdata);
6016                 return FAILED;
6017         }
6018
6019         scsi_event.event_type = FC_REG_SCSI_EVENT;
6020         scsi_event.subcategory = LPFC_EVENT_LUNRESET;
6021         scsi_event.lun = lun_id;
6022         memcpy(scsi_event.wwpn, &pnode->nlp_portname, sizeof(struct lpfc_name));
6023         memcpy(scsi_event.wwnn, &pnode->nlp_nodename, sizeof(struct lpfc_name));
6024
6025         fc_host_post_vendor_event(shost, fc_get_event_number(),
6026                 sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
6027
6028         status = lpfc_send_taskmgmt(vport, rport, tgt_id, lun_id,
6029                                                 FCP_LUN_RESET);
6030         if (status != SUCCESS)
6031                 logit =  LOG_TRACE_EVENT;
6032
6033         lpfc_printf_vlog(vport, KERN_ERR, logit,
6034                          "0713 SCSI layer issued Device Reset (%d, %llu) "
6035                          "return x%x\n", tgt_id, lun_id, status);
6036
6037         /*
6038          * We have to clean up i/o as : they may be orphaned by the TMF;
6039          * or if the TMF failed, they may be in an indeterminate state.
6040          * So, continue on.
6041          * We will report success if all the i/o aborts successfully.
6042          */
6043         if (status == SUCCESS)
6044                 status = lpfc_reset_flush_io_context(vport, tgt_id, lun_id,
6045                                                 LPFC_CTX_LUN);
6046
6047         return status;
6048 }
6049
6050 /**
6051  * lpfc_target_reset_handler - scsi_host_template eh_target_reset entry point
6052  * @cmnd: Pointer to scsi_cmnd data structure.
6053  *
6054  * This routine does a target reset by sending a TARGET_RESET task management
6055  * command.
6056  *
6057  * Return code :
6058  *  0x2003 - Error
6059  *  0x2002 - Success
6060  **/
6061 static int
6062 lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
6063 {
6064         struct Scsi_Host  *shost = cmnd->device->host;
6065         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
6066         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
6067         struct lpfc_rport_data *rdata;
6068         struct lpfc_nodelist *pnode;
6069         unsigned tgt_id = cmnd->device->id;
6070         uint64_t lun_id = cmnd->device->lun;
6071         struct lpfc_scsi_event_header scsi_event;
6072         int status;
6073         u32 logit = LOG_FCP;
6074         u32 dev_loss_tmo = vport->cfg_devloss_tmo;
6075         unsigned long flags;
6076         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
6077
6078         if (!rport)
6079                 return FAILED;
6080
6081         rdata = rport->dd_data;
6082         if (!rdata || !rdata->pnode) {
6083                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6084                                  "0799 Target Reset rdata failure: rdata x%px\n",
6085                                  rdata);
6086                 return FAILED;
6087         }
6088         pnode = rdata->pnode;
6089         status = fc_block_rport(rport);
6090         if (status != 0 && status != SUCCESS)
6091                 return status;
6092
6093         status = lpfc_chk_tgt_mapped(vport, rport);
6094         if (status == FAILED) {
6095                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6096                         "0722 Target Reset rport failure: rdata x%px\n", rdata);
6097                 if (pnode) {
6098                         spin_lock_irqsave(&pnode->lock, flags);
6099                         pnode->nlp_flag &= ~NLP_NPR_ADISC;
6100                         pnode->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
6101                         spin_unlock_irqrestore(&pnode->lock, flags);
6102                 }
6103                 lpfc_reset_flush_io_context(vport, tgt_id, lun_id,
6104                                           LPFC_CTX_TGT);
6105                 return FAST_IO_FAIL;
6106         }
6107
6108         scsi_event.event_type = FC_REG_SCSI_EVENT;
6109         scsi_event.subcategory = LPFC_EVENT_TGTRESET;
6110         scsi_event.lun = 0;
6111         memcpy(scsi_event.wwpn, &pnode->nlp_portname, sizeof(struct lpfc_name));
6112         memcpy(scsi_event.wwnn, &pnode->nlp_nodename, sizeof(struct lpfc_name));
6113
6114         fc_host_post_vendor_event(shost, fc_get_event_number(),
6115                 sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
6116
6117         status = lpfc_send_taskmgmt(vport, rport, tgt_id, lun_id,
6118                                         FCP_TARGET_RESET);
6119         if (status != SUCCESS) {
6120                 logit = LOG_TRACE_EVENT;
6121
6122                 /* Issue LOGO, if no LOGO is outstanding */
6123                 spin_lock_irqsave(&pnode->lock, flags);
6124                 if (!(pnode->save_flags & NLP_WAIT_FOR_LOGO) &&
6125                     !pnode->logo_waitq) {
6126                         pnode->logo_waitq = &waitq;
6127                         pnode->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
6128                         pnode->nlp_flag |= NLP_ISSUE_LOGO;
6129                         pnode->save_flags |= NLP_WAIT_FOR_LOGO;
6130                         spin_unlock_irqrestore(&pnode->lock, flags);
6131                         lpfc_unreg_rpi(vport, pnode);
6132                         wait_event_timeout(waitq,
6133                                            (!(pnode->save_flags &
6134                                               NLP_WAIT_FOR_LOGO)),
6135                                            msecs_to_jiffies(dev_loss_tmo *
6136                                                             1000));
6137
6138                         if (pnode->save_flags & NLP_WAIT_FOR_LOGO) {
6139                                 lpfc_printf_vlog(vport, KERN_ERR, logit,
6140                                                  "0725 SCSI layer TGTRST "
6141                                                  "failed & LOGO TMO (%d, %llu) "
6142                                                  "return x%x\n",
6143                                                  tgt_id, lun_id, status);
6144                                 spin_lock_irqsave(&pnode->lock, flags);
6145                                 pnode->save_flags &= ~NLP_WAIT_FOR_LOGO;
6146                         } else {
6147                                 spin_lock_irqsave(&pnode->lock, flags);
6148                         }
6149                         pnode->logo_waitq = NULL;
6150                         spin_unlock_irqrestore(&pnode->lock, flags);
6151                         status = SUCCESS;
6152
6153                 } else {
6154                         spin_unlock_irqrestore(&pnode->lock, flags);
6155                         status = FAILED;
6156                 }
6157         }
6158
6159         lpfc_printf_vlog(vport, KERN_ERR, logit,
6160                          "0723 SCSI layer issued Target Reset (%d, %llu) "
6161                          "return x%x\n", tgt_id, lun_id, status);
6162
6163         /*
6164          * We have to clean up i/o as : they may be orphaned by the TMF;
6165          * or if the TMF failed, they may be in an indeterminate state.
6166          * So, continue on.
6167          * We will report success if all the i/o aborts successfully.
6168          */
6169         if (status == SUCCESS)
6170                 status = lpfc_reset_flush_io_context(vport, tgt_id, lun_id,
6171                                           LPFC_CTX_TGT);
6172         return status;
6173 }
6174
6175 /**
6176  * lpfc_host_reset_handler - scsi_host_template eh_host_reset_handler entry pt
6177  * @cmnd: Pointer to scsi_cmnd data structure.
6178  *
6179  * This routine does host reset to the adaptor port. It brings the HBA
6180  * offline, performs a board restart, and then brings the board back online.
6181  * The lpfc_offline calls lpfc_sli_hba_down which will abort and local
6182  * reject all outstanding SCSI commands to the host and error returned
6183  * back to SCSI mid-level. As this will be SCSI mid-level's last resort
6184  * of error handling, it will only return error if resetting of the adapter
6185  * is not successful; in all other cases, will return success.
6186  *
6187  * Return code :
6188  *  0x2003 - Error
6189  *  0x2002 - Success
6190  **/
6191 static int
6192 lpfc_host_reset_handler(struct scsi_cmnd *cmnd)
6193 {
6194         struct Scsi_Host *shost = cmnd->device->host;
6195         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
6196         struct lpfc_hba *phba = vport->phba;
6197         int rc, ret = SUCCESS;
6198
6199         lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
6200                          "3172 SCSI layer issued Host Reset Data:\n");
6201
6202         lpfc_offline_prep(phba, LPFC_MBX_WAIT);
6203         lpfc_offline(phba);
6204         rc = lpfc_sli_brdrestart(phba);
6205         if (rc)
6206                 goto error;
6207
6208         /* Wait for successful restart of adapter */
6209         if (phba->sli_rev < LPFC_SLI_REV4) {
6210                 rc = lpfc_sli_chipset_init(phba);
6211                 if (rc)
6212                         goto error;
6213         }
6214
6215         rc = lpfc_online(phba);
6216         if (rc)
6217                 goto error;
6218
6219         lpfc_unblock_mgmt_io(phba);
6220
6221         return ret;
6222 error:
6223         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6224                          "3323 Failed host reset\n");
6225         lpfc_unblock_mgmt_io(phba);
6226         return FAILED;
6227 }
6228
6229 /**
6230  * lpfc_slave_alloc - scsi_host_template slave_alloc entry point
6231  * @sdev: Pointer to scsi_device.
6232  *
6233  * This routine populates the cmds_per_lun count + 2 scsi_bufs into  this host's
6234  * globally available list of scsi buffers. This routine also makes sure scsi
6235  * buffer is not allocated more than HBA limit conveyed to midlayer. This list
6236  * of scsi buffer exists for the lifetime of the driver.
6237  *
6238  * Return codes:
6239  *   non-0 - Error
6240  *   0 - Success
6241  **/
6242 static int
6243 lpfc_slave_alloc(struct scsi_device *sdev)
6244 {
6245         struct lpfc_vport *vport = (struct lpfc_vport *) sdev->host->hostdata;
6246         struct lpfc_hba   *phba = vport->phba;
6247         struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
6248         uint32_t total = 0;
6249         uint32_t num_to_alloc = 0;
6250         int num_allocated = 0;
6251         uint32_t sdev_cnt;
6252         struct lpfc_device_data *device_data;
6253         unsigned long flags;
6254         struct lpfc_name target_wwpn;
6255
6256         if (!rport || fc_remote_port_chkready(rport))
6257                 return -ENXIO;
6258
6259         if (phba->cfg_fof) {
6260
6261                 /*
6262                  * Check to see if the device data structure for the lun
6263                  * exists.  If not, create one.
6264                  */
6265
6266                 u64_to_wwn(rport->port_name, target_wwpn.u.wwn);
6267                 spin_lock_irqsave(&phba->devicelock, flags);
6268                 device_data = __lpfc_get_device_data(phba,
6269                                                      &phba->luns,
6270                                                      &vport->fc_portname,
6271                                                      &target_wwpn,
6272                                                      sdev->lun);
6273                 if (!device_data) {
6274                         spin_unlock_irqrestore(&phba->devicelock, flags);
6275                         device_data = lpfc_create_device_data(phba,
6276                                                         &vport->fc_portname,
6277                                                         &target_wwpn,
6278                                                         sdev->lun,
6279                                                         phba->cfg_XLanePriority,
6280                                                         true);
6281                         if (!device_data)
6282                                 return -ENOMEM;
6283                         spin_lock_irqsave(&phba->devicelock, flags);
6284                         list_add_tail(&device_data->listentry, &phba->luns);
6285                 }
6286                 device_data->rport_data = rport->dd_data;
6287                 device_data->available = true;
6288                 spin_unlock_irqrestore(&phba->devicelock, flags);
6289                 sdev->hostdata = device_data;
6290         } else {
6291                 sdev->hostdata = rport->dd_data;
6292         }
6293         sdev_cnt = atomic_inc_return(&phba->sdev_cnt);
6294
6295         /* For SLI4, all IO buffers are pre-allocated */
6296         if (phba->sli_rev == LPFC_SLI_REV4)
6297                 return 0;
6298
6299         /* This code path is now ONLY for SLI3 adapters */
6300
6301         /*
6302          * Populate the cmds_per_lun count scsi_bufs into this host's globally
6303          * available list of scsi buffers.  Don't allocate more than the
6304          * HBA limit conveyed to the midlayer via the host structure.  The
6305          * formula accounts for the lun_queue_depth + error handlers + 1
6306          * extra.  This list of scsi bufs exists for the lifetime of the driver.
6307          */
6308         total = phba->total_scsi_bufs;
6309         num_to_alloc = vport->cfg_lun_queue_depth + 2;
6310
6311         /* If allocated buffers are enough do nothing */
6312         if ((sdev_cnt * (vport->cfg_lun_queue_depth + 2)) < total)
6313                 return 0;
6314
6315         /* Allow some exchanges to be available always to complete discovery */
6316         if (total >= phba->cfg_hba_queue_depth - LPFC_DISC_IOCB_BUFF_COUNT ) {
6317                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
6318                                  "0704 At limitation of %d preallocated "
6319                                  "command buffers\n", total);
6320                 return 0;
6321         /* Allow some exchanges to be available always to complete discovery */
6322         } else if (total + num_to_alloc >
6323                 phba->cfg_hba_queue_depth - LPFC_DISC_IOCB_BUFF_COUNT ) {
6324                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
6325                                  "0705 Allocation request of %d "
6326                                  "command buffers will exceed max of %d.  "
6327                                  "Reducing allocation request to %d.\n",
6328                                  num_to_alloc, phba->cfg_hba_queue_depth,
6329                                  (phba->cfg_hba_queue_depth - total));
6330                 num_to_alloc = phba->cfg_hba_queue_depth - total;
6331         }
6332         num_allocated = lpfc_new_scsi_buf_s3(vport, num_to_alloc);
6333         if (num_to_alloc != num_allocated) {
6334                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6335                                          "0708 Allocation request of %d "
6336                                          "command buffers did not succeed.  "
6337                                          "Allocated %d buffers.\n",
6338                                          num_to_alloc, num_allocated);
6339         }
6340         if (num_allocated > 0)
6341                 phba->total_scsi_bufs += num_allocated;
6342         return 0;
6343 }
6344
6345 /**
6346  * lpfc_slave_configure - scsi_host_template slave_configure entry point
6347  * @sdev: Pointer to scsi_device.
6348  *
6349  * This routine configures following items
6350  *   - Tag command queuing support for @sdev if supported.
6351  *   - Enable SLI polling for fcp ring if ENABLE_FCP_RING_POLLING flag is set.
6352  *
6353  * Return codes:
6354  *   0 - Success
6355  **/
6356 static int
6357 lpfc_slave_configure(struct scsi_device *sdev)
6358 {
6359         struct lpfc_vport *vport = (struct lpfc_vport *) sdev->host->hostdata;
6360         struct lpfc_hba   *phba = vport->phba;
6361
6362         scsi_change_queue_depth(sdev, vport->cfg_lun_queue_depth);
6363
6364         if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
6365                 lpfc_sli_handle_fast_ring_event(phba,
6366                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
6367                 if (phba->cfg_poll & DISABLE_FCP_RING_INT)
6368                         lpfc_poll_rearm_timer(phba);
6369         }
6370
6371         return 0;
6372 }
6373
6374 /**
6375  * lpfc_slave_destroy - slave_destroy entry point of SHT data structure
6376  * @sdev: Pointer to scsi_device.
6377  *
6378  * This routine sets @sdev hostatdata filed to null.
6379  **/
6380 static void
6381 lpfc_slave_destroy(struct scsi_device *sdev)
6382 {
6383         struct lpfc_vport *vport = (struct lpfc_vport *) sdev->host->hostdata;
6384         struct lpfc_hba   *phba = vport->phba;
6385         unsigned long flags;
6386         struct lpfc_device_data *device_data = sdev->hostdata;
6387
6388         atomic_dec(&phba->sdev_cnt);
6389         if ((phba->cfg_fof) && (device_data)) {
6390                 spin_lock_irqsave(&phba->devicelock, flags);
6391                 device_data->available = false;
6392                 if (!device_data->oas_enabled)
6393                         lpfc_delete_device_data(phba, device_data);
6394                 spin_unlock_irqrestore(&phba->devicelock, flags);
6395         }
6396         sdev->hostdata = NULL;
6397         return;
6398 }
6399
6400 /**
6401  * lpfc_create_device_data - creates and initializes device data structure for OAS
6402  * @phba: Pointer to host bus adapter structure.
6403  * @vport_wwpn: Pointer to vport's wwpn information
6404  * @target_wwpn: Pointer to target's wwpn information
6405  * @lun: Lun on target
6406  * @pri: Priority
6407  * @atomic_create: Flag to indicate if memory should be allocated using the
6408  *                GFP_ATOMIC flag or not.
6409  *
6410  * This routine creates a device data structure which will contain identifying
6411  * information for the device (host wwpn, target wwpn, lun), state of OAS,
6412  * whether or not the corresponding lun is available by the system,
6413  * and pointer to the rport data.
6414  *
6415  * Return codes:
6416  *   NULL - Error
6417  *   Pointer to lpfc_device_data - Success
6418  **/
6419 struct lpfc_device_data*
6420 lpfc_create_device_data(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6421                         struct lpfc_name *target_wwpn, uint64_t lun,
6422                         uint32_t pri, bool atomic_create)
6423 {
6424
6425         struct lpfc_device_data *lun_info;
6426         int memory_flags;
6427
6428         if (unlikely(!phba) || !vport_wwpn || !target_wwpn  ||
6429             !(phba->cfg_fof))
6430                 return NULL;
6431
6432         /* Attempt to create the device data to contain lun info */
6433
6434         if (atomic_create)
6435                 memory_flags = GFP_ATOMIC;
6436         else
6437                 memory_flags = GFP_KERNEL;
6438         lun_info = mempool_alloc(phba->device_data_mem_pool, memory_flags);
6439         if (!lun_info)
6440                 return NULL;
6441         INIT_LIST_HEAD(&lun_info->listentry);
6442         lun_info->rport_data  = NULL;
6443         memcpy(&lun_info->device_id.vport_wwpn, vport_wwpn,
6444                sizeof(struct lpfc_name));
6445         memcpy(&lun_info->device_id.target_wwpn, target_wwpn,
6446                sizeof(struct lpfc_name));
6447         lun_info->device_id.lun = lun;
6448         lun_info->oas_enabled = false;
6449         lun_info->priority = pri;
6450         lun_info->available = false;
6451         return lun_info;
6452 }
6453
6454 /**
6455  * lpfc_delete_device_data - frees a device data structure for OAS
6456  * @phba: Pointer to host bus adapter structure.
6457  * @lun_info: Pointer to device data structure to free.
6458  *
6459  * This routine frees the previously allocated device data structure passed.
6460  *
6461  **/
6462 void
6463 lpfc_delete_device_data(struct lpfc_hba *phba,
6464                         struct lpfc_device_data *lun_info)
6465 {
6466
6467         if (unlikely(!phba) || !lun_info  ||
6468             !(phba->cfg_fof))
6469                 return;
6470
6471         if (!list_empty(&lun_info->listentry))
6472                 list_del(&lun_info->listentry);
6473         mempool_free(lun_info, phba->device_data_mem_pool);
6474         return;
6475 }
6476
6477 /**
6478  * __lpfc_get_device_data - returns the device data for the specified lun
6479  * @phba: Pointer to host bus adapter structure.
6480  * @list: Point to list to search.
6481  * @vport_wwpn: Pointer to vport's wwpn information
6482  * @target_wwpn: Pointer to target's wwpn information
6483  * @lun: Lun on target
6484  *
6485  * This routine searches the list passed for the specified lun's device data.
6486  * This function does not hold locks, it is the responsibility of the caller
6487  * to ensure the proper lock is held before calling the function.
6488  *
6489  * Return codes:
6490  *   NULL - Error
6491  *   Pointer to lpfc_device_data - Success
6492  **/
6493 struct lpfc_device_data*
6494 __lpfc_get_device_data(struct lpfc_hba *phba, struct list_head *list,
6495                        struct lpfc_name *vport_wwpn,
6496                        struct lpfc_name *target_wwpn, uint64_t lun)
6497 {
6498
6499         struct lpfc_device_data *lun_info;
6500
6501         if (unlikely(!phba) || !list || !vport_wwpn || !target_wwpn ||
6502             !phba->cfg_fof)
6503                 return NULL;
6504
6505         /* Check to see if the lun is already enabled for OAS. */
6506
6507         list_for_each_entry(lun_info, list, listentry) {
6508                 if ((memcmp(&lun_info->device_id.vport_wwpn, vport_wwpn,
6509                             sizeof(struct lpfc_name)) == 0) &&
6510                     (memcmp(&lun_info->device_id.target_wwpn, target_wwpn,
6511                             sizeof(struct lpfc_name)) == 0) &&
6512                     (lun_info->device_id.lun == lun))
6513                         return lun_info;
6514         }
6515
6516         return NULL;
6517 }
6518
6519 /**
6520  * lpfc_find_next_oas_lun - searches for the next oas lun
6521  * @phba: Pointer to host bus adapter structure.
6522  * @vport_wwpn: Pointer to vport's wwpn information
6523  * @target_wwpn: Pointer to target's wwpn information
6524  * @starting_lun: Pointer to the lun to start searching for
6525  * @found_vport_wwpn: Pointer to the found lun's vport wwpn information
6526  * @found_target_wwpn: Pointer to the found lun's target wwpn information
6527  * @found_lun: Pointer to the found lun.
6528  * @found_lun_status: Pointer to status of the found lun.
6529  * @found_lun_pri: Pointer to priority of the found lun.
6530  *
6531  * This routine searches the luns list for the specified lun
6532  * or the first lun for the vport/target.  If the vport wwpn contains
6533  * a zero value then a specific vport is not specified. In this case
6534  * any vport which contains the lun will be considered a match.  If the
6535  * target wwpn contains a zero value then a specific target is not specified.
6536  * In this case any target which contains the lun will be considered a
6537  * match.  If the lun is found, the lun, vport wwpn, target wwpn and lun status
6538  * are returned.  The function will also return the next lun if available.
6539  * If the next lun is not found, starting_lun parameter will be set to
6540  * NO_MORE_OAS_LUN.
6541  *
6542  * Return codes:
6543  *   non-0 - Error
6544  *   0 - Success
6545  **/
6546 bool
6547 lpfc_find_next_oas_lun(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6548                        struct lpfc_name *target_wwpn, uint64_t *starting_lun,
6549                        struct lpfc_name *found_vport_wwpn,
6550                        struct lpfc_name *found_target_wwpn,
6551                        uint64_t *found_lun,
6552                        uint32_t *found_lun_status,
6553                        uint32_t *found_lun_pri)
6554 {
6555
6556         unsigned long flags;
6557         struct lpfc_device_data *lun_info;
6558         struct lpfc_device_id *device_id;
6559         uint64_t lun;
6560         bool found = false;
6561
6562         if (unlikely(!phba) || !vport_wwpn || !target_wwpn ||
6563             !starting_lun || !found_vport_wwpn ||
6564             !found_target_wwpn || !found_lun || !found_lun_status ||
6565             (*starting_lun == NO_MORE_OAS_LUN) ||
6566             !phba->cfg_fof)
6567                 return false;
6568
6569         lun = *starting_lun;
6570         *found_lun = NO_MORE_OAS_LUN;
6571         *starting_lun = NO_MORE_OAS_LUN;
6572
6573         /* Search for lun or the lun closet in value */
6574
6575         spin_lock_irqsave(&phba->devicelock, flags);
6576         list_for_each_entry(lun_info, &phba->luns, listentry) {
6577                 if (((wwn_to_u64(vport_wwpn->u.wwn) == 0) ||
6578                      (memcmp(&lun_info->device_id.vport_wwpn, vport_wwpn,
6579                             sizeof(struct lpfc_name)) == 0)) &&
6580                     ((wwn_to_u64(target_wwpn->u.wwn) == 0) ||
6581                      (memcmp(&lun_info->device_id.target_wwpn, target_wwpn,
6582                             sizeof(struct lpfc_name)) == 0)) &&
6583                     (lun_info->oas_enabled)) {
6584                         device_id = &lun_info->device_id;
6585                         if ((!found) &&
6586                             ((lun == FIND_FIRST_OAS_LUN) ||
6587                              (device_id->lun == lun))) {
6588                                 *found_lun = device_id->lun;
6589                                 memcpy(found_vport_wwpn,
6590                                        &device_id->vport_wwpn,
6591                                        sizeof(struct lpfc_name));
6592                                 memcpy(found_target_wwpn,
6593                                        &device_id->target_wwpn,
6594                                        sizeof(struct lpfc_name));
6595                                 if (lun_info->available)
6596                                         *found_lun_status =
6597                                                 OAS_LUN_STATUS_EXISTS;
6598                                 else
6599                                         *found_lun_status = 0;
6600                                 *found_lun_pri = lun_info->priority;
6601                                 if (phba->cfg_oas_flags & OAS_FIND_ANY_VPORT)
6602                                         memset(vport_wwpn, 0x0,
6603                                                sizeof(struct lpfc_name));
6604                                 if (phba->cfg_oas_flags & OAS_FIND_ANY_TARGET)
6605                                         memset(target_wwpn, 0x0,
6606                                                sizeof(struct lpfc_name));
6607                                 found = true;
6608                         } else if (found) {
6609                                 *starting_lun = device_id->lun;
6610                                 memcpy(vport_wwpn, &device_id->vport_wwpn,
6611                                        sizeof(struct lpfc_name));
6612                                 memcpy(target_wwpn, &device_id->target_wwpn,
6613                                        sizeof(struct lpfc_name));
6614                                 break;
6615                         }
6616                 }
6617         }
6618         spin_unlock_irqrestore(&phba->devicelock, flags);
6619         return found;
6620 }
6621
6622 /**
6623  * lpfc_enable_oas_lun - enables a lun for OAS operations
6624  * @phba: Pointer to host bus adapter structure.
6625  * @vport_wwpn: Pointer to vport's wwpn information
6626  * @target_wwpn: Pointer to target's wwpn information
6627  * @lun: Lun
6628  * @pri: Priority
6629  *
6630  * This routine enables a lun for oas operations.  The routines does so by
6631  * doing the following :
6632  *
6633  *   1) Checks to see if the device data for the lun has been created.
6634  *   2) If found, sets the OAS enabled flag if not set and returns.
6635  *   3) Otherwise, creates a device data structure.
6636  *   4) If successfully created, indicates the device data is for an OAS lun,
6637  *   indicates the lun is not available and add to the list of luns.
6638  *
6639  * Return codes:
6640  *   false - Error
6641  *   true - Success
6642  **/
6643 bool
6644 lpfc_enable_oas_lun(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6645                     struct lpfc_name *target_wwpn, uint64_t lun, uint8_t pri)
6646 {
6647
6648         struct lpfc_device_data *lun_info;
6649         unsigned long flags;
6650
6651         if (unlikely(!phba) || !vport_wwpn || !target_wwpn ||
6652             !phba->cfg_fof)
6653                 return false;
6654
6655         spin_lock_irqsave(&phba->devicelock, flags);
6656
6657         /* Check to see if the device data for the lun has been created */
6658         lun_info = __lpfc_get_device_data(phba, &phba->luns, vport_wwpn,
6659                                           target_wwpn, lun);
6660         if (lun_info) {
6661                 if (!lun_info->oas_enabled)
6662                         lun_info->oas_enabled = true;
6663                 lun_info->priority = pri;
6664                 spin_unlock_irqrestore(&phba->devicelock, flags);
6665                 return true;
6666         }
6667
6668         /* Create an lun info structure and add to list of luns */
6669         lun_info = lpfc_create_device_data(phba, vport_wwpn, target_wwpn, lun,
6670                                            pri, true);
6671         if (lun_info) {
6672                 lun_info->oas_enabled = true;
6673                 lun_info->priority = pri;
6674                 lun_info->available = false;
6675                 list_add_tail(&lun_info->listentry, &phba->luns);
6676                 spin_unlock_irqrestore(&phba->devicelock, flags);
6677                 return true;
6678         }
6679         spin_unlock_irqrestore(&phba->devicelock, flags);
6680         return false;
6681 }
6682
6683 /**
6684  * lpfc_disable_oas_lun - disables a lun for OAS operations
6685  * @phba: Pointer to host bus adapter structure.
6686  * @vport_wwpn: Pointer to vport's wwpn information
6687  * @target_wwpn: Pointer to target's wwpn information
6688  * @lun: Lun
6689  * @pri: Priority
6690  *
6691  * This routine disables a lun for oas operations.  The routines does so by
6692  * doing the following :
6693  *
6694  *   1) Checks to see if the device data for the lun is created.
6695  *   2) If present, clears the flag indicating this lun is for OAS.
6696  *   3) If the lun is not available by the system, the device data is
6697  *   freed.
6698  *
6699  * Return codes:
6700  *   false - Error
6701  *   true - Success
6702  **/
6703 bool
6704 lpfc_disable_oas_lun(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6705                      struct lpfc_name *target_wwpn, uint64_t lun, uint8_t pri)
6706 {
6707
6708         struct lpfc_device_data *lun_info;
6709         unsigned long flags;
6710
6711         if (unlikely(!phba) || !vport_wwpn || !target_wwpn ||
6712             !phba->cfg_fof)
6713                 return false;
6714
6715         spin_lock_irqsave(&phba->devicelock, flags);
6716
6717         /* Check to see if the lun is available. */
6718         lun_info = __lpfc_get_device_data(phba,
6719                                           &phba->luns, vport_wwpn,
6720                                           target_wwpn, lun);
6721         if (lun_info) {
6722                 lun_info->oas_enabled = false;
6723                 lun_info->priority = pri;
6724                 if (!lun_info->available)
6725                         lpfc_delete_device_data(phba, lun_info);
6726                 spin_unlock_irqrestore(&phba->devicelock, flags);
6727                 return true;
6728         }
6729
6730         spin_unlock_irqrestore(&phba->devicelock, flags);
6731         return false;
6732 }
6733
6734 static int
6735 lpfc_no_command(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
6736 {
6737         return SCSI_MLQUEUE_HOST_BUSY;
6738 }
6739
6740 static int
6741 lpfc_no_slave(struct scsi_device *sdev)
6742 {
6743         return -ENODEV;
6744 }
6745
6746 struct scsi_host_template lpfc_template_nvme = {
6747         .module                 = THIS_MODULE,
6748         .name                   = LPFC_DRIVER_NAME,
6749         .proc_name              = LPFC_DRIVER_NAME,
6750         .info                   = lpfc_info,
6751         .queuecommand           = lpfc_no_command,
6752         .slave_alloc            = lpfc_no_slave,
6753         .slave_configure        = lpfc_no_slave,
6754         .scan_finished          = lpfc_scan_finished,
6755         .this_id                = -1,
6756         .sg_tablesize           = 1,
6757         .cmd_per_lun            = 1,
6758         .shost_groups           = lpfc_hba_groups,
6759         .max_sectors            = 0xFFFFFFFF,
6760         .vendor_id              = LPFC_NL_VENDOR_ID,
6761         .track_queue_depth      = 0,
6762 };
6763
6764 struct scsi_host_template lpfc_template = {
6765         .module                 = THIS_MODULE,
6766         .name                   = LPFC_DRIVER_NAME,
6767         .proc_name              = LPFC_DRIVER_NAME,
6768         .info                   = lpfc_info,
6769         .queuecommand           = lpfc_queuecommand,
6770         .eh_timed_out           = fc_eh_timed_out,
6771         .eh_should_retry_cmd    = fc_eh_should_retry_cmd,
6772         .eh_abort_handler       = lpfc_abort_handler,
6773         .eh_device_reset_handler = lpfc_device_reset_handler,
6774         .eh_target_reset_handler = lpfc_target_reset_handler,
6775         .eh_host_reset_handler  = lpfc_host_reset_handler,
6776         .slave_alloc            = lpfc_slave_alloc,
6777         .slave_configure        = lpfc_slave_configure,
6778         .slave_destroy          = lpfc_slave_destroy,
6779         .scan_finished          = lpfc_scan_finished,
6780         .this_id                = -1,
6781         .sg_tablesize           = LPFC_DEFAULT_SG_SEG_CNT,
6782         .cmd_per_lun            = LPFC_CMD_PER_LUN,
6783         .shost_groups           = lpfc_hba_groups,
6784         .max_sectors            = 0xFFFFFFFF,
6785         .vendor_id              = LPFC_NL_VENDOR_ID,
6786         .change_queue_depth     = scsi_change_queue_depth,
6787         .track_queue_depth      = 1,
6788 };
6789
6790 struct scsi_host_template lpfc_vport_template = {
6791         .module                 = THIS_MODULE,
6792         .name                   = LPFC_DRIVER_NAME,
6793         .proc_name              = LPFC_DRIVER_NAME,
6794         .info                   = lpfc_info,
6795         .queuecommand           = lpfc_queuecommand,
6796         .eh_timed_out           = fc_eh_timed_out,
6797         .eh_should_retry_cmd    = fc_eh_should_retry_cmd,
6798         .eh_abort_handler       = lpfc_abort_handler,
6799         .eh_device_reset_handler = lpfc_device_reset_handler,
6800         .eh_target_reset_handler = lpfc_target_reset_handler,
6801         .eh_bus_reset_handler   = NULL,
6802         .eh_host_reset_handler  = NULL,
6803         .slave_alloc            = lpfc_slave_alloc,
6804         .slave_configure        = lpfc_slave_configure,
6805         .slave_destroy          = lpfc_slave_destroy,
6806         .scan_finished          = lpfc_scan_finished,
6807         .this_id                = -1,
6808         .sg_tablesize           = LPFC_DEFAULT_SG_SEG_CNT,
6809         .cmd_per_lun            = LPFC_CMD_PER_LUN,
6810         .shost_groups           = lpfc_vport_groups,
6811         .max_sectors            = 0xFFFFFFFF,
6812         .vendor_id              = 0,
6813         .change_queue_depth     = scsi_change_queue_depth,
6814         .track_queue_depth      = 1,
6815 };