drivers/scsi/mpt3sas/mpt3sas_scsih.c

   1 /*
   2  * Scsi Host Layer for MPT (Message Passing Technology) based controllers
   3  *
   4  * This code is based on drivers/scsi/mpt3sas/mpt3sas_scsih.c
   5  * Copyright (C) 2012-2014  LSI Corporation
   6  * Copyright (C) 2013-2014 Avago Technologies
   7  *  (mailto: MPT-FusionLinux.pdl@avagotech.com)
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * as published by the Free Software Foundation; either version 2
  12  * of the License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * NO WARRANTY
  20  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
  21  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
  22  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
  23  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
  24  * solely responsible for determining the appropriateness of using and
  25  * distributing the Program and assumes all risks associated with its
  26  * exercise of rights under this Agreement, including but not limited to
  27  * the risks and costs of program errors, damage to or loss of data,
  28  * programs or equipment, and unavailability or interruption of operations.
  29
  30  * DISCLAIMER OF LIABILITY
  31  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
  32  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
  34  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  35  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  36  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
  37  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
  38
  39  * You should have received a copy of the GNU General Public License
  40  * along with this program; if not, write to the Free Software
  41  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  42  * USA.
  43  */
  44
  45 #include <linux/module.h>
  46 #include <linux/kernel.h>
  47 #include <linux/init.h>
  48 #include <linux/errno.h>
  49 #include <linux/blkdev.h>
  50 #include <linux/sched.h>
  51 #include <linux/workqueue.h>
  52 #include <linux/delay.h>
  53 #include <linux/pci.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/raid_class.h>
  56 #include <linux/blk-mq-pci.h>
  57 #include <asm/unaligned.h>
  58
  59 #include "mpt3sas_base.h"
  60
  61 #define RAID_CHANNEL 1
  62
  63 #define PCIE_CHANNEL 2
  64
  65 /* forward proto's */
  66 static void _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
  67         struct _sas_node *sas_expander);
  68 static void _firmware_event_work(struct work_struct *work);
  69
  70 static void _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
  71         struct _sas_device *sas_device);
  72 static int _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
  73         u8 retry_count, u8 is_pd);
  74 static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
  75 static void _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
  76         struct _pcie_device *pcie_device);
  77 static void
  78 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
  79 static u8 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid);
  80 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc);
  81
  82 /* global parameters */
  83 LIST_HEAD(mpt3sas_ioc_list);
  84 /* global ioc lock for list operations */
  85 DEFINE_SPINLOCK(gioc_lock);
  86
  87 MODULE_AUTHOR(MPT3SAS_AUTHOR);
  88 MODULE_DESCRIPTION(MPT3SAS_DESCRIPTION);
  89 MODULE_LICENSE("GPL");
  90 MODULE_VERSION(MPT3SAS_DRIVER_VERSION);
  91 MODULE_ALIAS("mpt2sas");
  92
  93 /* local parameters */
  94 static u8 scsi_io_cb_idx = -1;
  95 static u8 tm_cb_idx = -1;
  96 static u8 ctl_cb_idx = -1;
  97 static u8 base_cb_idx = -1;
  98 static u8 port_enable_cb_idx = -1;
  99 static u8 transport_cb_idx = -1;
 100 static u8 scsih_cb_idx = -1;
 101 static u8 config_cb_idx = -1;
 102 static int mpt2_ids;
 103 static int mpt3_ids;
 104
 105 static u8 tm_tr_cb_idx = -1 ;
 106 static u8 tm_tr_volume_cb_idx = -1 ;
 107 static u8 tm_sas_control_cb_idx = -1;
 108
 109 /* command line options */
 110 static u32 logging_level;
 111 MODULE_PARM_DESC(logging_level,
 112         " bits for enabling additional logging info (default=0)");
 113
 114
 115 static ushort max_sectors = 0xFFFF;
 116 module_param(max_sectors, ushort, 0444);
 117 MODULE_PARM_DESC(max_sectors, "max sectors, range 64 to 32767  default=32767");
 118
 119
 120 static int missing_delay[2] = {-1, -1};
 121 module_param_array(missing_delay, int, NULL, 0444);
 122 MODULE_PARM_DESC(missing_delay, " device missing delay , io missing delay");
 123
 124 /* scsi-mid layer global parmeter is max_report_luns, which is 511 */
 125 #define MPT3SAS_MAX_LUN (16895)
 126 static u64 max_lun = MPT3SAS_MAX_LUN;
 127 module_param(max_lun, ullong, 0444);
 128 MODULE_PARM_DESC(max_lun, " max lun, default=16895 ");
 129
 130 static ushort hbas_to_enumerate;
 131 module_param(hbas_to_enumerate, ushort, 0444);
 132 MODULE_PARM_DESC(hbas_to_enumerate,
 133                 " 0 - enumerates both SAS 2.0 & SAS 3.0 generation HBAs\n \
 134                   1 - enumerates only SAS 2.0 generation HBAs\n \
 135                   2 - enumerates only SAS 3.0 generation HBAs (default=0)");
 136
 137 /* diag_buffer_enable is bitwise
 138  * bit 0 set = TRACE
 139  * bit 1 set = SNAPSHOT
 140  * bit 2 set = EXTENDED
 141  *
 142  * Either bit can be set, or both
 143  */
 144 static int diag_buffer_enable = -1;
 145 module_param(diag_buffer_enable, int, 0444);
 146 MODULE_PARM_DESC(diag_buffer_enable,
 147         " post diag buffers (TRACE=1/SNAPSHOT=2/EXTENDED=4/default=0)");
 148 static int disable_discovery = -1;
 149 module_param(disable_discovery, int, 0444);
 150 MODULE_PARM_DESC(disable_discovery, " disable discovery ");
 151
 152
 153 /* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
 154 static int prot_mask = -1;
 155 module_param(prot_mask, int, 0444);
 156 MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=7 ");
 157
 158 static bool enable_sdev_max_qd;
 159 module_param(enable_sdev_max_qd, bool, 0444);
 160 MODULE_PARM_DESC(enable_sdev_max_qd,
 161         "Enable sdev max qd as can_queue, def=disabled(0)");
 162
 163 static int multipath_on_hba = -1;
 164 module_param(multipath_on_hba, int, 0);
 165 MODULE_PARM_DESC(multipath_on_hba,
 166         "Multipath support to add same target device\n\t\t"
 167         "as many times as it is visible to HBA from various paths\n\t\t"
 168         "(by default:\n\t\t"
 169         "\t SAS 2.0 & SAS 3.0 HBA - This will be disabled,\n\t\t"
 170         "\t SAS 3.5 HBA - This will be enabled)");
 171
 172 static int host_tagset_enable = 1;
 173 module_param(host_tagset_enable, int, 0444);
 174 MODULE_PARM_DESC(host_tagset_enable,
 175         "Shared host tagset enable/disable Default: enable(1)");
 176
 177 /* raid transport support */
 178 static struct raid_template *mpt3sas_raid_template;
 179 static struct raid_template *mpt2sas_raid_template;
 180
 181
 182 /**
 183  * struct sense_info - common structure for obtaining sense keys
 184  * @skey: sense key
 185  * @asc: additional sense code
 186  * @ascq: additional sense code qualifier
 187  */
 188 struct sense_info {
 189         u8 skey;
 190         u8 asc;
 191         u8 ascq;
 192 };
 193
 194 #define MPT3SAS_PROCESS_TRIGGER_DIAG (0xFFFB)
 195 #define MPT3SAS_TURN_ON_PFA_LED (0xFFFC)
 196 #define MPT3SAS_PORT_ENABLE_COMPLETE (0xFFFD)
 197 #define MPT3SAS_ABRT_TASK_SET (0xFFFE)
 198 #define MPT3SAS_REMOVE_UNRESPONDING_DEVICES (0xFFFF)
 199 /**
 200  * struct fw_event_work - firmware event struct
 201  * @list: link list framework
 202  * @work: work object (ioc->fault_reset_work_q)
 203  * @ioc: per adapter object
 204  * @device_handle: device handle
 205  * @VF_ID: virtual function id
 206  * @VP_ID: virtual port id
 207  * @ignore: flag meaning this event has been marked to ignore
 208  * @event: firmware event MPI2_EVENT_XXX defined in mpi2_ioc.h
 209  * @refcount: kref for this event
 210  * @event_data: reply event data payload follows
 211  *
 212  * This object stored on ioc->fw_event_list.
 213  */
 214 struct fw_event_work {
 215         struct list_head        list;
 216         struct work_struct      work;
 217
 218         struct MPT3SAS_ADAPTER *ioc;
 219         u16                     device_handle;
 220         u8                      VF_ID;
 221         u8                      VP_ID;
 222         u8                      ignore;
 223         u16                     event;
 224         struct kref             refcount;
 225         char                    event_data[] __aligned(4);
 226 };
 227
 228 static void fw_event_work_free(struct kref *r)
 229 {
 230         kfree(container_of(r, struct fw_event_work, refcount));
 231 }
 232
 233 static void fw_event_work_get(struct fw_event_work *fw_work)
 234 {
 235         kref_get(&fw_work->refcount);
 236 }
 237
 238 static void fw_event_work_put(struct fw_event_work *fw_work)
 239 {
 240         kref_put(&fw_work->refcount, fw_event_work_free);
 241 }
 242
 243 static struct fw_event_work *alloc_fw_event_work(int len)
 244 {
 245         struct fw_event_work *fw_event;
 246
 247         fw_event = kzalloc(sizeof(*fw_event) + len, GFP_ATOMIC);
 248         if (!fw_event)
 249                 return NULL;
 250
 251         kref_init(&fw_event->refcount);
 252         return fw_event;
 253 }
 254
 255 /**
 256  * struct _scsi_io_transfer - scsi io transfer
 257  * @handle: sas device handle (assigned by firmware)
 258  * @is_raid: flag set for hidden raid components
 259  * @dir: DMA_TO_DEVICE, DMA_FROM_DEVICE,
 260  * @data_length: data transfer length
 261  * @data_dma: dma pointer to data
 262  * @sense: sense data
 263  * @lun: lun number
 264  * @cdb_length: cdb length
 265  * @cdb: cdb contents
 266  * @timeout: timeout for this command
 267  * @VF_ID: virtual function id
 268  * @VP_ID: virtual port id
 269  * @valid_reply: flag set for reply message
 270  * @sense_length: sense length
 271  * @ioc_status: ioc status
 272  * @scsi_state: scsi state
 273  * @scsi_status: scsi staus
 274  * @log_info: log information
 275  * @transfer_length: data length transfer when there is a reply message
 276  *
 277  * Used for sending internal scsi commands to devices within this module.
 278  * Refer to _scsi_send_scsi_io().
 279  */
 280 struct _scsi_io_transfer {
 281         u16     handle;
 282         u8      is_raid;
 283         enum dma_data_direction dir;
 284         u32     data_length;
 285         dma_addr_t data_dma;
 286         u8      sense[SCSI_SENSE_BUFFERSIZE];
 287         u32     lun;
 288         u8      cdb_length;
 289         u8      cdb[32];
 290         u8      timeout;
 291         u8      VF_ID;
 292         u8      VP_ID;
 293         u8      valid_reply;
 294   /* the following bits are only valid when 'valid_reply = 1' */
 295         u32     sense_length;
 296         u16     ioc_status;
 297         u8      scsi_state;
 298         u8      scsi_status;
 299         u32     log_info;
 300         u32     transfer_length;
 301 };
 302
 303 /**
 304  * _scsih_set_debug_level - global setting of ioc->logging_level.
 305  * @val: ?
 306  * @kp: ?
 307  *
 308  * Note: The logging levels are defined in mpt3sas_debug.h.
 309  */
 310 static int
 311 _scsih_set_debug_level(const char *val, const struct kernel_param *kp)
 312 {
 313         int ret = param_set_int(val, kp);
 314         struct MPT3SAS_ADAPTER *ioc;
 315
 316         if (ret)
 317                 return ret;
 318
 319         pr_info("setting logging_level(0x%08x)\n", logging_level);
 320         spin_lock(&gioc_lock);
 321         list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
 322                 ioc->logging_level = logging_level;
 323         spin_unlock(&gioc_lock);
 324         return 0;
 325 }
 326 module_param_call(logging_level, _scsih_set_debug_level, param_get_int,
 327         &logging_level, 0644);
 328
 329 /**
 330  * _scsih_srch_boot_sas_address - search based on sas_address
 331  * @sas_address: sas address
 332  * @boot_device: boot device object from bios page 2
 333  *
 334  * Return: 1 when there's a match, 0 means no match.
 335  */
 336 static inline int
 337 _scsih_srch_boot_sas_address(u64 sas_address,
 338         Mpi2BootDeviceSasWwid_t *boot_device)
 339 {
 340         return (sas_address == le64_to_cpu(boot_device->SASAddress)) ?  1 : 0;
 341 }
 342
 343 /**
 344  * _scsih_srch_boot_device_name - search based on device name
 345  * @device_name: device name specified in INDENTIFY fram
 346  * @boot_device: boot device object from bios page 2
 347  *
 348  * Return: 1 when there's a match, 0 means no match.
 349  */
 350 static inline int
 351 _scsih_srch_boot_device_name(u64 device_name,
 352         Mpi2BootDeviceDeviceName_t *boot_device)
 353 {
 354         return (device_name == le64_to_cpu(boot_device->DeviceName)) ? 1 : 0;
 355 }
 356
 357 /**
 358  * _scsih_srch_boot_encl_slot - search based on enclosure_logical_id/slot
 359  * @enclosure_logical_id: enclosure logical id
 360  * @slot_number: slot number
 361  * @boot_device: boot device object from bios page 2
 362  *
 363  * Return: 1 when there's a match, 0 means no match.
 364  */
 365 static inline int
 366 _scsih_srch_boot_encl_slot(u64 enclosure_logical_id, u16 slot_number,
 367         Mpi2BootDeviceEnclosureSlot_t *boot_device)
 368 {
 369         return (enclosure_logical_id == le64_to_cpu(boot_device->
 370             EnclosureLogicalID) && slot_number == le16_to_cpu(boot_device->
 371             SlotNumber)) ? 1 : 0;
 372 }
 373
 374 /**
 375  * mpt3sas_get_port_by_id - get hba port entry corresponding to provided
 376  *                        port number from port list
 377  * @ioc: per adapter object
 378  * @port_id: port number
 379  * @bypass_dirty_port_flag: when set look the matching hba port entry even
 380  *                      if hba port entry is marked as dirty.
 381  *
 382  * Search for hba port entry corresponding to provided port number,
 383  * if available return port object otherwise return NULL.
 384  */
 385 struct hba_port *
 386 mpt3sas_get_port_by_id(struct MPT3SAS_ADAPTER *ioc,
 387         u8 port_id, u8 bypass_dirty_port_flag)
 388 {
 389         struct hba_port *port, *port_next;
 390
 391         /*
 392          * When multipath_on_hba is disabled then
 393          * search the hba_port entry using default
 394          * port id i.e. 255
 395          */
 396         if (!ioc->multipath_on_hba)
 397                 port_id = MULTIPATH_DISABLED_PORT_ID;
 398
 399         list_for_each_entry_safe(port, port_next,
 400             &ioc->port_table_list, list) {
 401                 if (port->port_id != port_id)
 402                         continue;
 403                 if (bypass_dirty_port_flag)
 404                         return port;
 405                 if (port->flags & HBA_PORT_FLAG_DIRTY_PORT)
 406                         continue;
 407                 return port;
 408         }
 409
 410         /*
 411          * Allocate hba_port object for default port id (i.e. 255)
 412          * when multipath_on_hba is disabled for the HBA.
 413          * And add this object to port_table_list.
 414          */
 415         if (!ioc->multipath_on_hba) {
 416                 port = kzalloc(sizeof(struct hba_port), GFP_ATOMIC);
 417                 if (!port)
 418                         return NULL;
 419
 420                 port->port_id = port_id;
 421                 ioc_info(ioc,
 422                    "hba_port entry: %p, port: %d is added to hba_port list\n",
 423                    port, port->port_id);
 424                 list_add_tail(&port->list,
 425                     &ioc->port_table_list);
 426                 return port;
 427         }
 428         return NULL;
 429 }
 430
 431 /**
 432  * mpt3sas_get_vphy_by_phy - get virtual_phy object corresponding to phy number
 433  * @ioc: per adapter object
 434  * @port: hba_port object
 435  * @phy: phy number
 436  *
 437  * Return virtual_phy object corresponding to phy number.
 438  */
 439 struct virtual_phy *
 440 mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,
 441         struct hba_port *port, u32 phy)
 442 {
 443         struct virtual_phy *vphy, *vphy_next;
 444
 445         if (!port->vphys_mask)
 446                 return NULL;
 447
 448         list_for_each_entry_safe(vphy, vphy_next, &port->vphys_list, list) {
 449                 if (vphy->phy_mask & (1 << phy))
 450                         return vphy;
 451         }
 452         return NULL;
 453 }
 454
 455 /**
 456  * _scsih_is_boot_device - search for matching boot device.
 457  * @sas_address: sas address
 458  * @device_name: device name specified in INDENTIFY fram
 459  * @enclosure_logical_id: enclosure logical id
 460  * @slot: slot number
 461  * @form: specifies boot device form
 462  * @boot_device: boot device object from bios page 2
 463  *
 464  * Return: 1 when there's a match, 0 means no match.
 465  */
 466 static int
 467 _scsih_is_boot_device(u64 sas_address, u64 device_name,
 468         u64 enclosure_logical_id, u16 slot, u8 form,
 469         Mpi2BiosPage2BootDevice_t *boot_device)
 470 {
 471         int rc = 0;
 472
 473         switch (form) {
 474         case MPI2_BIOSPAGE2_FORM_SAS_WWID:
 475                 if (!sas_address)
 476                         break;
 477                 rc = _scsih_srch_boot_sas_address(
 478                     sas_address, &boot_device->SasWwid);
 479                 break;
 480         case MPI2_BIOSPAGE2_FORM_ENCLOSURE_SLOT:
 481                 if (!enclosure_logical_id)
 482                         break;
 483                 rc = _scsih_srch_boot_encl_slot(
 484                     enclosure_logical_id,
 485                     slot, &boot_device->EnclosureSlot);
 486                 break;
 487         case MPI2_BIOSPAGE2_FORM_DEVICE_NAME:
 488                 if (!device_name)
 489                         break;
 490                 rc = _scsih_srch_boot_device_name(
 491                     device_name, &boot_device->DeviceName);
 492                 break;
 493         case MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED:
 494                 break;
 495         }
 496
 497         return rc;
 498 }
 499
 500 /**
 501  * _scsih_get_sas_address - set the sas_address for given device handle
 502  * @ioc: ?
 503  * @handle: device handle
 504  * @sas_address: sas address
 505  *
 506  * Return: 0 success, non-zero when failure
 507  */
 508 static int
 509 _scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 510         u64 *sas_address)
 511 {
 512         Mpi2SasDevicePage0_t sas_device_pg0;
 513         Mpi2ConfigReply_t mpi_reply;
 514         u32 ioc_status;
 515
 516         *sas_address = 0;
 517
 518         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
 519             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
 520                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
 521                         __FILE__, __LINE__, __func__);
 522                 return -ENXIO;
 523         }
 524
 525         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
 526         if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
 527                 /* For HBA, vSES doesn't return HBA SAS address. Instead return
 528                  * vSES's sas address.
 529                  */
 530                 if ((handle <= ioc->sas_hba.num_phys) &&
 531                    (!(le32_to_cpu(sas_device_pg0.DeviceInfo) &
 532                    MPI2_SAS_DEVICE_INFO_SEP)))
 533                         *sas_address = ioc->sas_hba.sas_address;
 534                 else
 535                         *sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
 536                 return 0;
 537         }
 538
 539         /* we hit this because the given parent handle doesn't exist */
 540         if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
 541                 return -ENXIO;
 542
 543         /* else error case */
 544         ioc_err(ioc, "handle(0x%04x), ioc_status(0x%04x), failure at %s:%d/%s()!\n",
 545                 handle, ioc_status, __FILE__, __LINE__, __func__);
 546         return -EIO;
 547 }
 548
 549 /**
 550  * _scsih_determine_boot_device - determine boot device.
 551  * @ioc: per adapter object
 552  * @device: sas_device or pcie_device object
 553  * @channel: SAS or PCIe channel
 554  *
 555  * Determines whether this device should be first reported device to
 556  * to scsi-ml or sas transport, this purpose is for persistent boot device.
 557  * There are primary, alternate, and current entries in bios page 2. The order
 558  * priority is primary, alternate, then current.  This routine saves
 559  * the corresponding device object.
 560  * The saved data to be used later in _scsih_probe_boot_devices().
 561  */
 562 static void
 563 _scsih_determine_boot_device(struct MPT3SAS_ADAPTER *ioc, void *device,
 564         u32 channel)
 565 {
 566         struct _sas_device *sas_device;
 567         struct _pcie_device *pcie_device;
 568         struct _raid_device *raid_device;
 569         u64 sas_address;
 570         u64 device_name;
 571         u64 enclosure_logical_id;
 572         u16 slot;
 573
 574          /* only process this function when driver loads */
 575         if (!ioc->is_driver_loading)
 576                 return;
 577
 578          /* no Bios, return immediately */
 579         if (!ioc->bios_pg3.BiosVersion)
 580                 return;
 581
 582         if (channel == RAID_CHANNEL) {
 583                 raid_device = device;
 584                 sas_address = raid_device->wwid;
 585                 device_name = 0;
 586                 enclosure_logical_id = 0;
 587                 slot = 0;
 588         } else if (channel == PCIE_CHANNEL) {
 589                 pcie_device = device;
 590                 sas_address = pcie_device->wwid;
 591                 device_name = 0;
 592                 enclosure_logical_id = 0;
 593                 slot = 0;
 594         } else {
 595                 sas_device = device;
 596                 sas_address = sas_device->sas_address;
 597                 device_name = sas_device->device_name;
 598                 enclosure_logical_id = sas_device->enclosure_logical_id;
 599                 slot = sas_device->slot;
 600         }
 601
 602         if (!ioc->req_boot_device.device) {
 603                 if (_scsih_is_boot_device(sas_address, device_name,
 604                     enclosure_logical_id, slot,
 605                     (ioc->bios_pg2.ReqBootDeviceForm &
 606                     MPI2_BIOSPAGE2_FORM_MASK),
 607                     &ioc->bios_pg2.RequestedBootDevice)) {
 608                         dinitprintk(ioc,
 609                                     ioc_info(ioc, "%s: req_boot_device(0x%016llx)\n",
 610                                              __func__, (u64)sas_address));
 611                         ioc->req_boot_device.device = device;
 612                         ioc->req_boot_device.channel = channel;
 613                 }
 614         }
 615
 616         if (!ioc->req_alt_boot_device.device) {
 617                 if (_scsih_is_boot_device(sas_address, device_name,
 618                     enclosure_logical_id, slot,
 619                     (ioc->bios_pg2.ReqAltBootDeviceForm &
 620                     MPI2_BIOSPAGE2_FORM_MASK),
 621                     &ioc->bios_pg2.RequestedAltBootDevice)) {
 622                         dinitprintk(ioc,
 623                                     ioc_info(ioc, "%s: req_alt_boot_device(0x%016llx)\n",
 624                                              __func__, (u64)sas_address));
 625                         ioc->req_alt_boot_device.device = device;
 626                         ioc->req_alt_boot_device.channel = channel;
 627                 }
 628         }
 629
 630         if (!ioc->current_boot_device.device) {
 631                 if (_scsih_is_boot_device(sas_address, device_name,
 632                     enclosure_logical_id, slot,
 633                     (ioc->bios_pg2.CurrentBootDeviceForm &
 634                     MPI2_BIOSPAGE2_FORM_MASK),
 635                     &ioc->bios_pg2.CurrentBootDevice)) {
 636                         dinitprintk(ioc,
 637                                     ioc_info(ioc, "%s: current_boot_device(0x%016llx)\n",
 638                                              __func__, (u64)sas_address));
 639                         ioc->current_boot_device.device = device;
 640                         ioc->current_boot_device.channel = channel;
 641                 }
 642         }
 643 }
 644
 645 static struct _sas_device *
 646 __mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 647                 struct MPT3SAS_TARGET *tgt_priv)
 648 {
 649         struct _sas_device *ret;
 650
 651         assert_spin_locked(&ioc->sas_device_lock);
 652
 653         ret = tgt_priv->sas_dev;
 654         if (ret)
 655                 sas_device_get(ret);
 656
 657         return ret;
 658 }
 659
 660 static struct _sas_device *
 661 mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 662                 struct MPT3SAS_TARGET *tgt_priv)
 663 {
 664         struct _sas_device *ret;
 665         unsigned long flags;
 666
 667         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 668         ret = __mpt3sas_get_sdev_from_target(ioc, tgt_priv);
 669         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 670
 671         return ret;
 672 }
 673
 674 static struct _pcie_device *
 675 __mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 676         struct MPT3SAS_TARGET *tgt_priv)
 677 {
 678         struct _pcie_device *ret;
 679
 680         assert_spin_locked(&ioc->pcie_device_lock);
 681
 682         ret = tgt_priv->pcie_dev;
 683         if (ret)
 684                 pcie_device_get(ret);
 685
 686         return ret;
 687 }
 688
 689 /**
 690  * mpt3sas_get_pdev_from_target - pcie device search
 691  * @ioc: per adapter object
 692  * @tgt_priv: starget private object
 693  *
 694  * Context: This function will acquire ioc->pcie_device_lock and will release
 695  * before returning the pcie_device object.
 696  *
 697  * This searches for pcie_device from target, then return pcie_device object.
 698  */
 699 static struct _pcie_device *
 700 mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 701         struct MPT3SAS_TARGET *tgt_priv)
 702 {
 703         struct _pcie_device *ret;
 704         unsigned long flags;
 705
 706         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
 707         ret = __mpt3sas_get_pdev_from_target(ioc, tgt_priv);
 708         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
 709
 710         return ret;
 711 }
 712
 713
 714 /**
 715  * __mpt3sas_get_sdev_by_rphy - sas device search
 716  * @ioc: per adapter object
 717  * @rphy: sas_rphy pointer
 718  *
 719  * Context: This function will acquire ioc->sas_device_lock and will release
 720  * before returning the sas_device object.
 721  *
 722  * This searches for sas_device from rphy object
 723  * then return sas_device object.
 724  */
 725 struct _sas_device *
 726 __mpt3sas_get_sdev_by_rphy(struct MPT3SAS_ADAPTER *ioc,
 727         struct sas_rphy *rphy)
 728 {
 729         struct _sas_device *sas_device;
 730
 731         assert_spin_locked(&ioc->sas_device_lock);
 732
 733         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
 734                 if (sas_device->rphy != rphy)
 735                         continue;
 736                 sas_device_get(sas_device);
 737                 return sas_device;
 738         }
 739
 740         sas_device = NULL;
 741         list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {
 742                 if (sas_device->rphy != rphy)
 743                         continue;
 744                 sas_device_get(sas_device);
 745                 return sas_device;
 746         }
 747
 748         return NULL;
 749 }
 750
 751 /**
 752  * __mpt3sas_get_sdev_by_addr - get _sas_device object corresponding to provided
 753  *                              sas address from sas_device_list list
 754  * @ioc: per adapter object
 755  * @sas_address: device sas address
 756  * @port: port number
 757  *
 758  * Search for _sas_device object corresponding to provided sas address,
 759  * if available return _sas_device object address otherwise return NULL.
 760  */
 761 struct _sas_device *
 762 __mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,
 763         u64 sas_address, struct hba_port *port)
 764 {
 765         struct _sas_device *sas_device;
 766
 767         if (!port)
 768                 return NULL;
 769
 770         assert_spin_locked(&ioc->sas_device_lock);
 771
 772         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
 773                 if (sas_device->sas_address != sas_address)
 774                         continue;
 775                 if (sas_device->port != port)
 776                         continue;
 777                 sas_device_get(sas_device);
 778                 return sas_device;
 779         }
 780
 781         list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {
 782                 if (sas_device->sas_address != sas_address)
 783                         continue;
 784                 if (sas_device->port != port)
 785                         continue;
 786                 sas_device_get(sas_device);
 787                 return sas_device;
 788         }
 789
 790         return NULL;
 791 }
 792
 793 /**
 794  * mpt3sas_get_sdev_by_addr - sas device search
 795  * @ioc: per adapter object
 796  * @sas_address: sas address
 797  * @port: hba port entry
 798  * Context: Calling function should acquire ioc->sas_device_lock
 799  *
 800  * This searches for sas_device based on sas_address & port number,
 801  * then return sas_device object.
 802  */
 803 struct _sas_device *
 804 mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,
 805         u64 sas_address, struct hba_port *port)
 806 {
 807         struct _sas_device *sas_device;
 808         unsigned long flags;
 809
 810         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 811         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
 812             sas_address, port);
 813         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 814
 815         return sas_device;
 816 }
 817
 818 static struct _sas_device *
 819 __mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 820 {
 821         struct _sas_device *sas_device;
 822
 823         assert_spin_locked(&ioc->sas_device_lock);
 824
 825         list_for_each_entry(sas_device, &ioc->sas_device_list, list)
 826                 if (sas_device->handle == handle)
 827                         goto found_device;
 828
 829         list_for_each_entry(sas_device, &ioc->sas_device_init_list, list)
 830                 if (sas_device->handle == handle)
 831                         goto found_device;
 832
 833         return NULL;
 834
 835 found_device:
 836         sas_device_get(sas_device);
 837         return sas_device;
 838 }
 839
 840 /**
 841  * mpt3sas_get_sdev_by_handle - sas device search
 842  * @ioc: per adapter object
 843  * @handle: sas device handle (assigned by firmware)
 844  * Context: Calling function should acquire ioc->sas_device_lock
 845  *
 846  * This searches for sas_device based on sas_address, then return sas_device
 847  * object.
 848  */
 849 struct _sas_device *
 850 mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 851 {
 852         struct _sas_device *sas_device;
 853         unsigned long flags;
 854
 855         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 856         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
 857         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 858
 859         return sas_device;
 860 }
 861
 862 /**
 863  * _scsih_display_enclosure_chassis_info - display device location info
 864  * @ioc: per adapter object
 865  * @sas_device: per sas device object
 866  * @sdev: scsi device struct
 867  * @starget: scsi target struct
 868  */
 869 static void
 870 _scsih_display_enclosure_chassis_info(struct MPT3SAS_ADAPTER *ioc,
 871         struct _sas_device *sas_device, struct scsi_device *sdev,
 872         struct scsi_target *starget)
 873 {
 874         if (sdev) {
 875                 if (sas_device->enclosure_handle != 0)
 876                         sdev_printk(KERN_INFO, sdev,
 877                             "enclosure logical id (0x%016llx), slot(%d) \n",
 878                             (unsigned long long)
 879                             sas_device->enclosure_logical_id,
 880                             sas_device->slot);
 881                 if (sas_device->connector_name[0] != '\0')
 882                         sdev_printk(KERN_INFO, sdev,
 883                             "enclosure level(0x%04x), connector name( %s)\n",
 884                             sas_device->enclosure_level,
 885                             sas_device->connector_name);
 886                 if (sas_device->is_chassis_slot_valid)
 887                         sdev_printk(KERN_INFO, sdev, "chassis slot(0x%04x)\n",
 888                             sas_device->chassis_slot);
 889         } else if (starget) {
 890                 if (sas_device->enclosure_handle != 0)
 891                         starget_printk(KERN_INFO, starget,
 892                             "enclosure logical id(0x%016llx), slot(%d) \n",
 893                             (unsigned long long)
 894                             sas_device->enclosure_logical_id,
 895                             sas_device->slot);
 896                 if (sas_device->connector_name[0] != '\0')
 897                         starget_printk(KERN_INFO, starget,
 898                             "enclosure level(0x%04x), connector name( %s)\n",
 899                             sas_device->enclosure_level,
 900                             sas_device->connector_name);
 901                 if (sas_device->is_chassis_slot_valid)
 902                         starget_printk(KERN_INFO, starget,
 903                             "chassis slot(0x%04x)\n",
 904                             sas_device->chassis_slot);
 905         } else {
 906                 if (sas_device->enclosure_handle != 0)
 907                         ioc_info(ioc, "enclosure logical id(0x%016llx), slot(%d)\n",
 908                                  (u64)sas_device->enclosure_logical_id,
 909                                  sas_device->slot);
 910                 if (sas_device->connector_name[0] != '\0')
 911                         ioc_info(ioc, "enclosure level(0x%04x), connector name( %s)\n",
 912                                  sas_device->enclosure_level,
 913                                  sas_device->connector_name);
 914                 if (sas_device->is_chassis_slot_valid)
 915                         ioc_info(ioc, "chassis slot(0x%04x)\n",
 916                                  sas_device->chassis_slot);
 917         }
 918 }
 919
 920 /**
 921  * _scsih_sas_device_remove - remove sas_device from list.
 922  * @ioc: per adapter object
 923  * @sas_device: the sas_device object
 924  * Context: This function will acquire ioc->sas_device_lock.
 925  *
 926  * If sas_device is on the list, remove it and decrement its reference count.
 927  */
 928 static void
 929 _scsih_sas_device_remove(struct MPT3SAS_ADAPTER *ioc,
 930         struct _sas_device *sas_device)
 931 {
 932         unsigned long flags;
 933
 934         if (!sas_device)
 935                 return;
 936         ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",
 937                  sas_device->handle, (u64)sas_device->sas_address);
 938
 939         _scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);
 940
 941         /*
 942          * The lock serializes access to the list, but we still need to verify
 943          * that nobody removed the entry while we were waiting on the lock.
 944          */
 945         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 946         if (!list_empty(&sas_device->list)) {
 947                 list_del_init(&sas_device->list);
 948                 sas_device_put(sas_device);
 949         }
 950         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 951 }
 952
 953 /**
 954  * _scsih_device_remove_by_handle - removing device object by handle
 955  * @ioc: per adapter object
 956  * @handle: device handle
 957  */
 958 static void
 959 _scsih_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 960 {
 961         struct _sas_device *sas_device;
 962         unsigned long flags;
 963
 964         if (ioc->shost_recovery)
 965                 return;
 966
 967         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 968         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
 969         if (sas_device) {
 970                 list_del_init(&sas_device->list);
 971                 sas_device_put(sas_device);
 972         }
 973         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 974         if (sas_device) {
 975                 _scsih_remove_device(ioc, sas_device);
 976                 sas_device_put(sas_device);
 977         }
 978 }
 979
 980 /**
 981  * mpt3sas_device_remove_by_sas_address - removing device object by
 982  *                                      sas address & port number
 983  * @ioc: per adapter object
 984  * @sas_address: device sas_address
 985  * @port: hba port entry
 986  *
 987  * Return nothing.
 988  */
 989 void
 990 mpt3sas_device_remove_by_sas_address(struct MPT3SAS_ADAPTER *ioc,
 991         u64 sas_address, struct hba_port *port)
 992 {
 993         struct _sas_device *sas_device;
 994         unsigned long flags;
 995
 996         if (ioc->shost_recovery)
 997                 return;
 998
 999         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1000         sas_device = __mpt3sas_get_sdev_by_addr(ioc, sas_address, port);
1001         if (sas_device) {
1002                 list_del_init(&sas_device->list);
1003                 sas_device_put(sas_device);
1004         }
1005         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1006         if (sas_device) {
1007                 _scsih_remove_device(ioc, sas_device);
1008                 sas_device_put(sas_device);
1009         }
1010 }
1011
1012 /**
1013  * _scsih_sas_device_add - insert sas_device to the list.
1014  * @ioc: per adapter object
1015  * @sas_device: the sas_device object
1016  * Context: This function will acquire ioc->sas_device_lock.
1017  *
1018  * Adding new object to the ioc->sas_device_list.
1019  */
1020 static void
1021 _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,
1022         struct _sas_device *sas_device)
1023 {
1024         unsigned long flags;
1025
1026         dewtprintk(ioc,
1027                    ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",
1028                             __func__, sas_device->handle,
1029                             (u64)sas_device->sas_address));
1030
1031         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
1032             NULL, NULL));
1033
1034         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1035         sas_device_get(sas_device);
1036         list_add_tail(&sas_device->list, &ioc->sas_device_list);
1037         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1038
1039         if (ioc->hide_drives) {
1040                 clear_bit(sas_device->handle, ioc->pend_os_device_add);
1041                 return;
1042         }
1043
1044         if (!mpt3sas_transport_port_add(ioc, sas_device->handle,
1045              sas_device->sas_address_parent, sas_device->port)) {
1046                 _scsih_sas_device_remove(ioc, sas_device);
1047         } else if (!sas_device->starget) {
1048                 /*
1049                  * When asyn scanning is enabled, its not possible to remove
1050                  * devices while scanning is turned on due to an oops in
1051                  * scsi_sysfs_add_sdev()->add_device()->sysfs_addrm_start()
1052                  */
1053                 if (!ioc->is_driver_loading) {
1054                         mpt3sas_transport_port_remove(ioc,
1055                             sas_device->sas_address,
1056                             sas_device->sas_address_parent,
1057                             sas_device->port);
1058                         _scsih_sas_device_remove(ioc, sas_device);
1059                 }
1060         } else
1061                 clear_bit(sas_device->handle, ioc->pend_os_device_add);
1062 }
1063
1064 /**
1065  * _scsih_sas_device_init_add - insert sas_device to the list.
1066  * @ioc: per adapter object
1067  * @sas_device: the sas_device object
1068  * Context: This function will acquire ioc->sas_device_lock.
1069  *
1070  * Adding new object at driver load time to the ioc->sas_device_init_list.
1071  */
1072 static void
1073 _scsih_sas_device_init_add(struct MPT3SAS_ADAPTER *ioc,
1074         struct _sas_device *sas_device)
1075 {
1076         unsigned long flags;
1077
1078         dewtprintk(ioc,
1079                    ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",
1080                             __func__, sas_device->handle,
1081                             (u64)sas_device->sas_address));
1082
1083         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
1084             NULL, NULL));
1085
1086         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1087         sas_device_get(sas_device);
1088         list_add_tail(&sas_device->list, &ioc->sas_device_init_list);
1089         _scsih_determine_boot_device(ioc, sas_device, 0);
1090         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1091 }
1092
1093
1094 static struct _pcie_device *
1095 __mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1096 {
1097         struct _pcie_device *pcie_device;
1098
1099         assert_spin_locked(&ioc->pcie_device_lock);
1100
1101         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1102                 if (pcie_device->wwid == wwid)
1103                         goto found_device;
1104
1105         list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1106                 if (pcie_device->wwid == wwid)
1107                         goto found_device;
1108
1109         return NULL;
1110
1111 found_device:
1112         pcie_device_get(pcie_device);
1113         return pcie_device;
1114 }
1115
1116
1117 /**
1118  * mpt3sas_get_pdev_by_wwid - pcie device search
1119  * @ioc: per adapter object
1120  * @wwid: wwid
1121  *
1122  * Context: This function will acquire ioc->pcie_device_lock and will release
1123  * before returning the pcie_device object.
1124  *
1125  * This searches for pcie_device based on wwid, then return pcie_device object.
1126  */
1127 static struct _pcie_device *
1128 mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1129 {
1130         struct _pcie_device *pcie_device;
1131         unsigned long flags;
1132
1133         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1134         pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
1135         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1136
1137         return pcie_device;
1138 }
1139
1140
1141 static struct _pcie_device *
1142 __mpt3sas_get_pdev_by_idchannel(struct MPT3SAS_ADAPTER *ioc, int id,
1143         int channel)
1144 {
1145         struct _pcie_device *pcie_device;
1146
1147         assert_spin_locked(&ioc->pcie_device_lock);
1148
1149         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1150                 if (pcie_device->id == id && pcie_device->channel == channel)
1151                         goto found_device;
1152
1153         list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1154                 if (pcie_device->id == id && pcie_device->channel == channel)
1155                         goto found_device;
1156
1157         return NULL;
1158
1159 found_device:
1160         pcie_device_get(pcie_device);
1161         return pcie_device;
1162 }
1163
1164 static struct _pcie_device *
1165 __mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1166 {
1167         struct _pcie_device *pcie_device;
1168
1169         assert_spin_locked(&ioc->pcie_device_lock);
1170
1171         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1172                 if (pcie_device->handle == handle)
1173                         goto found_device;
1174
1175         list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1176                 if (pcie_device->handle == handle)
1177                         goto found_device;
1178
1179         return NULL;
1180
1181 found_device:
1182         pcie_device_get(pcie_device);
1183         return pcie_device;
1184 }
1185
1186
1187 /**
1188  * mpt3sas_get_pdev_by_handle - pcie device search
1189  * @ioc: per adapter object
1190  * @handle: Firmware device handle
1191  *
1192  * Context: This function will acquire ioc->pcie_device_lock and will release
1193  * before returning the pcie_device object.
1194  *
1195  * This searches for pcie_device based on handle, then return pcie_device
1196  * object.
1197  */
1198 struct _pcie_device *
1199 mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1200 {
1201         struct _pcie_device *pcie_device;
1202         unsigned long flags;
1203
1204         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1205         pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
1206         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1207
1208         return pcie_device;
1209 }
1210
1211 /**
1212  * _scsih_set_nvme_max_shutdown_latency - Update max_shutdown_latency.
1213  * @ioc: per adapter object
1214  * Context: This function will acquire ioc->pcie_device_lock
1215  *
1216  * Update ioc->max_shutdown_latency to that NVMe drives RTD3 Entry Latency
1217  * which has reported maximum among all available NVMe drives.
1218  * Minimum max_shutdown_latency will be six seconds.
1219  */
1220 static void
1221 _scsih_set_nvme_max_shutdown_latency(struct MPT3SAS_ADAPTER *ioc)
1222 {
1223         struct _pcie_device *pcie_device;
1224         unsigned long flags;
1225         u16 shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;
1226
1227         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1228         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
1229                 if (pcie_device->shutdown_latency) {
1230                         if (shutdown_latency < pcie_device->shutdown_latency)
1231                                 shutdown_latency =
1232                                         pcie_device->shutdown_latency;
1233                 }
1234         }
1235         ioc->max_shutdown_latency = shutdown_latency;
1236         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1237 }
1238
1239 /**
1240  * _scsih_pcie_device_remove - remove pcie_device from list.
1241  * @ioc: per adapter object
1242  * @pcie_device: the pcie_device object
1243  * Context: This function will acquire ioc->pcie_device_lock.
1244  *
1245  * If pcie_device is on the list, remove it and decrement its reference count.
1246  */
1247 static void
1248 _scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,
1249         struct _pcie_device *pcie_device)
1250 {
1251         unsigned long flags;
1252         int was_on_pcie_device_list = 0;
1253         u8 update_latency = 0;
1254
1255         if (!pcie_device)
1256                 return;
1257         ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
1258                  pcie_device->handle, (u64)pcie_device->wwid);
1259         if (pcie_device->enclosure_handle != 0)
1260                 ioc_info(ioc, "removing enclosure logical id(0x%016llx), slot(%d)\n",
1261                          (u64)pcie_device->enclosure_logical_id,
1262                          pcie_device->slot);
1263         if (pcie_device->connector_name[0] != '\0')
1264                 ioc_info(ioc, "removing enclosure level(0x%04x), connector name( %s)\n",
1265                          pcie_device->enclosure_level,
1266                          pcie_device->connector_name);
1267
1268         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1269         if (!list_empty(&pcie_device->list)) {
1270                 list_del_init(&pcie_device->list);
1271                 was_on_pcie_device_list = 1;
1272         }
1273         if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)
1274                 update_latency = 1;
1275         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1276         if (was_on_pcie_device_list) {
1277                 kfree(pcie_device->serial_number);
1278                 pcie_device_put(pcie_device);
1279         }
1280
1281         /*
1282          * This device's RTD3 Entry Latency matches IOC's
1283          * max_shutdown_latency. Recalculate IOC's max_shutdown_latency
1284          * from the available drives as current drive is getting removed.
1285          */
1286         if (update_latency)
1287                 _scsih_set_nvme_max_shutdown_latency(ioc);
1288 }
1289
1290
1291 /**
1292  * _scsih_pcie_device_remove_by_handle - removing pcie device object by handle
1293  * @ioc: per adapter object
1294  * @handle: device handle
1295  */
1296 static void
1297 _scsih_pcie_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1298 {
1299         struct _pcie_device *pcie_device;
1300         unsigned long flags;
1301         int was_on_pcie_device_list = 0;
1302         u8 update_latency = 0;
1303
1304         if (ioc->shost_recovery)
1305                 return;
1306
1307         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1308         pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
1309         if (pcie_device) {
1310                 if (!list_empty(&pcie_device->list)) {
1311                         list_del_init(&pcie_device->list);
1312                         was_on_pcie_device_list = 1;
1313                         pcie_device_put(pcie_device);
1314                 }
1315                 if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)
1316                         update_latency = 1;
1317         }
1318         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1319         if (was_on_pcie_device_list) {
1320                 _scsih_pcie_device_remove_from_sml(ioc, pcie_device);
1321                 pcie_device_put(pcie_device);
1322         }
1323
1324         /*
1325          * This device's RTD3 Entry Latency matches IOC's
1326          * max_shutdown_latency. Recalculate IOC's max_shutdown_latency
1327          * from the available drives as current drive is getting removed.
1328          */
1329         if (update_latency)
1330                 _scsih_set_nvme_max_shutdown_latency(ioc);
1331 }
1332
1333 /**
1334  * _scsih_pcie_device_add - add pcie_device object
1335  * @ioc: per adapter object
1336  * @pcie_device: pcie_device object
1337  *
1338  * This is added to the pcie_device_list link list.
1339  */
1340 static void
1341 _scsih_pcie_device_add(struct MPT3SAS_ADAPTER *ioc,
1342         struct _pcie_device *pcie_device)
1343 {
1344         unsigned long flags;
1345
1346         dewtprintk(ioc,
1347                    ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",
1348                             __func__,
1349                             pcie_device->handle, (u64)pcie_device->wwid));
1350         if (pcie_device->enclosure_handle != 0)
1351                 dewtprintk(ioc,
1352                            ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",
1353                                     __func__,
1354                                     (u64)pcie_device->enclosure_logical_id,
1355                                     pcie_device->slot));
1356         if (pcie_device->connector_name[0] != '\0')
1357                 dewtprintk(ioc,
1358                            ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",
1359                                     __func__, pcie_device->enclosure_level,
1360                                     pcie_device->connector_name));
1361
1362         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1363         pcie_device_get(pcie_device);
1364         list_add_tail(&pcie_device->list, &ioc->pcie_device_list);
1365         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1366
1367         if (pcie_device->access_status ==
1368             MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED) {
1369                 clear_bit(pcie_device->handle, ioc->pend_os_device_add);
1370                 return;
1371         }
1372         if (scsi_add_device(ioc->shost, PCIE_CHANNEL, pcie_device->id, 0)) {
1373                 _scsih_pcie_device_remove(ioc, pcie_device);
1374         } else if (!pcie_device->starget) {
1375                 if (!ioc->is_driver_loading) {
1376 /*TODO-- Need to find out whether this condition will occur or not*/
1377                         clear_bit(pcie_device->handle, ioc->pend_os_device_add);
1378                 }
1379         } else
1380                 clear_bit(pcie_device->handle, ioc->pend_os_device_add);
1381 }
1382
1383 /*
1384  * _scsih_pcie_device_init_add - insert pcie_device to the init list.
1385  * @ioc: per adapter object
1386  * @pcie_device: the pcie_device object
1387  * Context: This function will acquire ioc->pcie_device_lock.
1388  *
1389  * Adding new object at driver load time to the ioc->pcie_device_init_list.
1390  */
1391 static void
1392 _scsih_pcie_device_init_add(struct MPT3SAS_ADAPTER *ioc,
1393                                 struct _pcie_device *pcie_device)
1394 {
1395         unsigned long flags;
1396
1397         dewtprintk(ioc,
1398                    ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",
1399                             __func__,
1400                             pcie_device->handle, (u64)pcie_device->wwid));
1401         if (pcie_device->enclosure_handle != 0)
1402                 dewtprintk(ioc,
1403                            ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",
1404                                     __func__,
1405                                     (u64)pcie_device->enclosure_logical_id,
1406                                     pcie_device->slot));
1407         if (pcie_device->connector_name[0] != '\0')
1408                 dewtprintk(ioc,
1409                            ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",
1410                                     __func__, pcie_device->enclosure_level,
1411                                     pcie_device->connector_name));
1412
1413         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1414         pcie_device_get(pcie_device);
1415         list_add_tail(&pcie_device->list, &ioc->pcie_device_init_list);
1416         if (pcie_device->access_status !=
1417             MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED)
1418                 _scsih_determine_boot_device(ioc, pcie_device, PCIE_CHANNEL);
1419         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1420 }
1421 /**
1422  * _scsih_raid_device_find_by_id - raid device search
1423  * @ioc: per adapter object
1424  * @id: sas device target id
1425  * @channel: sas device channel
1426  * Context: Calling function should acquire ioc->raid_device_lock
1427  *
1428  * This searches for raid_device based on target id, then return raid_device
1429  * object.
1430  */
1431 static struct _raid_device *
1432 _scsih_raid_device_find_by_id(struct MPT3SAS_ADAPTER *ioc, int id, int channel)
1433 {
1434         struct _raid_device *raid_device, *r;
1435
1436         r = NULL;
1437         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1438                 if (raid_device->id == id && raid_device->channel == channel) {
1439                         r = raid_device;
1440                         goto out;
1441                 }
1442         }
1443
1444  out:
1445         return r;
1446 }
1447
1448 /**
1449  * mpt3sas_raid_device_find_by_handle - raid device search
1450  * @ioc: per adapter object
1451  * @handle: sas device handle (assigned by firmware)
1452  * Context: Calling function should acquire ioc->raid_device_lock
1453  *
1454  * This searches for raid_device based on handle, then return raid_device
1455  * object.
1456  */
1457 struct _raid_device *
1458 mpt3sas_raid_device_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1459 {
1460         struct _raid_device *raid_device, *r;
1461
1462         r = NULL;
1463         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1464                 if (raid_device->handle != handle)
1465                         continue;
1466                 r = raid_device;
1467                 goto out;
1468         }
1469
1470  out:
1471         return r;
1472 }
1473
1474 /**
1475  * _scsih_raid_device_find_by_wwid - raid device search
1476  * @ioc: per adapter object
1477  * @wwid: ?
1478  * Context: Calling function should acquire ioc->raid_device_lock
1479  *
1480  * This searches for raid_device based on wwid, then return raid_device
1481  * object.
1482  */
1483 static struct _raid_device *
1484 _scsih_raid_device_find_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1485 {
1486         struct _raid_device *raid_device, *r;
1487
1488         r = NULL;
1489         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1490                 if (raid_device->wwid != wwid)
1491                         continue;
1492                 r = raid_device;
1493                 goto out;
1494         }
1495
1496  out:
1497         return r;
1498 }
1499
1500 /**
1501  * _scsih_raid_device_add - add raid_device object
1502  * @ioc: per adapter object
1503  * @raid_device: raid_device object
1504  *
1505  * This is added to the raid_device_list link list.
1506  */
1507 static void
1508 _scsih_raid_device_add(struct MPT3SAS_ADAPTER *ioc,
1509         struct _raid_device *raid_device)
1510 {
1511         unsigned long flags;
1512
1513         dewtprintk(ioc,
1514                    ioc_info(ioc, "%s: handle(0x%04x), wwid(0x%016llx)\n",
1515                             __func__,
1516                             raid_device->handle, (u64)raid_device->wwid));
1517
1518         spin_lock_irqsave(&ioc->raid_device_lock, flags);
1519         list_add_tail(&raid_device->list, &ioc->raid_device_list);
1520         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1521 }
1522
1523 /**
1524  * _scsih_raid_device_remove - delete raid_device object
1525  * @ioc: per adapter object
1526  * @raid_device: raid_device object
1527  *
1528  */
1529 static void
1530 _scsih_raid_device_remove(struct MPT3SAS_ADAPTER *ioc,
1531         struct _raid_device *raid_device)
1532 {
1533         unsigned long flags;
1534
1535         spin_lock_irqsave(&ioc->raid_device_lock, flags);
1536         list_del(&raid_device->list);
1537         kfree(raid_device);
1538         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1539 }
1540
1541 /**
1542  * mpt3sas_scsih_expander_find_by_handle - expander device search
1543  * @ioc: per adapter object
1544  * @handle: expander handle (assigned by firmware)
1545  * Context: Calling function should acquire ioc->sas_device_lock
1546  *
1547  * This searches for expander device based on handle, then returns the
1548  * sas_node object.
1549  */
1550 struct _sas_node *
1551 mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1552 {
1553         struct _sas_node *sas_expander, *r;
1554
1555         r = NULL;
1556         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
1557                 if (sas_expander->handle != handle)
1558                         continue;
1559                 r = sas_expander;
1560                 goto out;
1561         }
1562  out:
1563         return r;
1564 }
1565
1566 /**
1567  * mpt3sas_scsih_enclosure_find_by_handle - exclosure device search
1568  * @ioc: per adapter object
1569  * @handle: enclosure handle (assigned by firmware)
1570  * Context: Calling function should acquire ioc->sas_device_lock
1571  *
1572  * This searches for enclosure device based on handle, then returns the
1573  * enclosure object.
1574  */
1575 static struct _enclosure_node *
1576 mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1577 {
1578         struct _enclosure_node *enclosure_dev, *r;
1579
1580         r = NULL;
1581         list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {
1582                 if (le16_to_cpu(enclosure_dev->pg0.EnclosureHandle) != handle)
1583                         continue;
1584                 r = enclosure_dev;
1585                 goto out;
1586         }
1587 out:
1588         return r;
1589 }
1590 /**
1591  * mpt3sas_scsih_expander_find_by_sas_address - expander device search
1592  * @ioc: per adapter object
1593  * @sas_address: sas address
1594  * @port: hba port entry
1595  * Context: Calling function should acquire ioc->sas_node_lock.
1596  *
1597  * This searches for expander device based on sas_address & port number,
1598  * then returns the sas_node object.
1599  */
1600 struct _sas_node *
1601 mpt3sas_scsih_expander_find_by_sas_address(struct MPT3SAS_ADAPTER *ioc,
1602         u64 sas_address, struct hba_port *port)
1603 {
1604         struct _sas_node *sas_expander, *r = NULL;
1605
1606         if (!port)
1607                 return r;
1608
1609         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
1610                 if (sas_expander->sas_address != sas_address)
1611                         continue;
1612                 if (sas_expander->port != port)
1613                         continue;
1614                 r = sas_expander;
1615                 goto out;
1616         }
1617  out:
1618         return r;
1619 }
1620
1621 /**
1622  * _scsih_expander_node_add - insert expander device to the list.
1623  * @ioc: per adapter object
1624  * @sas_expander: the sas_device object
1625  * Context: This function will acquire ioc->sas_node_lock.
1626  *
1627  * Adding new object to the ioc->sas_expander_list.
1628  */
1629 static void
1630 _scsih_expander_node_add(struct MPT3SAS_ADAPTER *ioc,
1631         struct _sas_node *sas_expander)
1632 {
1633         unsigned long flags;
1634
1635         spin_lock_irqsave(&ioc->sas_node_lock, flags);
1636         list_add_tail(&sas_expander->list, &ioc->sas_expander_list);
1637         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
1638 }
1639
1640 /**
1641  * _scsih_is_end_device - determines if device is an end device
1642  * @device_info: bitfield providing information about the device.
1643  * Context: none
1644  *
1645  * Return: 1 if end device.
1646  */
1647 static int
1648 _scsih_is_end_device(u32 device_info)
1649 {
1650         if (device_info & MPI2_SAS_DEVICE_INFO_END_DEVICE &&
1651                 ((device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET) |
1652                 (device_info & MPI2_SAS_DEVICE_INFO_STP_TARGET) |
1653                 (device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)))
1654                 return 1;
1655         else
1656                 return 0;
1657 }
1658
1659 /**
1660  * _scsih_is_nvme_pciescsi_device - determines if
1661  *                      device is an pcie nvme/scsi device
1662  * @device_info: bitfield providing information about the device.
1663  * Context: none
1664  *
1665  * Returns 1 if device is pcie device type nvme/scsi.
1666  */
1667 static int
1668 _scsih_is_nvme_pciescsi_device(u32 device_info)
1669 {
1670         if (((device_info & MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE)
1671             == MPI26_PCIE_DEVINFO_NVME) ||
1672             ((device_info & MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE)
1673             == MPI26_PCIE_DEVINFO_SCSI))
1674                 return 1;
1675         else
1676                 return 0;
1677 }
1678
1679 /**
1680  * _scsih_scsi_lookup_find_by_target - search for matching channel:id
1681  * @ioc: per adapter object
1682  * @id: target id
1683  * @channel: channel
1684  * Context: This function will acquire ioc->scsi_lookup_lock.
1685  *
1686  * This will search for a matching channel:id in the scsi_lookup array,
1687  * returning 1 if found.
1688  */
1689 static u8
1690 _scsih_scsi_lookup_find_by_target(struct MPT3SAS_ADAPTER *ioc, int id,
1691         int channel)
1692 {
1693         int smid;
1694         struct scsi_cmnd *scmd;
1695
1696         for (smid = 1;
1697              smid <= ioc->shost->can_queue; smid++) {
1698                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1699                 if (!scmd)
1700                         continue;
1701                 if (scmd->device->id == id &&
1702                     scmd->device->channel == channel)
1703                         return 1;
1704         }
1705         return 0;
1706 }
1707
1708 /**
1709  * _scsih_scsi_lookup_find_by_lun - search for matching channel:id:lun
1710  * @ioc: per adapter object
1711  * @id: target id
1712  * @lun: lun number
1713  * @channel: channel
1714  * Context: This function will acquire ioc->scsi_lookup_lock.
1715  *
1716  * This will search for a matching channel:id:lun in the scsi_lookup array,
1717  * returning 1 if found.
1718  */
1719 static u8
1720 _scsih_scsi_lookup_find_by_lun(struct MPT3SAS_ADAPTER *ioc, int id,
1721         unsigned int lun, int channel)
1722 {
1723         int smid;
1724         struct scsi_cmnd *scmd;
1725
1726         for (smid = 1; smid <= ioc->shost->can_queue; smid++) {
1727
1728                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1729                 if (!scmd)
1730                         continue;
1731                 if (scmd->device->id == id &&
1732                     scmd->device->channel == channel &&
1733                     scmd->device->lun == lun)
1734                         return 1;
1735         }
1736         return 0;
1737 }
1738
1739 /**
1740  * mpt3sas_scsih_scsi_lookup_get - returns scmd entry
1741  * @ioc: per adapter object
1742  * @smid: system request message index
1743  *
1744  * Return: the smid stored scmd pointer.
1745  * Then will dereference the stored scmd pointer.
1746  */
1747 struct scsi_cmnd *
1748 mpt3sas_scsih_scsi_lookup_get(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1749 {
1750         struct scsi_cmnd *scmd = NULL;
1751         struct scsiio_tracker *st;
1752         Mpi25SCSIIORequest_t *mpi_request;
1753         u16 tag = smid - 1;
1754
1755         if (smid > 0  &&
1756             smid <= ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT) {
1757                 u32 unique_tag =
1758                     ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag;
1759
1760                 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
1761
1762                 /*
1763                  * If SCSI IO request is outstanding at driver level then
1764                  * DevHandle filed must be non-zero. If DevHandle is zero
1765                  * then it means that this smid is free at driver level,
1766                  * so return NULL.
1767                  */
1768                 if (!mpi_request->DevHandle)
1769                         return scmd;
1770
1771                 scmd = scsi_host_find_tag(ioc->shost, unique_tag);
1772                 if (scmd) {
1773                         st = scsi_cmd_priv(scmd);
1774                         if (st->cb_idx == 0xFF || st->smid == 0)
1775                                 scmd = NULL;
1776                 }
1777         }
1778         return scmd;
1779 }
1780
1781 /**
1782  * scsih_change_queue_depth - setting device queue depth
1783  * @sdev: scsi device struct
1784  * @qdepth: requested queue depth
1785  *
1786  * Return: queue depth.
1787  */
1788 static int
1789 scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
1790 {
1791         struct Scsi_Host *shost = sdev->host;
1792         int max_depth;
1793         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1794         struct MPT3SAS_DEVICE *sas_device_priv_data;
1795         struct MPT3SAS_TARGET *sas_target_priv_data;
1796         struct _sas_device *sas_device;
1797         unsigned long flags;
1798
1799         max_depth = shost->can_queue;
1800
1801         /*
1802          * limit max device queue for SATA to 32 if enable_sdev_max_qd
1803          * is disabled.
1804          */
1805         if (ioc->enable_sdev_max_qd || ioc->is_gen35_ioc)
1806                 goto not_sata;
1807
1808         sas_device_priv_data = sdev->hostdata;
1809         if (!sas_device_priv_data)
1810                 goto not_sata;
1811         sas_target_priv_data = sas_device_priv_data->sas_target;
1812         if (!sas_target_priv_data)
1813                 goto not_sata;
1814         if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME))
1815                 goto not_sata;
1816
1817         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1818         sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);
1819         if (sas_device) {
1820                 if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
1821                         max_depth = MPT3SAS_SATA_QUEUE_DEPTH;
1822
1823                 sas_device_put(sas_device);
1824         }
1825         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1826
1827  not_sata:
1828
1829         if (!sdev->tagged_supported)
1830                 max_depth = 1;
1831         if (qdepth > max_depth)
1832                 qdepth = max_depth;
1833         scsi_change_queue_depth(sdev, qdepth);
1834         sdev_printk(KERN_INFO, sdev,
1835             "qdepth(%d), tagged(%d), scsi_level(%d), cmd_que(%d)\n",
1836             sdev->queue_depth, sdev->tagged_supported,
1837             sdev->scsi_level, ((sdev->inquiry[7] & 2) >> 1));
1838         return sdev->queue_depth;
1839 }
1840
1841 /**
1842  * mpt3sas_scsih_change_queue_depth - setting device queue depth
1843  * @sdev: scsi device struct
1844  * @qdepth: requested queue depth
1845  *
1846  * Returns nothing.
1847  */
1848 void
1849 mpt3sas_scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
1850 {
1851         struct Scsi_Host *shost = sdev->host;
1852         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1853
1854         if (ioc->enable_sdev_max_qd)
1855                 qdepth = shost->can_queue;
1856
1857         scsih_change_queue_depth(sdev, qdepth);
1858 }
1859
1860 /**
1861  * scsih_target_alloc - target add routine
1862  * @starget: scsi target struct
1863  *
1864  * Return: 0 if ok. Any other return is assumed to be an error and
1865  * the device is ignored.
1866  */
1867 static int
1868 scsih_target_alloc(struct scsi_target *starget)
1869 {
1870         struct Scsi_Host *shost = dev_to_shost(&starget->dev);
1871         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1872         struct MPT3SAS_TARGET *sas_target_priv_data;
1873         struct _sas_device *sas_device;
1874         struct _raid_device *raid_device;
1875         struct _pcie_device *pcie_device;
1876         unsigned long flags;
1877         struct sas_rphy *rphy;
1878
1879         sas_target_priv_data = kzalloc(sizeof(*sas_target_priv_data),
1880                                        GFP_KERNEL);
1881         if (!sas_target_priv_data)
1882                 return -ENOMEM;
1883
1884         starget->hostdata = sas_target_priv_data;
1885         sas_target_priv_data->starget = starget;
1886         sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
1887
1888         /* RAID volumes */
1889         if (starget->channel == RAID_CHANNEL) {
1890                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
1891                 raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,
1892                     starget->channel);
1893                 if (raid_device) {
1894                         sas_target_priv_data->handle = raid_device->handle;
1895                         sas_target_priv_data->sas_address = raid_device->wwid;
1896                         sas_target_priv_data->flags |= MPT_TARGET_FLAGS_VOLUME;
1897                         if (ioc->is_warpdrive)
1898                                 sas_target_priv_data->raid_device = raid_device;
1899                         raid_device->starget = starget;
1900                 }
1901                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1902                 return 0;
1903         }
1904
1905         /* PCIe devices */
1906         if (starget->channel == PCIE_CHANNEL) {
1907                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1908                 pcie_device = __mpt3sas_get_pdev_by_idchannel(ioc, starget->id,
1909                         starget->channel);
1910                 if (pcie_device) {
1911                         sas_target_priv_data->handle = pcie_device->handle;
1912                         sas_target_priv_data->sas_address = pcie_device->wwid;
1913                         sas_target_priv_data->port = NULL;
1914                         sas_target_priv_data->pcie_dev = pcie_device;
1915                         pcie_device->starget = starget;
1916                         pcie_device->id = starget->id;
1917                         pcie_device->channel = starget->channel;
1918                         sas_target_priv_data->flags |=
1919                                 MPT_TARGET_FLAGS_PCIE_DEVICE;
1920                         if (pcie_device->fast_path)
1921                                 sas_target_priv_data->flags |=
1922                                         MPT_TARGET_FASTPATH_IO;
1923                 }
1924                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1925                 return 0;
1926         }
1927
1928         /* sas/sata devices */
1929         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1930         rphy = dev_to_rphy(starget->dev.parent);
1931         sas_device = __mpt3sas_get_sdev_by_rphy(ioc, rphy);
1932
1933         if (sas_device) {
1934                 sas_target_priv_data->handle = sas_device->handle;
1935                 sas_target_priv_data->sas_address = sas_device->sas_address;
1936                 sas_target_priv_data->port = sas_device->port;
1937                 sas_target_priv_data->sas_dev = sas_device;
1938                 sas_device->starget = starget;
1939                 sas_device->id = starget->id;
1940                 sas_device->channel = starget->channel;
1941                 if (test_bit(sas_device->handle, ioc->pd_handles))
1942                         sas_target_priv_data->flags |=
1943                             MPT_TARGET_FLAGS_RAID_COMPONENT;
1944                 if (sas_device->fast_path)
1945                         sas_target_priv_data->flags |=
1946                                         MPT_TARGET_FASTPATH_IO;
1947         }
1948         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1949
1950         return 0;
1951 }
1952
1953 /**
1954  * scsih_target_destroy - target destroy routine
1955  * @starget: scsi target struct
1956  */
1957 static void
1958 scsih_target_destroy(struct scsi_target *starget)
1959 {
1960         struct Scsi_Host *shost = dev_to_shost(&starget->dev);
1961         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1962         struct MPT3SAS_TARGET *sas_target_priv_data;
1963         struct _sas_device *sas_device;
1964         struct _raid_device *raid_device;
1965         struct _pcie_device *pcie_device;
1966         unsigned long flags;
1967
1968         sas_target_priv_data = starget->hostdata;
1969         if (!sas_target_priv_data)
1970                 return;
1971
1972         if (starget->channel == RAID_CHANNEL) {
1973                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
1974                 raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,
1975                     starget->channel);
1976                 if (raid_device) {
1977                         raid_device->starget = NULL;
1978                         raid_device->sdev = NULL;
1979                 }
1980                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1981                 goto out;
1982         }
1983
1984         if (starget->channel == PCIE_CHANNEL) {
1985                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1986                 pcie_device = __mpt3sas_get_pdev_from_target(ioc,
1987                                                         sas_target_priv_data);
1988                 if (pcie_device && (pcie_device->starget == starget) &&
1989                         (pcie_device->id == starget->id) &&
1990                         (pcie_device->channel == starget->channel))
1991                         pcie_device->starget = NULL;
1992
1993                 if (pcie_device) {
1994                         /*
1995                          * Corresponding get() is in _scsih_target_alloc()
1996                          */
1997                         sas_target_priv_data->pcie_dev = NULL;
1998                         pcie_device_put(pcie_device);
1999                         pcie_device_put(pcie_device);
2000                 }
2001                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2002                 goto out;
2003         }
2004
2005         spin_lock_irqsave(&ioc->sas_device_lock, flags);
2006         sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);
2007         if (sas_device && (sas_device->starget == starget) &&
2008             (sas_device->id == starget->id) &&
2009             (sas_device->channel == starget->channel))
2010                 sas_device->starget = NULL;
2011
2012         if (sas_device) {
2013                 /*
2014                  * Corresponding get() is in _scsih_target_alloc()
2015                  */
2016                 sas_target_priv_data->sas_dev = NULL;
2017                 sas_device_put(sas_device);
2018
2019                 sas_device_put(sas_device);
2020         }
2021         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2022
2023  out:
2024         kfree(sas_target_priv_data);
2025         starget->hostdata = NULL;
2026 }
2027
2028 /**
2029  * scsih_slave_alloc - device add routine
2030  * @sdev: scsi device struct
2031  *
2032  * Return: 0 if ok. Any other return is assumed to be an error and
2033  * the device is ignored.
2034  */
2035 static int
2036 scsih_slave_alloc(struct scsi_device *sdev)
2037 {
2038         struct Scsi_Host *shost;
2039         struct MPT3SAS_ADAPTER *ioc;
2040         struct MPT3SAS_TARGET *sas_target_priv_data;
2041         struct MPT3SAS_DEVICE *sas_device_priv_data;
2042         struct scsi_target *starget;
2043         struct _raid_device *raid_device;
2044         struct _sas_device *sas_device;
2045         struct _pcie_device *pcie_device;
2046         unsigned long flags;
2047
2048         sas_device_priv_data = kzalloc(sizeof(*sas_device_priv_data),
2049                                        GFP_KERNEL);
2050         if (!sas_device_priv_data)
2051                 return -ENOMEM;
2052
2053         sas_device_priv_data->lun = sdev->lun;
2054         sas_device_priv_data->flags = MPT_DEVICE_FLAGS_INIT;
2055
2056         starget = scsi_target(sdev);
2057         sas_target_priv_data = starget->hostdata;
2058         sas_target_priv_data->num_luns++;
2059         sas_device_priv_data->sas_target = sas_target_priv_data;
2060         sdev->hostdata = sas_device_priv_data;
2061         if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT))
2062                 sdev->no_uld_attach = 1;
2063
2064         shost = dev_to_shost(&starget->dev);
2065         ioc = shost_priv(shost);
2066         if (starget->channel == RAID_CHANNEL) {
2067                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
2068                 raid_device = _scsih_raid_device_find_by_id(ioc,
2069                     starget->id, starget->channel);
2070                 if (raid_device)
2071                         raid_device->sdev = sdev; /* raid is single lun */
2072                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2073         }
2074         if (starget->channel == PCIE_CHANNEL) {
2075                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2076                 pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,
2077                                 sas_target_priv_data->sas_address);
2078                 if (pcie_device && (pcie_device->starget == NULL)) {
2079                         sdev_printk(KERN_INFO, sdev,
2080                             "%s : pcie_device->starget set to starget @ %d\n",
2081                             __func__, __LINE__);
2082                         pcie_device->starget = starget;
2083                 }
2084
2085                 if (pcie_device)
2086                         pcie_device_put(pcie_device);
2087                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2088
2089         } else  if (!(sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)) {
2090                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
2091                 sas_device = __mpt3sas_get_sdev_by_addr(ioc,
2092                     sas_target_priv_data->sas_address,
2093                     sas_target_priv_data->port);
2094                 if (sas_device && (sas_device->starget == NULL)) {
2095                         sdev_printk(KERN_INFO, sdev,
2096                         "%s : sas_device->starget set to starget @ %d\n",
2097                              __func__, __LINE__);
2098                         sas_device->starget = starget;
2099                 }
2100
2101                 if (sas_device)
2102                         sas_device_put(sas_device);
2103
2104                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2105         }
2106
2107         return 0;
2108 }
2109
2110 /**
2111  * scsih_slave_destroy - device destroy routine
2112  * @sdev: scsi device struct
2113  */
2114 static void
2115 scsih_slave_destroy(struct scsi_device *sdev)
2116 {
2117         struct MPT3SAS_TARGET *sas_target_priv_data;
2118         struct scsi_target *starget;
2119         struct Scsi_Host *shost;
2120         struct MPT3SAS_ADAPTER *ioc;
2121         struct _sas_device *sas_device;
2122         struct _pcie_device *pcie_device;
2123         unsigned long flags;
2124
2125         if (!sdev->hostdata)
2126                 return;
2127
2128         starget = scsi_target(sdev);
2129         sas_target_priv_data = starget->hostdata;
2130         sas_target_priv_data->num_luns--;
2131
2132         shost = dev_to_shost(&starget->dev);
2133         ioc = shost_priv(shost);
2134
2135         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
2136                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2137                 pcie_device = __mpt3sas_get_pdev_from_target(ioc,
2138                                 sas_target_priv_data);
2139                 if (pcie_device && !sas_target_priv_data->num_luns)
2140                         pcie_device->starget = NULL;
2141
2142                 if (pcie_device)
2143                         pcie_device_put(pcie_device);
2144
2145                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2146
2147         } else if (!(sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)) {
2148                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
2149                 sas_device = __mpt3sas_get_sdev_from_target(ioc,
2150                                 sas_target_priv_data);
2151                 if (sas_device && !sas_target_priv_data->num_luns)
2152                         sas_device->starget = NULL;
2153
2154                 if (sas_device)
2155                         sas_device_put(sas_device);
2156                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2157         }
2158
2159         kfree(sdev->hostdata);
2160         sdev->hostdata = NULL;
2161 }
2162
2163 /**
2164  * _scsih_display_sata_capabilities - sata capabilities
2165  * @ioc: per adapter object
2166  * @handle: device handle
2167  * @sdev: scsi device struct
2168  */
2169 static void
2170 _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,
2171         u16 handle, struct scsi_device *sdev)
2172 {
2173         Mpi2ConfigReply_t mpi_reply;
2174         Mpi2SasDevicePage0_t sas_device_pg0;
2175         u32 ioc_status;
2176         u16 flags;
2177         u32 device_info;
2178
2179         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
2180             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
2181                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2182                         __FILE__, __LINE__, __func__);
2183                 return;
2184         }
2185
2186         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
2187             MPI2_IOCSTATUS_MASK;
2188         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
2189                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2190                         __FILE__, __LINE__, __func__);
2191                 return;
2192         }
2193
2194         flags = le16_to_cpu(sas_device_pg0.Flags);
2195         device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
2196
2197         sdev_printk(KERN_INFO, sdev,
2198             "atapi(%s), ncq(%s), asyn_notify(%s), smart(%s), fua(%s), "
2199             "sw_preserve(%s)\n",
2200             (device_info & MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE) ? "y" : "n",
2201             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_NCQ_SUPPORTED) ? "y" : "n",
2202             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_ASYNCHRONOUS_NOTIFY) ? "y" :
2203             "n",
2204             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_SMART_SUPPORTED) ? "y" : "n",
2205             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_FUA_SUPPORTED) ? "y" : "n",
2206             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_SW_PRESERVE) ? "y" : "n");
2207 }
2208
2209 /*
2210  * raid transport support -
2211  * Enabled for SLES11 and newer, in older kernels the driver will panic when
2212  * unloading the driver followed by a load - I believe that the subroutine
2213  * raid_class_release() is not cleaning up properly.
2214  */
2215
2216 /**
2217  * scsih_is_raid - return boolean indicating device is raid volume
2218  * @dev: the device struct object
2219  */
2220 static int
2221 scsih_is_raid(struct device *dev)
2222 {
2223         struct scsi_device *sdev = to_scsi_device(dev);
2224         struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);
2225
2226         if (ioc->is_warpdrive)
2227                 return 0;
2228         return (sdev->channel == RAID_CHANNEL) ? 1 : 0;
2229 }
2230
2231 static int
2232 scsih_is_nvme(struct device *dev)
2233 {
2234         struct scsi_device *sdev = to_scsi_device(dev);
2235
2236         return (sdev->channel == PCIE_CHANNEL) ? 1 : 0;
2237 }
2238
2239 /**
2240  * scsih_get_resync - get raid volume resync percent complete
2241  * @dev: the device struct object
2242  */
2243 static void
2244 scsih_get_resync(struct device *dev)
2245 {
2246         struct scsi_device *sdev = to_scsi_device(dev);
2247         struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);
2248         static struct _raid_device *raid_device;
2249         unsigned long flags;
2250         Mpi2RaidVolPage0_t vol_pg0;
2251         Mpi2ConfigReply_t mpi_reply;
2252         u32 volume_status_flags;
2253         u8 percent_complete;
2254         u16 handle;
2255
2256         percent_complete = 0;
2257         handle = 0;
2258         if (ioc->is_warpdrive)
2259                 goto out;
2260
2261         spin_lock_irqsave(&ioc->raid_device_lock, flags);
2262         raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,
2263             sdev->channel);
2264         if (raid_device) {
2265                 handle = raid_device->handle;
2266                 percent_complete = raid_device->percent_complete;
2267         }
2268         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2269
2270         if (!handle)
2271                 goto out;
2272
2273         if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,
2274              MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
2275              sizeof(Mpi2RaidVolPage0_t))) {
2276                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2277                         __FILE__, __LINE__, __func__);
2278                 percent_complete = 0;
2279                 goto out;
2280         }
2281
2282         volume_status_flags = le32_to_cpu(vol_pg0.VolumeStatusFlags);
2283         if (!(volume_status_flags &
2284             MPI2_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS))
2285                 percent_complete = 0;
2286
2287  out:
2288
2289         switch (ioc->hba_mpi_version_belonged) {
2290         case MPI2_VERSION:
2291                 raid_set_resync(mpt2sas_raid_template, dev, percent_complete);
2292                 break;
2293         case MPI25_VERSION:
2294         case MPI26_VERSION:
2295                 raid_set_resync(mpt3sas_raid_template, dev, percent_complete);
2296                 break;
2297         }
2298 }
2299
2300 /**
2301  * scsih_get_state - get raid volume level
2302  * @dev: the device struct object
2303  */
2304 static void
2305 scsih_get_state(struct device *dev)
2306 {
2307         struct scsi_device *sdev = to_scsi_device(dev);
2308         struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);
2309         static struct _raid_device *raid_device;
2310         unsigned long flags;
2311         Mpi2RaidVolPage0_t vol_pg0;
2312         Mpi2ConfigReply_t mpi_reply;
2313         u32 volstate;
2314         enum raid_state state = RAID_STATE_UNKNOWN;
2315         u16 handle = 0;
2316
2317         spin_lock_irqsave(&ioc->raid_device_lock, flags);
2318         raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,
2319             sdev->channel);
2320         if (raid_device)
2321                 handle = raid_device->handle;
2322         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2323
2324         if (!raid_device)
2325                 goto out;
2326
2327         if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,
2328              MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
2329              sizeof(Mpi2RaidVolPage0_t))) {
2330                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2331                         __FILE__, __LINE__, __func__);
2332                 goto out;
2333         }
2334
2335         volstate = le32_to_cpu(vol_pg0.VolumeStatusFlags);
2336         if (volstate & MPI2_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS) {
2337                 state = RAID_STATE_RESYNCING;
2338                 goto out;
2339         }
2340
2341         switch (vol_pg0.VolumeState) {
2342         case MPI2_RAID_VOL_STATE_OPTIMAL:
2343         case MPI2_RAID_VOL_STATE_ONLINE:
2344                 state = RAID_STATE_ACTIVE;
2345                 break;
2346         case  MPI2_RAID_VOL_STATE_DEGRADED:
2347                 state = RAID_STATE_DEGRADED;
2348                 break;
2349         case MPI2_RAID_VOL_STATE_FAILED:
2350         case MPI2_RAID_VOL_STATE_MISSING:
2351                 state = RAID_STATE_OFFLINE;
2352                 break;
2353         }
2354  out:
2355         switch (ioc->hba_mpi_version_belonged) {
2356         case MPI2_VERSION:
2357                 raid_set_state(mpt2sas_raid_template, dev, state);
2358                 break;
2359         case MPI25_VERSION:
2360         case MPI26_VERSION:
2361                 raid_set_state(mpt3sas_raid_template, dev, state);
2362                 break;
2363         }
2364 }
2365
2366 /**
2367  * _scsih_set_level - set raid level
2368  * @ioc: ?
2369  * @sdev: scsi device struct
2370  * @volume_type: volume type
2371  */
2372 static void
2373 _scsih_set_level(struct MPT3SAS_ADAPTER *ioc,
2374         struct scsi_device *sdev, u8 volume_type)
2375 {
2376         enum raid_level level = RAID_LEVEL_UNKNOWN;
2377
2378         switch (volume_type) {
2379         case MPI2_RAID_VOL_TYPE_RAID0:
2380                 level = RAID_LEVEL_0;
2381                 break;
2382         case MPI2_RAID_VOL_TYPE_RAID10:
2383                 level = RAID_LEVEL_10;
2384                 break;
2385         case MPI2_RAID_VOL_TYPE_RAID1E:
2386                 level = RAID_LEVEL_1E;
2387                 break;
2388         case MPI2_RAID_VOL_TYPE_RAID1:
2389                 level = RAID_LEVEL_1;
2390                 break;
2391         }
2392
2393         switch (ioc->hba_mpi_version_belonged) {
2394         case MPI2_VERSION:
2395                 raid_set_level(mpt2sas_raid_template,
2396                         &sdev->sdev_gendev, level);
2397                 break;
2398         case MPI25_VERSION:
2399         case MPI26_VERSION:
2400                 raid_set_level(mpt3sas_raid_template,
2401                         &sdev->sdev_gendev, level);
2402                 break;
2403         }
2404 }
2405
2406
2407 /**
2408  * _scsih_get_volume_capabilities - volume capabilities
2409  * @ioc: per adapter object
2410  * @raid_device: the raid_device object
2411  *
2412  * Return: 0 for success, else 1
2413  */
2414 static int
2415 _scsih_get_volume_capabilities(struct MPT3SAS_ADAPTER *ioc,
2416         struct _raid_device *raid_device)
2417 {
2418         Mpi2RaidVolPage0_t *vol_pg0;
2419         Mpi2RaidPhysDiskPage0_t pd_pg0;
2420         Mpi2SasDevicePage0_t sas_device_pg0;
2421         Mpi2ConfigReply_t mpi_reply;
2422         u16 sz;
2423         u8 num_pds;
2424
2425         if ((mpt3sas_config_get_number_pds(ioc, raid_device->handle,
2426             &num_pds)) || !num_pds) {
2427                 dfailprintk(ioc,
2428                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2429                                      __FILE__, __LINE__, __func__));
2430                 return 1;
2431         }
2432
2433         raid_device->num_pds = num_pds;
2434         sz = offsetof(Mpi2RaidVolPage0_t, PhysDisk) + (num_pds *
2435             sizeof(Mpi2RaidVol0PhysDisk_t));
2436         vol_pg0 = kzalloc(sz, GFP_KERNEL);
2437         if (!vol_pg0) {
2438                 dfailprintk(ioc,
2439                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2440                                      __FILE__, __LINE__, __func__));
2441                 return 1;
2442         }
2443
2444         if ((mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, vol_pg0,
2445              MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, raid_device->handle, sz))) {
2446                 dfailprintk(ioc,
2447                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2448                                      __FILE__, __LINE__, __func__));
2449                 kfree(vol_pg0);
2450                 return 1;
2451         }
2452
2453         raid_device->volume_type = vol_pg0->VolumeType;
2454
2455         /* figure out what the underlying devices are by
2456          * obtaining the device_info bits for the 1st device
2457          */
2458         if (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
2459             &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_PHYSDISKNUM,
2460             vol_pg0->PhysDisk[0].PhysDiskNum))) {
2461                 if (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
2462                     &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
2463                     le16_to_cpu(pd_pg0.DevHandle)))) {
2464                         raid_device->device_info =
2465                             le32_to_cpu(sas_device_pg0.DeviceInfo);
2466                 }
2467         }
2468
2469         kfree(vol_pg0);
2470         return 0;
2471 }
2472
2473 /**
2474  * _scsih_enable_tlr - setting TLR flags
2475  * @ioc: per adapter object
2476  * @sdev: scsi device struct
2477  *
2478  * Enabling Transaction Layer Retries for tape devices when
2479  * vpd page 0x90 is present
2480  *
2481  */
2482 static void
2483 _scsih_enable_tlr(struct MPT3SAS_ADAPTER *ioc, struct scsi_device *sdev)
2484 {
2485
2486         /* only for TAPE */
2487         if (sdev->type != TYPE_TAPE)
2488                 return;
2489
2490         if (!(ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR))
2491                 return;
2492
2493         sas_enable_tlr(sdev);
2494         sdev_printk(KERN_INFO, sdev, "TLR %s\n",
2495             sas_is_tlr_enabled(sdev) ? "Enabled" : "Disabled");
2496         return;
2497
2498 }
2499
2500 /**
2501  * scsih_slave_configure - device configure routine.
2502  * @sdev: scsi device struct
2503  *
2504  * Return: 0 if ok. Any other return is assumed to be an error and
2505  * the device is ignored.
2506  */
2507 static int
2508 scsih_slave_configure(struct scsi_device *sdev)
2509 {
2510         struct Scsi_Host *shost = sdev->host;
2511         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
2512         struct MPT3SAS_DEVICE *sas_device_priv_data;
2513         struct MPT3SAS_TARGET *sas_target_priv_data;
2514         struct _sas_device *sas_device;
2515         struct _pcie_device *pcie_device;
2516         struct _raid_device *raid_device;
2517         unsigned long flags;
2518         int qdepth;
2519         u8 ssp_target = 0;
2520         char *ds = "";
2521         char *r_level = "";
2522         u16 handle, volume_handle = 0;
2523         u64 volume_wwid = 0;
2524
2525         qdepth = 1;
2526         sas_device_priv_data = sdev->hostdata;
2527         sas_device_priv_data->configured_lun = 1;
2528         sas_device_priv_data->flags &= ~MPT_DEVICE_FLAGS_INIT;
2529         sas_target_priv_data = sas_device_priv_data->sas_target;
2530         handle = sas_target_priv_data->handle;
2531
2532         /* raid volume handling */
2533         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME) {
2534
2535                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
2536                 raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
2537                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2538                 if (!raid_device) {
2539                         dfailprintk(ioc,
2540                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2541                                              __FILE__, __LINE__, __func__));
2542                         return 1;
2543                 }
2544
2545                 if (_scsih_get_volume_capabilities(ioc, raid_device)) {
2546                         dfailprintk(ioc,
2547                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2548                                              __FILE__, __LINE__, __func__));
2549                         return 1;
2550                 }
2551
2552                 /*
2553                  * WARPDRIVE: Initialize the required data for Direct IO
2554                  */
2555                 mpt3sas_init_warpdrive_properties(ioc, raid_device);
2556
2557                 /* RAID Queue Depth Support
2558                  * IS volume = underlying qdepth of drive type, either
2559                  *    MPT3SAS_SAS_QUEUE_DEPTH or MPT3SAS_SATA_QUEUE_DEPTH
2560                  * IM/IME/R10 = 128 (MPT3SAS_RAID_QUEUE_DEPTH)
2561                  */
2562                 if (raid_device->device_info &
2563                     MPI2_SAS_DEVICE_INFO_SSP_TARGET) {
2564                         qdepth = MPT3SAS_SAS_QUEUE_DEPTH;
2565                         ds = "SSP";
2566                 } else {
2567                         qdepth = MPT3SAS_SATA_QUEUE_DEPTH;
2568                         if (raid_device->device_info &
2569                             MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
2570                                 ds = "SATA";
2571                         else
2572                                 ds = "STP";
2573                 }
2574
2575                 switch (raid_device->volume_type) {
2576                 case MPI2_RAID_VOL_TYPE_RAID0:
2577                         r_level = "RAID0";
2578                         break;
2579                 case MPI2_RAID_VOL_TYPE_RAID1E:
2580                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2581                         if (ioc->manu_pg10.OEMIdentifier &&
2582                             (le32_to_cpu(ioc->manu_pg10.GenericFlags0) &
2583                             MFG10_GF0_R10_DISPLAY) &&
2584                             !(raid_device->num_pds % 2))
2585                                 r_level = "RAID10";
2586                         else
2587                                 r_level = "RAID1E";
2588                         break;
2589                 case MPI2_RAID_VOL_TYPE_RAID1:
2590                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2591                         r_level = "RAID1";
2592                         break;
2593                 case MPI2_RAID_VOL_TYPE_RAID10:
2594                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2595                         r_level = "RAID10";
2596                         break;
2597                 case MPI2_RAID_VOL_TYPE_UNKNOWN:
2598                 default:
2599                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2600                         r_level = "RAIDX";
2601                         break;
2602                 }
2603
2604                 if (!ioc->hide_ir_msg)
2605                         sdev_printk(KERN_INFO, sdev,
2606                            "%s: handle(0x%04x), wwid(0x%016llx),"
2607                             " pd_count(%d), type(%s)\n",
2608                             r_level, raid_device->handle,
2609                             (unsigned long long)raid_device->wwid,
2610                             raid_device->num_pds, ds);
2611
2612                 if (shost->max_sectors > MPT3SAS_RAID_MAX_SECTORS) {
2613                         blk_queue_max_hw_sectors(sdev->request_queue,
2614                                                 MPT3SAS_RAID_MAX_SECTORS);
2615                         sdev_printk(KERN_INFO, sdev,
2616                                         "Set queue's max_sector to: %u\n",
2617                                                 MPT3SAS_RAID_MAX_SECTORS);
2618                 }
2619
2620                 mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2621
2622                 /* raid transport support */
2623                 if (!ioc->is_warpdrive)
2624                         _scsih_set_level(ioc, sdev, raid_device->volume_type);
2625                 return 0;
2626         }
2627
2628         /* non-raid handling */
2629         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT) {
2630                 if (mpt3sas_config_get_volume_handle(ioc, handle,
2631                     &volume_handle)) {
2632                         dfailprintk(ioc,
2633                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2634                                              __FILE__, __LINE__, __func__));
2635                         return 1;
2636                 }
2637                 if (volume_handle && mpt3sas_config_get_volume_wwid(ioc,
2638                     volume_handle, &volume_wwid)) {
2639                         dfailprintk(ioc,
2640                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2641                                              __FILE__, __LINE__, __func__));
2642                         return 1;
2643                 }
2644         }
2645
2646         /* PCIe handling */
2647         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
2648                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2649                 pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,
2650                                 sas_device_priv_data->sas_target->sas_address);
2651                 if (!pcie_device) {
2652                         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2653                         dfailprintk(ioc,
2654                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2655                                              __FILE__, __LINE__, __func__));
2656                         return 1;
2657                 }
2658
2659                 qdepth = ioc->max_nvme_qd;
2660                 ds = "NVMe";
2661                 sdev_printk(KERN_INFO, sdev,
2662                         "%s: handle(0x%04x), wwid(0x%016llx), port(%d)\n",
2663                         ds, handle, (unsigned long long)pcie_device->wwid,
2664                         pcie_device->port_num);
2665                 if (pcie_device->enclosure_handle != 0)
2666                         sdev_printk(KERN_INFO, sdev,
2667                         "%s: enclosure logical id(0x%016llx), slot(%d)\n",
2668                         ds,
2669                         (unsigned long long)pcie_device->enclosure_logical_id,
2670                         pcie_device->slot);
2671                 if (pcie_device->connector_name[0] != '\0')
2672                         sdev_printk(KERN_INFO, sdev,
2673                                 "%s: enclosure level(0x%04x),"
2674                                 "connector name( %s)\n", ds,
2675                                 pcie_device->enclosure_level,
2676                                 pcie_device->connector_name);
2677
2678                 if (pcie_device->nvme_mdts)
2679                         blk_queue_max_hw_sectors(sdev->request_queue,
2680                                         pcie_device->nvme_mdts/512);
2681
2682                 pcie_device_put(pcie_device);
2683                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2684                 mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2685                 /* Enable QUEUE_FLAG_NOMERGES flag, so that IOs won't be
2686                  ** merged and can eliminate holes created during merging
2687                  ** operation.
2688                  **/
2689                 blk_queue_flag_set(QUEUE_FLAG_NOMERGES,
2690                                 sdev->request_queue);
2691                 blk_queue_virt_boundary(sdev->request_queue,
2692                                 ioc->page_size - 1);
2693                 return 0;
2694         }
2695
2696         spin_lock_irqsave(&ioc->sas_device_lock, flags);
2697         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
2698            sas_device_priv_data->sas_target->sas_address,
2699            sas_device_priv_data->sas_target->port);
2700         if (!sas_device) {
2701                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2702                 dfailprintk(ioc,
2703                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2704                                      __FILE__, __LINE__, __func__));
2705                 return 1;
2706         }
2707
2708         sas_device->volume_handle = volume_handle;
2709         sas_device->volume_wwid = volume_wwid;
2710         if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET) {
2711                 qdepth = (sas_device->port_type > 1) ?
2712                         ioc->max_wideport_qd : ioc->max_narrowport_qd;
2713                 ssp_target = 1;
2714                 if (sas_device->device_info &
2715                                 MPI2_SAS_DEVICE_INFO_SEP) {
2716                         sdev_printk(KERN_WARNING, sdev,
2717                         "set ignore_delay_remove for handle(0x%04x)\n",
2718                         sas_device_priv_data->sas_target->handle);
2719                         sas_device_priv_data->ignore_delay_remove = 1;
2720                         ds = "SES";
2721                 } else
2722                         ds = "SSP";
2723         } else {
2724                 qdepth = ioc->max_sata_qd;
2725                 if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_STP_TARGET)
2726                         ds = "STP";
2727                 else if (sas_device->device_info &
2728                     MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
2729                         ds = "SATA";
2730         }
2731
2732         sdev_printk(KERN_INFO, sdev, "%s: handle(0x%04x), " \
2733             "sas_addr(0x%016llx), phy(%d), device_name(0x%016llx)\n",
2734             ds, handle, (unsigned long long)sas_device->sas_address,
2735             sas_device->phy, (unsigned long long)sas_device->device_name);
2736
2737         _scsih_display_enclosure_chassis_info(NULL, sas_device, sdev, NULL);
2738
2739         sas_device_put(sas_device);
2740         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2741
2742         if (!ssp_target)
2743                 _scsih_display_sata_capabilities(ioc, handle, sdev);
2744
2745
2746         mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2747
2748         if (ssp_target) {
2749                 sas_read_port_mode_page(sdev);
2750                 _scsih_enable_tlr(ioc, sdev);
2751         }
2752
2753         return 0;
2754 }
2755
2756 /**
2757  * scsih_bios_param - fetch head, sector, cylinder info for a disk
2758  * @sdev: scsi device struct
2759  * @bdev: pointer to block device context
2760  * @capacity: device size (in 512 byte sectors)
2761  * @params: three element array to place output:
2762  *              params[0] number of heads (max 255)
2763  *              params[1] number of sectors (max 63)
2764  *              params[2] number of cylinders
2765  */
2766 static int
2767 scsih_bios_param(struct scsi_device *sdev, struct block_device *bdev,
2768         sector_t capacity, int params[])
2769 {
2770         int             heads;
2771         int             sectors;
2772         sector_t        cylinders;
2773         ulong           dummy;
2774
2775         heads = 64;
2776         sectors = 32;
2777
2778         dummy = heads * sectors;
2779         cylinders = capacity;
2780         sector_div(cylinders, dummy);
2781
2782         /*
2783          * Handle extended translation size for logical drives
2784          * > 1Gb
2785          */
2786         if ((ulong)capacity >= 0x200000) {
2787                 heads = 255;
2788                 sectors = 63;
2789                 dummy = heads * sectors;
2790                 cylinders = capacity;
2791                 sector_div(cylinders, dummy);
2792         }
2793
2794         /* return result */
2795         params[0] = heads;
2796         params[1] = sectors;
2797         params[2] = cylinders;
2798
2799         return 0;
2800 }
2801
2802 /**
2803  * _scsih_response_code - translation of device response code
2804  * @ioc: per adapter object
2805  * @response_code: response code returned by the device
2806  */
2807 static void
2808 _scsih_response_code(struct MPT3SAS_ADAPTER *ioc, u8 response_code)
2809 {
2810         char *desc;
2811
2812         switch (response_code) {
2813         case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
2814                 desc = "task management request completed";
2815                 break;
2816         case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
2817                 desc = "invalid frame";
2818                 break;
2819         case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
2820                 desc = "task management request not supported";
2821                 break;
2822         case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
2823                 desc = "task management request failed";
2824                 break;
2825         case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
2826                 desc = "task management request succeeded";
2827                 break;
2828         case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
2829                 desc = "invalid lun";
2830                 break;
2831         case 0xA:
2832                 desc = "overlapped tag attempted";
2833                 break;
2834         case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
2835                 desc = "task queued, however not sent to target";
2836                 break;
2837         default:
2838                 desc = "unknown";
2839                 break;
2840         }
2841         ioc_warn(ioc, "response_code(0x%01x): %s\n", response_code, desc);
2842 }
2843
2844 /**
2845  * _scsih_tm_done - tm completion routine
2846  * @ioc: per adapter object
2847  * @smid: system request message index
2848  * @msix_index: MSIX table index supplied by the OS
2849  * @reply: reply message frame(lower 32bit addr)
2850  * Context: none.
2851  *
2852  * The callback handler when using scsih_issue_tm.
2853  *
2854  * Return: 1 meaning mf should be freed from _base_interrupt
2855  *         0 means the mf is freed from this function.
2856  */
2857 static u8
2858 _scsih_tm_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
2859 {
2860         MPI2DefaultReply_t *mpi_reply;
2861
2862         if (ioc->tm_cmds.status == MPT3_CMD_NOT_USED)
2863                 return 1;
2864         if (ioc->tm_cmds.smid != smid)
2865                 return 1;
2866         ioc->tm_cmds.status |= MPT3_CMD_COMPLETE;
2867         mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);
2868         if (mpi_reply) {
2869                 memcpy(ioc->tm_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
2870                 ioc->tm_cmds.status |= MPT3_CMD_REPLY_VALID;
2871         }
2872         ioc->tm_cmds.status &= ~MPT3_CMD_PENDING;
2873         complete(&ioc->tm_cmds.done);
2874         return 1;
2875 }
2876
2877 /**
2878  * mpt3sas_scsih_set_tm_flag - set per target tm_busy
2879  * @ioc: per adapter object
2880  * @handle: device handle
2881  *
2882  * During taskmangement request, we need to freeze the device queue.
2883  */
2884 void
2885 mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
2886 {
2887         struct MPT3SAS_DEVICE *sas_device_priv_data;
2888         struct scsi_device *sdev;
2889         u8 skip = 0;
2890
2891         shost_for_each_device(sdev, ioc->shost) {
2892                 if (skip)
2893                         continue;
2894                 sas_device_priv_data = sdev->hostdata;
2895                 if (!sas_device_priv_data)
2896                         continue;
2897                 if (sas_device_priv_data->sas_target->handle == handle) {
2898                         sas_device_priv_data->sas_target->tm_busy = 1;
2899                         skip = 1;
2900                         ioc->ignore_loginfos = 1;
2901                 }
2902         }
2903 }
2904
2905 /**
2906  * mpt3sas_scsih_clear_tm_flag - clear per target tm_busy
2907  * @ioc: per adapter object
2908  * @handle: device handle
2909  *
2910  * During taskmangement request, we need to freeze the device queue.
2911  */
2912 void
2913 mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
2914 {
2915         struct MPT3SAS_DEVICE *sas_device_priv_data;
2916         struct scsi_device *sdev;
2917         u8 skip = 0;
2918
2919         shost_for_each_device(sdev, ioc->shost) {
2920                 if (skip)
2921                         continue;
2922                 sas_device_priv_data = sdev->hostdata;
2923                 if (!sas_device_priv_data)
2924                         continue;
2925                 if (sas_device_priv_data->sas_target->handle == handle) {
2926                         sas_device_priv_data->sas_target->tm_busy = 0;
2927                         skip = 1;
2928                         ioc->ignore_loginfos = 0;
2929                 }
2930         }
2931 }
2932
2933 /**
2934  * scsih_tm_cmd_map_status - map the target reset & LUN reset TM status
2935  * @ioc: per adapter object
2936  * @channel: the channel assigned by the OS
2937  * @id: the id assigned by the OS
2938  * @lun: lun number
2939  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
2940  * @smid_task: smid assigned to the task
2941  *
2942  * Look whether TM has aborted the timed out SCSI command, if
2943  * TM has aborted the IO then return SUCCESS else return FAILED.
2944  */
2945 static int
2946 scsih_tm_cmd_map_status(struct MPT3SAS_ADAPTER *ioc, uint channel,
2947         uint id, uint lun, u8 type, u16 smid_task)
2948 {
2949
2950         if (smid_task <= ioc->shost->can_queue) {
2951                 switch (type) {
2952                 case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
2953                         if (!(_scsih_scsi_lookup_find_by_target(ioc,
2954                             id, channel)))
2955                                 return SUCCESS;
2956                         break;
2957                 case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
2958                 case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
2959                         if (!(_scsih_scsi_lookup_find_by_lun(ioc, id,
2960                             lun, channel)))
2961                                 return SUCCESS;
2962                         break;
2963                 default:
2964                         return SUCCESS;
2965                 }
2966         } else if (smid_task == ioc->scsih_cmds.smid) {
2967                 if ((ioc->scsih_cmds.status & MPT3_CMD_COMPLETE) ||
2968                     (ioc->scsih_cmds.status & MPT3_CMD_NOT_USED))
2969                         return SUCCESS;
2970         } else if (smid_task == ioc->ctl_cmds.smid) {
2971                 if ((ioc->ctl_cmds.status & MPT3_CMD_COMPLETE) ||
2972                     (ioc->ctl_cmds.status & MPT3_CMD_NOT_USED))
2973                         return SUCCESS;
2974         }
2975
2976         return FAILED;
2977 }
2978
2979 /**
2980  * scsih_tm_post_processing - post processing of target & LUN reset
2981  * @ioc: per adapter object
2982  * @handle: device handle
2983  * @channel: the channel assigned by the OS
2984  * @id: the id assigned by the OS
2985  * @lun: lun number
2986  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
2987  * @smid_task: smid assigned to the task
2988  *
2989  * Post processing of target & LUN reset. Due to interrupt latency
2990  * issue it possible that interrupt for aborted IO might not be
2991  * received yet. So before returning failure status, poll the
2992  * reply descriptor pools for the reply of timed out SCSI command.
2993  * Return FAILED status if reply for timed out is not received
2994  * otherwise return SUCCESS.
2995  */
2996 static int
2997 scsih_tm_post_processing(struct MPT3SAS_ADAPTER *ioc, u16 handle,
2998         uint channel, uint id, uint lun, u8 type, u16 smid_task)
2999 {
3000         int rc;
3001
3002         rc = scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);
3003         if (rc == SUCCESS)
3004                 return rc;
3005
3006         ioc_info(ioc,
3007             "Poll ReplyDescriptor queues for completion of"
3008             " smid(%d), task_type(0x%02x), handle(0x%04x)\n",
3009             smid_task, type, handle);
3010
3011         /*
3012          * Due to interrupt latency issues, driver may receive interrupt for
3013          * TM first and then for aborted SCSI IO command. So, poll all the
3014          * ReplyDescriptor pools before returning the FAILED status to SML.
3015          */
3016         mpt3sas_base_mask_interrupts(ioc);
3017         mpt3sas_base_sync_reply_irqs(ioc, 1);
3018         mpt3sas_base_unmask_interrupts(ioc);
3019
3020         return scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);
3021 }
3022
3023 /**
3024  * mpt3sas_scsih_issue_tm - main routine for sending tm requests
3025  * @ioc: per adapter struct
3026  * @handle: device handle
3027  * @channel: the channel assigned by the OS
3028  * @id: the id assigned by the OS
3029  * @lun: lun number
3030  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
3031  * @smid_task: smid assigned to the task
3032  * @msix_task: MSIX table index supplied by the OS
3033  * @timeout: timeout in seconds
3034  * @tr_method: Target Reset Method
3035  * Context: user
3036  *
3037  * A generic API for sending task management requests to firmware.
3038  *
3039  * The callback index is set inside `ioc->tm_cb_idx`.
3040  * The caller is responsible to check for outstanding commands.
3041  *
3042  * Return: SUCCESS or FAILED.
3043  */
3044 int
3045 mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
3046         uint id, u64 lun, u8 type, u16 smid_task, u16 msix_task,
3047         u8 timeout, u8 tr_method)
3048 {
3049         Mpi2SCSITaskManagementRequest_t *mpi_request;
3050         Mpi2SCSITaskManagementReply_t *mpi_reply;
3051         Mpi25SCSIIORequest_t *request;
3052         u16 smid = 0;
3053         u32 ioc_state;
3054         int rc;
3055         u8 issue_reset = 0;
3056
3057         lockdep_assert_held(&ioc->tm_cmds.mutex);
3058
3059         if (ioc->tm_cmds.status != MPT3_CMD_NOT_USED) {
3060                 ioc_info(ioc, "%s: tm_cmd busy!!!\n", __func__);
3061                 return FAILED;
3062         }
3063
3064         if (ioc->shost_recovery || ioc->remove_host ||
3065             ioc->pci_error_recovery) {
3066                 ioc_info(ioc, "%s: host reset in progress!\n", __func__);
3067                 return FAILED;
3068         }
3069
3070         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
3071         if (ioc_state & MPI2_DOORBELL_USED) {
3072                 dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));
3073                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3074                 return (!rc) ? SUCCESS : FAILED;
3075         }
3076
3077         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
3078                 mpt3sas_print_fault_code(ioc, ioc_state &
3079                     MPI2_DOORBELL_DATA_MASK);
3080                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3081                 return (!rc) ? SUCCESS : FAILED;
3082         } else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
3083             MPI2_IOC_STATE_COREDUMP) {
3084                 mpt3sas_print_coredump_info(ioc, ioc_state &
3085                     MPI2_DOORBELL_DATA_MASK);
3086                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3087                 return (!rc) ? SUCCESS : FAILED;
3088         }
3089
3090         smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_cb_idx);
3091         if (!smid) {
3092                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
3093                 return FAILED;
3094         }
3095
3096         dtmprintk(ioc,
3097                   ioc_info(ioc, "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%x)\n",
3098                            handle, type, smid_task, timeout, tr_method));
3099         ioc->tm_cmds.status = MPT3_CMD_PENDING;
3100         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
3101         ioc->tm_cmds.smid = smid;
3102         memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
3103         memset(ioc->tm_cmds.reply, 0, sizeof(Mpi2SCSITaskManagementReply_t));
3104         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3105         mpi_request->DevHandle = cpu_to_le16(handle);
3106         mpi_request->TaskType = type;
3107         if (type == MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK ||
3108             type == MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK)
3109                 mpi_request->MsgFlags = tr_method;
3110         mpi_request->TaskMID = cpu_to_le16(smid_task);
3111         int_to_scsilun(lun, (struct scsi_lun *)mpi_request->LUN);
3112         mpt3sas_scsih_set_tm_flag(ioc, handle);
3113         init_completion(&ioc->tm_cmds.done);
3114         ioc->put_smid_hi_priority(ioc, smid, msix_task);
3115         wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);
3116         if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {
3117                 mpt3sas_check_cmd_timeout(ioc,
3118                     ioc->tm_cmds.status, mpi_request,
3119                     sizeof(Mpi2SCSITaskManagementRequest_t)/4, issue_reset);
3120                 if (issue_reset) {
3121                         rc = mpt3sas_base_hard_reset_handler(ioc,
3122                                         FORCE_BIG_HAMMER);
3123                         rc = (!rc) ? SUCCESS : FAILED;
3124                         goto out;
3125                 }
3126         }
3127
3128         /* sync IRQs in case those were busy during flush. */
3129         mpt3sas_base_sync_reply_irqs(ioc, 0);
3130
3131         if (ioc->tm_cmds.status & MPT3_CMD_REPLY_VALID) {
3132                 mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);
3133                 mpi_reply = ioc->tm_cmds.reply;
3134                 dtmprintk(ioc,
3135                           ioc_info(ioc, "complete tm: ioc_status(0x%04x), loginfo(0x%08x), term_count(0x%08x)\n",
3136                                    le16_to_cpu(mpi_reply->IOCStatus),
3137                                    le32_to_cpu(mpi_reply->IOCLogInfo),
3138                                    le32_to_cpu(mpi_reply->TerminationCount)));
3139                 if (ioc->logging_level & MPT_DEBUG_TM) {
3140                         _scsih_response_code(ioc, mpi_reply->ResponseCode);
3141                         if (mpi_reply->IOCStatus)
3142                                 _debug_dump_mf(mpi_request,
3143                                     sizeof(Mpi2SCSITaskManagementRequest_t)/4);
3144                 }
3145         }
3146
3147         switch (type) {
3148         case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
3149                 rc = SUCCESS;
3150                 /*
3151                  * If DevHandle filed in smid_task's entry of request pool
3152                  * doesn't match with device handle on which this task abort
3153                  * TM is received then it means that TM has successfully
3154                  * aborted the timed out command. Since smid_task's entry in
3155                  * request pool will be memset to zero once the timed out
3156                  * command is returned to the SML. If the command is not
3157                  * aborted then smid_task’s entry won’t be cleared and it
3158                  * will have same DevHandle value on which this task abort TM
3159                  * is received and driver will return the TM status as FAILED.
3160                  */
3161                 request = mpt3sas_base_get_msg_frame(ioc, smid_task);
3162                 if (le16_to_cpu(request->DevHandle) != handle)
3163                         break;
3164
3165                 ioc_info(ioc, "Task abort tm failed: handle(0x%04x),"
3166                     "timeout(%d) tr_method(0x%x) smid(%d) msix_index(%d)\n",
3167                     handle, timeout, tr_method, smid_task, msix_task);
3168                 rc = FAILED;
3169                 break;
3170
3171         case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
3172         case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
3173         case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
3174                 rc = scsih_tm_post_processing(ioc, handle, channel, id, lun,
3175                     type, smid_task);
3176                 break;
3177         case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
3178                 rc = SUCCESS;
3179                 break;
3180         default:
3181                 rc = FAILED;
3182                 break;
3183         }
3184
3185 out:
3186         mpt3sas_scsih_clear_tm_flag(ioc, handle);
3187         ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
3188         return rc;
3189 }
3190
3191 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
3192                 uint channel, uint id, u64 lun, u8 type, u16 smid_task,
3193                 u16 msix_task, u8 timeout, u8 tr_method)
3194 {
3195         int ret;
3196
3197         mutex_lock(&ioc->tm_cmds.mutex);
3198         ret = mpt3sas_scsih_issue_tm(ioc, handle, channel, id, lun, type,
3199                         smid_task, msix_task, timeout, tr_method);
3200         mutex_unlock(&ioc->tm_cmds.mutex);
3201
3202         return ret;
3203 }
3204
3205 /**
3206  * _scsih_tm_display_info - displays info about the device
3207  * @ioc: per adapter struct
3208  * @scmd: pointer to scsi command object
3209  *
3210  * Called by task management callback handlers.
3211  */
3212 static void
3213 _scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd)
3214 {
3215         struct scsi_target *starget = scmd->device->sdev_target;
3216         struct MPT3SAS_TARGET *priv_target = starget->hostdata;
3217         struct _sas_device *sas_device = NULL;
3218         struct _pcie_device *pcie_device = NULL;
3219         unsigned long flags;
3220         char *device_str = NULL;
3221
3222         if (!priv_target)
3223                 return;
3224         if (ioc->hide_ir_msg)
3225                 device_str = "WarpDrive";
3226         else
3227                 device_str = "volume";
3228
3229         scsi_print_command(scmd);
3230         if (priv_target->flags & MPT_TARGET_FLAGS_VOLUME) {
3231                 starget_printk(KERN_INFO, starget,
3232                         "%s handle(0x%04x), %s wwid(0x%016llx)\n",
3233                         device_str, priv_target->handle,
3234                     device_str, (unsigned long long)priv_target->sas_address);
3235
3236         } else if (priv_target->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
3237                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
3238                 pcie_device = __mpt3sas_get_pdev_from_target(ioc, priv_target);
3239                 if (pcie_device) {
3240                         starget_printk(KERN_INFO, starget,
3241                                 "handle(0x%04x), wwid(0x%016llx), port(%d)\n",
3242                                 pcie_device->handle,
3243                                 (unsigned long long)pcie_device->wwid,
3244                                 pcie_device->port_num);
3245                         if (pcie_device->enclosure_handle != 0)
3246                                 starget_printk(KERN_INFO, starget,
3247                                         "enclosure logical id(0x%016llx), slot(%d)\n",
3248                                         (unsigned long long)
3249                                         pcie_device->enclosure_logical_id,
3250                                         pcie_device->slot);
3251                         if (pcie_device->connector_name[0] != '\0')
3252                                 starget_printk(KERN_INFO, starget,
3253                                         "enclosure level(0x%04x), connector name( %s)\n",
3254                                         pcie_device->enclosure_level,
3255                                         pcie_device->connector_name);
3256                         pcie_device_put(pcie_device);
3257                 }
3258                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
3259
3260         } else {
3261                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
3262                 sas_device = __mpt3sas_get_sdev_from_target(ioc, priv_target);
3263                 if (sas_device) {
3264                         if (priv_target->flags &
3265                             MPT_TARGET_FLAGS_RAID_COMPONENT) {
3266                                 starget_printk(KERN_INFO, starget,
3267                                     "volume handle(0x%04x), "
3268                                     "volume wwid(0x%016llx)\n",
3269                                     sas_device->volume_handle,
3270                                    (unsigned long long)sas_device->volume_wwid);
3271                         }
3272                         starget_printk(KERN_INFO, starget,
3273                             "handle(0x%04x), sas_address(0x%016llx), phy(%d)\n",
3274                             sas_device->handle,
3275                             (unsigned long long)sas_device->sas_address,
3276                             sas_device->phy);
3277
3278                         _scsih_display_enclosure_chassis_info(NULL, sas_device,
3279                             NULL, starget);
3280
3281                         sas_device_put(sas_device);
3282                 }
3283                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
3284         }
3285 }
3286
3287 /**
3288  * scsih_abort - eh threads main abort routine
3289  * @scmd: pointer to scsi command object
3290  *
3291  * Return: SUCCESS if command aborted else FAILED
3292  */
3293 static int
3294 scsih_abort(struct scsi_cmnd *scmd)
3295 {
3296         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3297         struct MPT3SAS_DEVICE *sas_device_priv_data;
3298         struct scsiio_tracker *st = scsi_cmd_priv(scmd);
3299         u16 handle;
3300         int r;
3301
3302         u8 timeout = 30;
3303         struct _pcie_device *pcie_device = NULL;
3304         sdev_printk(KERN_INFO, scmd->device, "attempting task abort!"
3305             "scmd(0x%p), outstanding for %u ms & timeout %u ms\n",
3306             scmd, jiffies_to_msecs(jiffies - scmd->jiffies_at_alloc),
3307             (scsi_cmd_to_rq(scmd)->timeout / HZ) * 1000);
3308         _scsih_tm_display_info(ioc, scmd);
3309
3310         sas_device_priv_data = scmd->device->hostdata;
3311         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3312             ioc->remove_host) {
3313                 sdev_printk(KERN_INFO, scmd->device,
3314                     "device been deleted! scmd(0x%p)\n", scmd);
3315                 scmd->result = DID_NO_CONNECT << 16;
3316                 scsi_done(scmd);
3317                 r = SUCCESS;
3318                 goto out;
3319         }
3320
3321         /* check for completed command */
3322         if (st == NULL || st->cb_idx == 0xFF) {
3323                 sdev_printk(KERN_INFO, scmd->device, "No reference found at "
3324                     "driver, assuming scmd(0x%p) might have completed\n", scmd);
3325                 scmd->result = DID_RESET << 16;
3326                 r = SUCCESS;
3327                 goto out;
3328         }
3329
3330         /* for hidden raid components and volumes this is not supported */
3331         if (sas_device_priv_data->sas_target->flags &
3332             MPT_TARGET_FLAGS_RAID_COMPONENT ||
3333             sas_device_priv_data->sas_target->flags & MPT_TARGET_FLAGS_VOLUME) {
3334                 scmd->result = DID_RESET << 16;
3335                 r = FAILED;
3336                 goto out;
3337         }
3338
3339         mpt3sas_halt_firmware(ioc);
3340
3341         handle = sas_device_priv_data->sas_target->handle;
3342         pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3343         if (pcie_device && (!ioc->tm_custom_handling) &&
3344             (!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))))
3345                 timeout = ioc->nvme_abort_timeout;
3346         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
3347                 scmd->device->id, scmd->device->lun,
3348                 MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
3349                 st->smid, st->msix_io, timeout, 0);
3350         /* Command must be cleared after abort */
3351         if (r == SUCCESS && st->cb_idx != 0xFF)
3352                 r = FAILED;
3353  out:
3354         sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(0x%p)\n",
3355             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3356         if (pcie_device)
3357                 pcie_device_put(pcie_device);
3358         return r;
3359 }
3360
3361 /**
3362  * scsih_dev_reset - eh threads main device reset routine
3363  * @scmd: pointer to scsi command object
3364  *
3365  * Return: SUCCESS if command aborted else FAILED
3366  */
3367 static int
3368 scsih_dev_reset(struct scsi_cmnd *scmd)
3369 {
3370         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3371         struct MPT3SAS_DEVICE *sas_device_priv_data;
3372         struct _sas_device *sas_device = NULL;
3373         struct _pcie_device *pcie_device = NULL;
3374         u16     handle;
3375         u8      tr_method = 0;
3376         u8      tr_timeout = 30;
3377         int r;
3378
3379         struct scsi_target *starget = scmd->device->sdev_target;
3380         struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
3381
3382         sdev_printk(KERN_INFO, scmd->device,
3383             "attempting device reset! scmd(0x%p)\n", scmd);
3384         _scsih_tm_display_info(ioc, scmd);
3385
3386         sas_device_priv_data = scmd->device->hostdata;
3387         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3388             ioc->remove_host) {
3389                 sdev_printk(KERN_INFO, scmd->device,
3390                     "device been deleted! scmd(0x%p)\n", scmd);
3391                 scmd->result = DID_NO_CONNECT << 16;
3392                 scsi_done(scmd);
3393                 r = SUCCESS;
3394                 goto out;
3395         }
3396
3397         /* for hidden raid components obtain the volume_handle */
3398         handle = 0;
3399         if (sas_device_priv_data->sas_target->flags &
3400             MPT_TARGET_FLAGS_RAID_COMPONENT) {
3401                 sas_device = mpt3sas_get_sdev_from_target(ioc,
3402                                 target_priv_data);
3403                 if (sas_device)
3404                         handle = sas_device->volume_handle;
3405         } else
3406                 handle = sas_device_priv_data->sas_target->handle;
3407
3408         if (!handle) {
3409                 scmd->result = DID_RESET << 16;
3410                 r = FAILED;
3411                 goto out;
3412         }
3413
3414         pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3415
3416         if (pcie_device && (!ioc->tm_custom_handling) &&
3417             (!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info)))) {
3418                 tr_timeout = pcie_device->reset_timeout;
3419                 tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
3420         } else
3421                 tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3422
3423         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
3424                 scmd->device->id, scmd->device->lun,
3425                 MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 0,
3426                 tr_timeout, tr_method);
3427         /* Check for busy commands after reset */
3428         if (r == SUCCESS && scsi_device_busy(scmd->device))
3429                 r = FAILED;
3430  out:
3431         sdev_printk(KERN_INFO, scmd->device, "device reset: %s scmd(0x%p)\n",
3432             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3433
3434         if (sas_device)
3435                 sas_device_put(sas_device);
3436         if (pcie_device)
3437                 pcie_device_put(pcie_device);
3438
3439         return r;
3440 }
3441
3442 /**
3443  * scsih_target_reset - eh threads main target reset routine
3444  * @scmd: pointer to scsi command object
3445  *
3446  * Return: SUCCESS if command aborted else FAILED
3447  */
3448 static int
3449 scsih_target_reset(struct scsi_cmnd *scmd)
3450 {
3451         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3452         struct MPT3SAS_DEVICE *sas_device_priv_data;
3453         struct _sas_device *sas_device = NULL;
3454         struct _pcie_device *pcie_device = NULL;
3455         u16     handle;
3456         u8      tr_method = 0;
3457         u8      tr_timeout = 30;
3458         int r;
3459         struct scsi_target *starget = scmd->device->sdev_target;
3460         struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
3461
3462         starget_printk(KERN_INFO, starget,
3463             "attempting target reset! scmd(0x%p)\n", scmd);
3464         _scsih_tm_display_info(ioc, scmd);
3465
3466         sas_device_priv_data = scmd->device->hostdata;
3467         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3468             ioc->remove_host) {
3469                 starget_printk(KERN_INFO, starget,
3470                     "target been deleted! scmd(0x%p)\n", scmd);
3471                 scmd->result = DID_NO_CONNECT << 16;
3472                 scsi_done(scmd);
3473                 r = SUCCESS;
3474                 goto out;
3475         }
3476
3477         /* for hidden raid components obtain the volume_handle */
3478         handle = 0;
3479         if (sas_device_priv_data->sas_target->flags &
3480             MPT_TARGET_FLAGS_RAID_COMPONENT) {
3481                 sas_device = mpt3sas_get_sdev_from_target(ioc,
3482                                 target_priv_data);
3483                 if (sas_device)
3484                         handle = sas_device->volume_handle;
3485         } else
3486                 handle = sas_device_priv_data->sas_target->handle;
3487
3488         if (!handle) {
3489                 scmd->result = DID_RESET << 16;
3490                 r = FAILED;
3491                 goto out;
3492         }
3493
3494         pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3495
3496         if (pcie_device && (!ioc->tm_custom_handling) &&
3497             (!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info)))) {
3498                 tr_timeout = pcie_device->reset_timeout;
3499                 tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
3500         } else
3501                 tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3502         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
3503                 scmd->device->id, 0,
3504                 MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0,
3505             tr_timeout, tr_method);
3506         /* Check for busy commands after reset */
3507         if (r == SUCCESS && atomic_read(&starget->target_busy))
3508                 r = FAILED;
3509  out:
3510         starget_printk(KERN_INFO, starget, "target reset: %s scmd(0x%p)\n",
3511             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3512
3513         if (sas_device)
3514                 sas_device_put(sas_device);
3515         if (pcie_device)
3516                 pcie_device_put(pcie_device);
3517         return r;
3518 }
3519
3520
3521 /**
3522  * scsih_host_reset - eh threads main host reset routine
3523  * @scmd: pointer to scsi command object
3524  *
3525  * Return: SUCCESS if command aborted else FAILED
3526  */
3527 static int
3528 scsih_host_reset(struct scsi_cmnd *scmd)
3529 {
3530         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3531         int r, retval;
3532
3533         ioc_info(ioc, "attempting host reset! scmd(0x%p)\n", scmd);
3534         scsi_print_command(scmd);
3535
3536         if (ioc->is_driver_loading || ioc->remove_host) {
3537                 ioc_info(ioc, "Blocking the host reset\n");
3538                 r = FAILED;
3539                 goto out;
3540         }
3541
3542         retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3543         r = (retval < 0) ? FAILED : SUCCESS;
3544 out:
3545         ioc_info(ioc, "host reset: %s scmd(0x%p)\n",
3546                  r == SUCCESS ? "SUCCESS" : "FAILED", scmd);
3547
3548         return r;
3549 }
3550
3551 /**
3552  * _scsih_fw_event_add - insert and queue up fw_event
3553  * @ioc: per adapter object
3554  * @fw_event: object describing the event
3555  * Context: This function will acquire ioc->fw_event_lock.
3556  *
3557  * This adds the firmware event object into link list, then queues it up to
3558  * be processed from user context.
3559  */
3560 static void
3561 _scsih_fw_event_add(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
3562 {
3563         unsigned long flags;
3564
3565         if (ioc->firmware_event_thread == NULL)
3566                 return;
3567
3568         spin_lock_irqsave(&ioc->fw_event_lock, flags);
3569         fw_event_work_get(fw_event);
3570         INIT_LIST_HEAD(&fw_event->list);
3571         list_add_tail(&fw_event->list, &ioc->fw_event_list);
3572         INIT_WORK(&fw_event->work, _firmware_event_work);
3573         fw_event_work_get(fw_event);
3574         queue_work(ioc->firmware_event_thread, &fw_event->work);
3575         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
3576 }
3577
3578 /**
3579  * _scsih_fw_event_del_from_list - delete fw_event from the list
3580  * @ioc: per adapter object
3581  * @fw_event: object describing the event
3582  * Context: This function will acquire ioc->fw_event_lock.
3583  *
3584  * If the fw_event is on the fw_event_list, remove it and do a put.
3585  */
3586 static void
3587 _scsih_fw_event_del_from_list(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work
3588         *fw_event)
3589 {
3590         unsigned long flags;
3591
3592         spin_lock_irqsave(&ioc->fw_event_lock, flags);
3593         if (!list_empty(&fw_event->list)) {
3594                 list_del_init(&fw_event->list);
3595                 fw_event_work_put(fw_event);
3596         }
3597         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
3598 }
3599
3600
3601  /**
3602  * mpt3sas_send_trigger_data_event - send event for processing trigger data
3603  * @ioc: per adapter object
3604  * @event_data: trigger event data
3605  */
3606 void
3607 mpt3sas_send_trigger_data_event(struct MPT3SAS_ADAPTER *ioc,
3608         struct SL_WH_TRIGGERS_EVENT_DATA_T *event_data)
3609 {
3610         struct fw_event_work *fw_event;
3611         u16 sz;
3612
3613         if (ioc->is_driver_loading)
3614                 return;
3615         sz = sizeof(*event_data);
3616         fw_event = alloc_fw_event_work(sz);
3617         if (!fw_event)
3618                 return;
3619         fw_event->event = MPT3SAS_PROCESS_TRIGGER_DIAG;
3620         fw_event->ioc = ioc;
3621         memcpy(fw_event->event_data, event_data, sizeof(*event_data));
3622         _scsih_fw_event_add(ioc, fw_event);
3623         fw_event_work_put(fw_event);
3624 }
3625
3626 /**
3627  * _scsih_error_recovery_delete_devices - remove devices not responding
3628  * @ioc: per adapter object
3629  */
3630 static void
3631 _scsih_error_recovery_delete_devices(struct MPT3SAS_ADAPTER *ioc)
3632 {
3633         struct fw_event_work *fw_event;
3634
3635         fw_event = alloc_fw_event_work(0);
3636         if (!fw_event)
3637                 return;
3638         fw_event->event = MPT3SAS_REMOVE_UNRESPONDING_DEVICES;
3639         fw_event->ioc = ioc;
3640         _scsih_fw_event_add(ioc, fw_event);
3641         fw_event_work_put(fw_event);
3642 }
3643
3644 /**
3645  * mpt3sas_port_enable_complete - port enable completed (fake event)
3646  * @ioc: per adapter object
3647  */
3648 void
3649 mpt3sas_port_enable_complete(struct MPT3SAS_ADAPTER *ioc)
3650 {
3651         struct fw_event_work *fw_event;
3652
3653         fw_event = alloc_fw_event_work(0);
3654         if (!fw_event)
3655                 return;
3656         fw_event->event = MPT3SAS_PORT_ENABLE_COMPLETE;
3657         fw_event->ioc = ioc;
3658         _scsih_fw_event_add(ioc, fw_event);
3659         fw_event_work_put(fw_event);
3660 }
3661
3662 static struct fw_event_work *dequeue_next_fw_event(struct MPT3SAS_ADAPTER *ioc)
3663 {
3664         unsigned long flags;
3665         struct fw_event_work *fw_event = NULL;
3666
3667         spin_lock_irqsave(&ioc->fw_event_lock, flags);
3668         if (!list_empty(&ioc->fw_event_list)) {
3669                 fw_event = list_first_entry(&ioc->fw_event_list,
3670                                 struct fw_event_work, list);
3671                 list_del_init(&fw_event->list);
3672                 fw_event_work_put(fw_event);
3673         }
3674         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
3675
3676         return fw_event;
3677 }
3678
3679 /**
3680  * _scsih_fw_event_cleanup_queue - cleanup event queue
3681  * @ioc: per adapter object
3682  *
3683  * Walk the firmware event queue, either killing timers, or waiting
3684  * for outstanding events to complete
3685  *
3686  * Context: task, can sleep
3687  */
3688 static void
3689 _scsih_fw_event_cleanup_queue(struct MPT3SAS_ADAPTER *ioc)
3690 {
3691         struct fw_event_work *fw_event;
3692
3693         if ((list_empty(&ioc->fw_event_list) && !ioc->current_event) ||
3694             !ioc->firmware_event_thread)
3695                 return;
3696         /*
3697          * Set current running event as ignore, so that
3698          * current running event will exit quickly.
3699          * As diag reset has occurred it is of no use
3700          * to process remaining stale event data entries.
3701          */
3702         if (ioc->shost_recovery && ioc->current_event)
3703                 ioc->current_event->ignore = 1;
3704
3705         ioc->fw_events_cleanup = 1;
3706         while ((fw_event = dequeue_next_fw_event(ioc)) ||
3707              (fw_event = ioc->current_event)) {
3708
3709                 /*
3710                  * Don't call cancel_work_sync() for current_event
3711                  * other than MPT3SAS_REMOVE_UNRESPONDING_DEVICES;
3712                  * otherwise we may observe deadlock if current
3713                  * hard reset issued as part of processing the current_event.
3714                  *
3715                  * Orginal logic of cleaning the current_event is added
3716                  * for handling the back to back host reset issued by the user.
3717                  * i.e. during back to back host reset, driver use to process
3718                  * the two instances of MPT3SAS_REMOVE_UNRESPONDING_DEVICES
3719                  * event back to back and this made the drives to unregister
3720                  * the devices from SML.
3721                  */
3722
3723                 if (fw_event == ioc->current_event &&
3724                     ioc->current_event->event !=
3725                     MPT3SAS_REMOVE_UNRESPONDING_DEVICES) {
3726                         ioc->current_event = NULL;
3727                         continue;
3728                 }
3729
3730                 /*
3731                  * Driver has to clear ioc->start_scan flag when
3732                  * it is cleaning up MPT3SAS_PORT_ENABLE_COMPLETE,
3733                  * otherwise scsi_scan_host() API waits for the
3734                  * 5 minute timer to expire. If we exit from
3735                  * scsi_scan_host() early then we can issue the
3736                  * new port enable request as part of current diag reset.
3737                  */
3738                 if (fw_event->event == MPT3SAS_PORT_ENABLE_COMPLETE) {
3739                         ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
3740                         ioc->start_scan = 0;
3741                 }
3742
3743                 /*
3744                  * Wait on the fw_event to complete. If this returns 1, then
3745                  * the event was never executed, and we need a put for the
3746                  * reference the work had on the fw_event.
3747                  *
3748                  * If it did execute, we wait for it to finish, and the put will
3749                  * happen from _firmware_event_work()
3750                  */
3751                 if (cancel_work_sync(&fw_event->work))
3752                         fw_event_work_put(fw_event);
3753
3754         }
3755         ioc->fw_events_cleanup = 0;
3756 }
3757
3758 /**
3759  * _scsih_internal_device_block - block the sdev device
3760  * @sdev: per device object
3761  * @sas_device_priv_data : per device driver private data
3762  *
3763  * make sure device is blocked without error, if not
3764  * print an error
3765  */
3766 static void
3767 _scsih_internal_device_block(struct scsi_device *sdev,
3768                         struct MPT3SAS_DEVICE *sas_device_priv_data)
3769 {
3770         int r = 0;
3771
3772         sdev_printk(KERN_INFO, sdev, "device_block, handle(0x%04x)\n",
3773             sas_device_priv_data->sas_target->handle);
3774         sas_device_priv_data->block = 1;
3775
3776         r = scsi_internal_device_block_nowait(sdev);
3777         if (r == -EINVAL)
3778                 sdev_printk(KERN_WARNING, sdev,
3779                     "device_block failed with return(%d) for handle(0x%04x)\n",
3780                     r, sas_device_priv_data->sas_target->handle);
3781 }
3782
3783 /**
3784  * _scsih_internal_device_unblock - unblock the sdev device
3785  * @sdev: per device object
3786  * @sas_device_priv_data : per device driver private data
3787  * make sure device is unblocked without error, if not retry
3788  * by blocking and then unblocking
3789  */
3790
3791 static void
3792 _scsih_internal_device_unblock(struct scsi_device *sdev,
3793                         struct MPT3SAS_DEVICE *sas_device_priv_data)
3794 {
3795         int r = 0;
3796
3797         sdev_printk(KERN_WARNING, sdev, "device_unblock and setting to running, "
3798             "handle(0x%04x)\n", sas_device_priv_data->sas_target->handle);
3799         sas_device_priv_data->block = 0;
3800         r = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
3801         if (r == -EINVAL) {
3802                 /* The device has been set to SDEV_RUNNING by SD layer during
3803                  * device addition but the request queue is still stopped by
3804                  * our earlier block call. We need to perform a block again
3805                  * to get the device to SDEV_BLOCK and then to SDEV_RUNNING */
3806
3807                 sdev_printk(KERN_WARNING, sdev,
3808                     "device_unblock failed with return(%d) for handle(0x%04x) "
3809                     "performing a block followed by an unblock\n",
3810                     r, sas_device_priv_data->sas_target->handle);
3811                 sas_device_priv_data->block = 1;
3812                 r = scsi_internal_device_block_nowait(sdev);
3813                 if (r)
3814                         sdev_printk(KERN_WARNING, sdev, "retried device_block "
3815                             "failed with return(%d) for handle(0x%04x)\n",
3816                             r, sas_device_priv_data->sas_target->handle);
3817
3818                 sas_device_priv_data->block = 0;
3819                 r = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
3820                 if (r)
3821                         sdev_printk(KERN_WARNING, sdev, "retried device_unblock"
3822                             " failed with return(%d) for handle(0x%04x)\n",
3823                             r, sas_device_priv_data->sas_target->handle);
3824         }
3825 }
3826
3827 /**
3828  * _scsih_ublock_io_all_device - unblock every device
3829  * @ioc: per adapter object
3830  *
3831  * change the device state from block to running
3832  */
3833 static void
3834 _scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc)
3835 {
3836         struct MPT3SAS_DEVICE *sas_device_priv_data;
3837         struct scsi_device *sdev;
3838
3839         shost_for_each_device(sdev, ioc->shost) {
3840                 sas_device_priv_data = sdev->hostdata;
3841                 if (!sas_device_priv_data)
3842                         continue;
3843                 if (!sas_device_priv_data->block)
3844                         continue;
3845
3846                 dewtprintk(ioc, sdev_printk(KERN_INFO, sdev,
3847                         "device_running, handle(0x%04x)\n",
3848                     sas_device_priv_data->sas_target->handle));
3849                 _scsih_internal_device_unblock(sdev, sas_device_priv_data);
3850         }
3851 }
3852
3853
3854 /**
3855  * _scsih_ublock_io_device - prepare device to be deleted
3856  * @ioc: per adapter object
3857  * @sas_address: sas address
3858  * @port: hba port entry
3859  *
3860  * unblock then put device in offline state
3861  */
3862 static void
3863 _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc,
3864         u64 sas_address, struct hba_port *port)
3865 {
3866         struct MPT3SAS_DEVICE *sas_device_priv_data;
3867         struct scsi_device *sdev;
3868
3869         shost_for_each_device(sdev, ioc->shost) {
3870                 sas_device_priv_data = sdev->hostdata;
3871                 if (!sas_device_priv_data || !sas_device_priv_data->sas_target)
3872                         continue;
3873                 if (sas_device_priv_data->sas_target->sas_address
3874                     != sas_address)
3875                         continue;
3876                 if (sas_device_priv_data->sas_target->port != port)
3877                         continue;
3878                 if (sas_device_priv_data->block)
3879                         _scsih_internal_device_unblock(sdev,
3880                                 sas_device_priv_data);
3881         }
3882 }
3883
3884 /**
3885  * _scsih_block_io_all_device - set the device state to SDEV_BLOCK
3886  * @ioc: per adapter object
3887  *
3888  * During device pull we need to appropriately set the sdev state.
3889  */
3890 static void
3891 _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)
3892 {
3893         struct MPT3SAS_DEVICE *sas_device_priv_data;
3894         struct scsi_device *sdev;
3895
3896         shost_for_each_device(sdev, ioc->shost) {
3897                 sas_device_priv_data = sdev->hostdata;
3898                 if (!sas_device_priv_data)
3899                         continue;
3900                 if (sas_device_priv_data->block)
3901                         continue;
3902                 if (sas_device_priv_data->ignore_delay_remove) {
3903                         sdev_printk(KERN_INFO, sdev,
3904                         "%s skip device_block for SES handle(0x%04x)\n",
3905                         __func__, sas_device_priv_data->sas_target->handle);
3906                         continue;
3907                 }
3908                 _scsih_internal_device_block(sdev, sas_device_priv_data);
3909         }
3910 }
3911
3912 /**
3913  * _scsih_block_io_device - set the device state to SDEV_BLOCK
3914  * @ioc: per adapter object
3915  * @handle: device handle
3916  *
3917  * During device pull we need to appropriately set the sdev state.
3918  */
3919 static void
3920 _scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
3921 {
3922         struct MPT3SAS_DEVICE *sas_device_priv_data;
3923         struct scsi_device *sdev;
3924         struct _sas_device *sas_device;
3925
3926         sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
3927
3928         shost_for_each_device(sdev, ioc->shost) {
3929                 sas_device_priv_data = sdev->hostdata;
3930                 if (!sas_device_priv_data)
3931                         continue;
3932                 if (sas_device_priv_data->sas_target->handle != handle)
3933                         continue;
3934                 if (sas_device_priv_data->block)
3935                         continue;
3936                 if (sas_device && sas_device->pend_sas_rphy_add)
3937                         continue;
3938                 if (sas_device_priv_data->ignore_delay_remove) {
3939                         sdev_printk(KERN_INFO, sdev,
3940                         "%s skip device_block for SES handle(0x%04x)\n",
3941                         __func__, sas_device_priv_data->sas_target->handle);
3942                         continue;
3943                 }
3944                 _scsih_internal_device_block(sdev, sas_device_priv_data);
3945         }
3946
3947         if (sas_device)
3948                 sas_device_put(sas_device);
3949 }
3950
3951 /**
3952  * _scsih_block_io_to_children_attached_to_ex
3953  * @ioc: per adapter object
3954  * @sas_expander: the sas_device object
3955  *
3956  * This routine set sdev state to SDEV_BLOCK for all devices
3957  * attached to this expander. This function called when expander is
3958  * pulled.
3959  */
3960 static void
3961 _scsih_block_io_to_children_attached_to_ex(struct MPT3SAS_ADAPTER *ioc,
3962         struct _sas_node *sas_expander)
3963 {
3964         struct _sas_port *mpt3sas_port;
3965         struct _sas_device *sas_device;
3966         struct _sas_node *expander_sibling;
3967         unsigned long flags;
3968
3969         if (!sas_expander)
3970                 return;
3971
3972         list_for_each_entry(mpt3sas_port,
3973            &sas_expander->sas_port_list, port_list) {
3974                 if (mpt3sas_port->remote_identify.device_type ==
3975                     SAS_END_DEVICE) {
3976                         spin_lock_irqsave(&ioc->sas_device_lock, flags);
3977                         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
3978                             mpt3sas_port->remote_identify.sas_address,
3979                             mpt3sas_port->hba_port);
3980                         if (sas_device) {
3981                                 set_bit(sas_device->handle,
3982                                                 ioc->blocking_handles);
3983                                 sas_device_put(sas_device);
3984                         }
3985                         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
3986                 }
3987         }
3988
3989         list_for_each_entry(mpt3sas_port,
3990            &sas_expander->sas_port_list, port_list) {
3991
3992                 if (mpt3sas_port->remote_identify.device_type ==
3993                     SAS_EDGE_EXPANDER_DEVICE ||
3994                     mpt3sas_port->remote_identify.device_type ==
3995                     SAS_FANOUT_EXPANDER_DEVICE) {
3996                         expander_sibling =
3997                             mpt3sas_scsih_expander_find_by_sas_address(
3998                             ioc, mpt3sas_port->remote_identify.sas_address,
3999                             mpt3sas_port->hba_port);
4000                         _scsih_block_io_to_children_attached_to_ex(ioc,
4001                             expander_sibling);
4002                 }
4003         }
4004 }
4005
4006 /**
4007  * _scsih_block_io_to_children_attached_directly
4008  * @ioc: per adapter object
4009  * @event_data: topology change event data
4010  *
4011  * This routine set sdev state to SDEV_BLOCK for all devices
4012  * direct attached during device pull.
4013  */
4014 static void
4015 _scsih_block_io_to_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,
4016         Mpi2EventDataSasTopologyChangeList_t *event_data)
4017 {
4018         int i;
4019         u16 handle;
4020         u16 reason_code;
4021
4022         for (i = 0; i < event_data->NumEntries; i++) {
4023                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
4024                 if (!handle)
4025                         continue;
4026                 reason_code = event_data->PHY[i].PhyStatus &
4027                     MPI2_EVENT_SAS_TOPO_RC_MASK;
4028                 if (reason_code == MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING)
4029                         _scsih_block_io_device(ioc, handle);
4030         }
4031 }
4032
4033 /**
4034  * _scsih_block_io_to_pcie_children_attached_directly
4035  * @ioc: per adapter object
4036  * @event_data: topology change event data
4037  *
4038  * This routine set sdev state to SDEV_BLOCK for all devices
4039  * direct attached during device pull/reconnect.
4040  */
4041 static void
4042 _scsih_block_io_to_pcie_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,
4043                 Mpi26EventDataPCIeTopologyChangeList_t *event_data)
4044 {
4045         int i;
4046         u16 handle;
4047         u16 reason_code;
4048
4049         for (i = 0; i < event_data->NumEntries; i++) {
4050                 handle =
4051                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
4052                 if (!handle)
4053                         continue;
4054                 reason_code = event_data->PortEntry[i].PortStatus;
4055                 if (reason_code ==
4056                                 MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING)
4057                         _scsih_block_io_device(ioc, handle);
4058         }
4059 }
4060 /**
4061  * _scsih_tm_tr_send - send task management request
4062  * @ioc: per adapter object
4063  * @handle: device handle
4064  * Context: interrupt time.
4065  *
4066  * This code is to initiate the device removal handshake protocol
4067  * with controller firmware.  This function will issue target reset
4068  * using high priority request queue.  It will send a sas iounit
4069  * control request (MPI2_SAS_OP_REMOVE_DEVICE) from this completion.
4070  *
4071  * This is designed to send muliple task management request at the same
4072  * time to the fifo. If the fifo is full, we will append the request,
4073  * and process it in a future completion.
4074  */
4075 static void
4076 _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4077 {
4078         Mpi2SCSITaskManagementRequest_t *mpi_request;
4079         u16 smid;
4080         struct _sas_device *sas_device = NULL;
4081         struct _pcie_device *pcie_device = NULL;
4082         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
4083         u64 sas_address = 0;
4084         unsigned long flags;
4085         struct _tr_list *delayed_tr;
4086         u32 ioc_state;
4087         u8 tr_method = 0;
4088         struct hba_port *port = NULL;
4089
4090         if (ioc->pci_error_recovery) {
4091                 dewtprintk(ioc,
4092                            ioc_info(ioc, "%s: host in pci error recovery: handle(0x%04x)\n",
4093                                     __func__, handle));
4094                 return;
4095         }
4096         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
4097         if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4098                 dewtprintk(ioc,
4099                            ioc_info(ioc, "%s: host is not operational: handle(0x%04x)\n",
4100                                     __func__, handle));
4101                 return;
4102         }
4103
4104         /* if PD, then return */
4105         if (test_bit(handle, ioc->pd_handles))
4106                 return;
4107
4108         clear_bit(handle, ioc->pend_os_device_add);
4109
4110         spin_lock_irqsave(&ioc->sas_device_lock, flags);
4111         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
4112         if (sas_device && sas_device->starget &&
4113             sas_device->starget->hostdata) {
4114                 sas_target_priv_data = sas_device->starget->hostdata;
4115                 sas_target_priv_data->deleted = 1;
4116                 sas_address = sas_device->sas_address;
4117                 port = sas_device->port;
4118         }
4119         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
4120         if (!sas_device) {
4121                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
4122                 pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
4123                 if (pcie_device && pcie_device->starget &&
4124                         pcie_device->starget->hostdata) {
4125                         sas_target_priv_data = pcie_device->starget->hostdata;
4126                         sas_target_priv_data->deleted = 1;
4127                         sas_address = pcie_device->wwid;
4128                 }
4129                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
4130                 if (pcie_device && (!ioc->tm_custom_handling) &&
4131                     (!(mpt3sas_scsih_is_pcie_scsi_device(
4132                     pcie_device->device_info))))
4133                         tr_method =
4134                             MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
4135                 else
4136                         tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
4137         }
4138         if (sas_target_priv_data) {
4139                 dewtprintk(ioc,
4140                            ioc_info(ioc, "setting delete flag: handle(0x%04x), sas_addr(0x%016llx)\n",
4141                                     handle, (u64)sas_address));
4142                 if (sas_device) {
4143                         if (sas_device->enclosure_handle != 0)
4144                                 dewtprintk(ioc,
4145                                            ioc_info(ioc, "setting delete flag:enclosure logical id(0x%016llx), slot(%d)\n",
4146                                                     (u64)sas_device->enclosure_logical_id,
4147                                                     sas_device->slot));
4148                         if (sas_device->connector_name[0] != '\0')
4149                                 dewtprintk(ioc,
4150                                            ioc_info(ioc, "setting delete flag: enclosure level(0x%04x), connector name( %s)\n",
4151                                                     sas_device->enclosure_level,
4152                                                     sas_device->connector_name));
4153                 } else if (pcie_device) {
4154                         if (pcie_device->enclosure_handle != 0)
4155                                 dewtprintk(ioc,
4156                                            ioc_info(ioc, "setting delete flag: logical id(0x%016llx), slot(%d)\n",
4157                                                     (u64)pcie_device->enclosure_logical_id,
4158                                                     pcie_device->slot));
4159                         if (pcie_device->connector_name[0] != '\0')
4160                                 dewtprintk(ioc,
4161                                            ioc_info(ioc, "setting delete flag:, enclosure level(0x%04x), connector name( %s)\n",
4162                                                     pcie_device->enclosure_level,
4163                                                     pcie_device->connector_name));
4164                 }
4165                 _scsih_ublock_io_device(ioc, sas_address, port);
4166                 sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
4167         }
4168
4169         smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_cb_idx);
4170         if (!smid) {
4171                 delayed_tr = kzalloc(sizeof(*delayed_tr), GFP_ATOMIC);
4172                 if (!delayed_tr)
4173                         goto out;
4174                 INIT_LIST_HEAD(&delayed_tr->list);
4175                 delayed_tr->handle = handle;
4176                 list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);
4177                 dewtprintk(ioc,
4178                            ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4179                                     handle));
4180                 goto out;
4181         }
4182
4183         dewtprintk(ioc,
4184                    ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4185                             handle, smid, ioc->tm_tr_cb_idx));
4186         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4187         memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
4188         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4189         mpi_request->DevHandle = cpu_to_le16(handle);
4190         mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
4191         mpi_request->MsgFlags = tr_method;
4192         set_bit(handle, ioc->device_remove_in_progress);
4193         ioc->put_smid_hi_priority(ioc, smid, 0);
4194         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);
4195
4196 out:
4197         if (sas_device)
4198                 sas_device_put(sas_device);
4199         if (pcie_device)
4200                 pcie_device_put(pcie_device);
4201 }
4202
4203 /**
4204  * _scsih_tm_tr_complete -
4205  * @ioc: per adapter object
4206  * @smid: system request message index
4207  * @msix_index: MSIX table index supplied by the OS
4208  * @reply: reply message frame(lower 32bit addr)
4209  * Context: interrupt time.
4210  *
4211  * This is the target reset completion routine.
4212  * This code is part of the code to initiate the device removal
4213  * handshake protocol with controller firmware.
4214  * It will send a sas iounit control request (MPI2_SAS_OP_REMOVE_DEVICE)
4215  *
4216  * Return: 1 meaning mf should be freed from _base_interrupt
4217  *         0 means the mf is freed from this function.
4218  */
4219 static u8
4220 _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
4221         u32 reply)
4222 {
4223         u16 handle;
4224         Mpi2SCSITaskManagementRequest_t *mpi_request_tm;
4225         Mpi2SCSITaskManagementReply_t *mpi_reply =
4226             mpt3sas_base_get_reply_virt_addr(ioc, reply);
4227         Mpi2SasIoUnitControlRequest_t *mpi_request;
4228         u16 smid_sas_ctrl;
4229         u32 ioc_state;
4230         struct _sc_list *delayed_sc;
4231
4232         if (ioc->pci_error_recovery) {
4233                 dewtprintk(ioc,
4234                            ioc_info(ioc, "%s: host in pci error recovery\n",
4235                                     __func__));
4236                 return 1;
4237         }
4238         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
4239         if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4240                 dewtprintk(ioc,
4241                            ioc_info(ioc, "%s: host is not operational\n",
4242                                     __func__));
4243                 return 1;
4244         }
4245         if (unlikely(!mpi_reply)) {
4246                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4247                         __FILE__, __LINE__, __func__);
4248                 return 1;
4249         }
4250         mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);
4251         handle = le16_to_cpu(mpi_request_tm->DevHandle);
4252         if (handle != le16_to_cpu(mpi_reply->DevHandle)) {
4253                 dewtprintk(ioc,
4254                            ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",
4255                                    handle,
4256                                    le16_to_cpu(mpi_reply->DevHandle), smid));
4257                 return 0;
4258         }
4259
4260         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);
4261         dewtprintk(ioc,
4262                    ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), completed(%d)\n",
4263                             handle, smid, le16_to_cpu(mpi_reply->IOCStatus),
4264                             le32_to_cpu(mpi_reply->IOCLogInfo),
4265                             le32_to_cpu(mpi_reply->TerminationCount)));
4266
4267         smid_sas_ctrl = mpt3sas_base_get_smid(ioc, ioc->tm_sas_control_cb_idx);
4268         if (!smid_sas_ctrl) {
4269                 delayed_sc = kzalloc(sizeof(*delayed_sc), GFP_ATOMIC);
4270                 if (!delayed_sc)
4271                         return _scsih_check_for_pending_tm(ioc, smid);
4272                 INIT_LIST_HEAD(&delayed_sc->list);
4273                 delayed_sc->handle = le16_to_cpu(mpi_request_tm->DevHandle);
4274                 list_add_tail(&delayed_sc->list, &ioc->delayed_sc_list);
4275                 dewtprintk(ioc,
4276                            ioc_info(ioc, "DELAYED:sc:handle(0x%04x), (open)\n",
4277                                     handle));
4278                 return _scsih_check_for_pending_tm(ioc, smid);
4279         }
4280
4281         dewtprintk(ioc,
4282                    ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4283                             handle, smid_sas_ctrl, ioc->tm_sas_control_cb_idx));
4284         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid_sas_ctrl);
4285         memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
4286         mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
4287         mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
4288         mpi_request->DevHandle = mpi_request_tm->DevHandle;
4289         ioc->put_smid_default(ioc, smid_sas_ctrl);
4290
4291         return _scsih_check_for_pending_tm(ioc, smid);
4292 }
4293
4294 /** _scsih_allow_scmd_to_device - check whether scmd needs to
4295  *                               issue to IOC or not.
4296  * @ioc: per adapter object
4297  * @scmd: pointer to scsi command object
4298  *
4299  * Returns true if scmd can be issued to IOC otherwise returns false.
4300  */
4301 inline bool _scsih_allow_scmd_to_device(struct MPT3SAS_ADAPTER *ioc,
4302         struct scsi_cmnd *scmd)
4303 {
4304
4305         if (ioc->pci_error_recovery)
4306                 return false;
4307
4308         if (ioc->hba_mpi_version_belonged == MPI2_VERSION) {
4309                 if (ioc->remove_host)
4310                         return false;
4311
4312                 return true;
4313         }
4314
4315         if (ioc->remove_host) {
4316
4317                 switch (scmd->cmnd[0]) {
4318                 case SYNCHRONIZE_CACHE:
4319                 case START_STOP:
4320                         return true;
4321                 default:
4322                         return false;
4323                 }
4324         }
4325
4326         return true;
4327 }
4328
4329 /**
4330  * _scsih_sas_control_complete - completion routine
4331  * @ioc: per adapter object
4332  * @smid: system request message index
4333  * @msix_index: MSIX table index supplied by the OS
4334  * @reply: reply message frame(lower 32bit addr)
4335  * Context: interrupt time.
4336  *
4337  * This is the sas iounit control completion routine.
4338  * This code is part of the code to initiate the device removal
4339  * handshake protocol with controller firmware.
4340  *
4341  * Return: 1 meaning mf should be freed from _base_interrupt
4342  *         0 means the mf is freed from this function.
4343  */
4344 static u8
4345 _scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4346         u8 msix_index, u32 reply)
4347 {
4348         Mpi2SasIoUnitControlReply_t *mpi_reply =
4349             mpt3sas_base_get_reply_virt_addr(ioc, reply);
4350
4351         if (likely(mpi_reply)) {
4352                 dewtprintk(ioc,
4353                            ioc_info(ioc, "sc_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x)\n",
4354                                     le16_to_cpu(mpi_reply->DevHandle), smid,
4355                                     le16_to_cpu(mpi_reply->IOCStatus),
4356                                     le32_to_cpu(mpi_reply->IOCLogInfo)));
4357                 if (le16_to_cpu(mpi_reply->IOCStatus) ==
4358                      MPI2_IOCSTATUS_SUCCESS) {
4359                         clear_bit(le16_to_cpu(mpi_reply->DevHandle),
4360                             ioc->device_remove_in_progress);
4361                 }
4362         } else {
4363                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4364                         __FILE__, __LINE__, __func__);
4365         }
4366         return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
4367 }
4368
4369 /**
4370  * _scsih_tm_tr_volume_send - send target reset request for volumes
4371  * @ioc: per adapter object
4372  * @handle: device handle
4373  * Context: interrupt time.
4374  *
4375  * This is designed to send muliple task management request at the same
4376  * time to the fifo. If the fifo is full, we will append the request,
4377  * and process it in a future completion.
4378  */
4379 static void
4380 _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4381 {
4382         Mpi2SCSITaskManagementRequest_t *mpi_request;
4383         u16 smid;
4384         struct _tr_list *delayed_tr;
4385
4386         if (ioc->pci_error_recovery) {
4387                 dewtprintk(ioc,
4388                            ioc_info(ioc, "%s: host reset in progress!\n",
4389                                     __func__));
4390                 return;
4391         }
4392
4393         smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_volume_cb_idx);
4394         if (!smid) {
4395                 delayed_tr = kzalloc(sizeof(*delayed_tr), GFP_ATOMIC);
4396                 if (!delayed_tr)
4397                         return;
4398                 INIT_LIST_HEAD(&delayed_tr->list);
4399                 delayed_tr->handle = handle;
4400                 list_add_tail(&delayed_tr->list, &ioc->delayed_tr_volume_list);
4401                 dewtprintk(ioc,
4402                            ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4403                                     handle));
4404                 return;
4405         }
4406
4407         dewtprintk(ioc,
4408                    ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4409                             handle, smid, ioc->tm_tr_volume_cb_idx));
4410         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4411         memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
4412         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4413         mpi_request->DevHandle = cpu_to_le16(handle);
4414         mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
4415         ioc->put_smid_hi_priority(ioc, smid, 0);
4416 }
4417
4418 /**
4419  * _scsih_tm_volume_tr_complete - target reset completion
4420  * @ioc: per adapter object
4421  * @smid: system request message index
4422  * @msix_index: MSIX table index supplied by the OS
4423  * @reply: reply message frame(lower 32bit addr)
4424  * Context: interrupt time.
4425  *
4426  * Return: 1 meaning mf should be freed from _base_interrupt
4427  *         0 means the mf is freed from this function.
4428  */
4429 static u8
4430 _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4431         u8 msix_index, u32 reply)
4432 {
4433         u16 handle;
4434         Mpi2SCSITaskManagementRequest_t *mpi_request_tm;
4435         Mpi2SCSITaskManagementReply_t *mpi_reply =
4436             mpt3sas_base_get_reply_virt_addr(ioc, reply);
4437
4438         if (ioc->shost_recovery || ioc->pci_error_recovery) {
4439                 dewtprintk(ioc,
4440                            ioc_info(ioc, "%s: host reset in progress!\n",
4441                                     __func__));
4442                 return 1;
4443         }
4444         if (unlikely(!mpi_reply)) {
4445                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4446                         __FILE__, __LINE__, __func__);
4447                 return 1;
4448         }
4449
4450         mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);
4451         handle = le16_to_cpu(mpi_request_tm->DevHandle);
4452         if (handle != le16_to_cpu(mpi_reply->DevHandle)) {
4453                 dewtprintk(ioc,
4454                            ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",
4455                                    handle, le16_to_cpu(mpi_reply->DevHandle),
4456                                    smid));
4457                 return 0;
4458         }
4459
4460         dewtprintk(ioc,
4461                    ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), completed(%d)\n",
4462                             handle, smid, le16_to_cpu(mpi_reply->IOCStatus),
4463                             le32_to_cpu(mpi_reply->IOCLogInfo),
4464                             le32_to_cpu(mpi_reply->TerminationCount)));
4465
4466         return _scsih_check_for_pending_tm(ioc, smid);
4467 }
4468
4469 /**
4470  * _scsih_issue_delayed_event_ack - issue delayed Event ACK messages
4471  * @ioc: per adapter object
4472  * @smid: system request message index
4473  * @event: Event ID
4474  * @event_context: used to track events uniquely
4475  *
4476  * Context - processed in interrupt context.
4477  */
4478 static void
4479 _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,
4480                                 U32 event_context)
4481 {
4482         Mpi2EventAckRequest_t *ack_request;
4483         int i = smid - ioc->internal_smid;
4484         unsigned long flags;
4485
4486         /* Without releasing the smid just update the
4487          * call back index and reuse the same smid for
4488          * processing this delayed request
4489          */
4490         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4491         ioc->internal_lookup[i].cb_idx = ioc->base_cb_idx;
4492         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
4493
4494         dewtprintk(ioc,
4495                    ioc_info(ioc, "EVENT ACK: event(0x%04x), smid(%d), cb(%d)\n",
4496                             le16_to_cpu(event), smid, ioc->base_cb_idx));
4497         ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
4498         memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
4499         ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
4500         ack_request->Event = event;
4501         ack_request->EventContext = event_context;
4502         ack_request->VF_ID = 0;  /* TODO */
4503         ack_request->VP_ID = 0;
4504         ioc->put_smid_default(ioc, smid);
4505 }
4506
4507 /**
4508  * _scsih_issue_delayed_sas_io_unit_ctrl - issue delayed
4509  *                              sas_io_unit_ctrl messages
4510  * @ioc: per adapter object
4511  * @smid: system request message index
4512  * @handle: device handle
4513  *
4514  * Context - processed in interrupt context.
4515  */
4516 static void
4517 _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
4518                                         u16 smid, u16 handle)
4519 {
4520         Mpi2SasIoUnitControlRequest_t *mpi_request;
4521         u32 ioc_state;
4522         int i = smid - ioc->internal_smid;
4523         unsigned long flags;
4524
4525         if (ioc->remove_host) {
4526                 dewtprintk(ioc,
4527                            ioc_info(ioc, "%s: host has been removed\n",
4528                                     __func__));
4529                 return;
4530         } else if (ioc->pci_error_recovery) {
4531                 dewtprintk(ioc,
4532                            ioc_info(ioc, "%s: host in pci error recovery\n",
4533                                     __func__));
4534                 return;
4535         }
4536         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
4537         if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4538                 dewtprintk(ioc,
4539                            ioc_info(ioc, "%s: host is not operational\n",
4540                                     __func__));
4541                 return;
4542         }
4543
4544         /* Without releasing the smid just update the
4545          * call back index and reuse the same smid for
4546          * processing this delayed request
4547          */
4548         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4549         ioc->internal_lookup[i].cb_idx = ioc->tm_sas_control_cb_idx;
4550         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
4551
4552         dewtprintk(ioc,
4553                    ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4554                             handle, smid, ioc->tm_sas_control_cb_idx));
4555         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4556         memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
4557         mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
4558         mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
4559         mpi_request->DevHandle = cpu_to_le16(handle);
4560         ioc->put_smid_default(ioc, smid);
4561 }
4562
4563 /**
4564  * mpt3sas_check_for_pending_internal_cmds - check for pending internal messages
4565  * @ioc: per adapter object
4566  * @smid: system request message index
4567  *
4568  * Context: Executed in interrupt context
4569  *
4570  * This will check delayed internal messages list, and process the
4571  * next request.
4572  *
4573  * Return: 1 meaning mf should be freed from _base_interrupt
4574  *         0 means the mf is freed from this function.
4575  */
4576 u8
4577 mpt3sas_check_for_pending_internal_cmds(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4578 {
4579         struct _sc_list *delayed_sc;
4580         struct _event_ack_list *delayed_event_ack;
4581
4582         if (!list_empty(&ioc->delayed_event_ack_list)) {
4583                 delayed_event_ack = list_entry(ioc->delayed_event_ack_list.next,
4584                                                 struct _event_ack_list, list);
4585                 _scsih_issue_delayed_event_ack(ioc, smid,
4586                   delayed_event_ack->Event, delayed_event_ack->EventContext);
4587                 list_del(&delayed_event_ack->list);
4588                 kfree(delayed_event_ack);
4589                 return 0;
4590         }
4591
4592         if (!list_empty(&ioc->delayed_sc_list)) {
4593                 delayed_sc = list_entry(ioc->delayed_sc_list.next,
4594                                                 struct _sc_list, list);
4595                 _scsih_issue_delayed_sas_io_unit_ctrl(ioc, smid,
4596                                                  delayed_sc->handle);
4597                 list_del(&delayed_sc->list);
4598                 kfree(delayed_sc);
4599                 return 0;
4600         }
4601         return 1;
4602 }
4603
4604 /**
4605  * _scsih_check_for_pending_tm - check for pending task management
4606  * @ioc: per adapter object
4607  * @smid: system request message index
4608  *
4609  * This will check delayed target reset list, and feed the
4610  * next reqeust.
4611  *
4612  * Return: 1 meaning mf should be freed from _base_interrupt
4613  *         0 means the mf is freed from this function.
4614  */
4615 static u8
4616 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4617 {
4618         struct _tr_list *delayed_tr;
4619
4620         if (!list_empty(&ioc->delayed_tr_volume_list)) {
4621                 delayed_tr = list_entry(ioc->delayed_tr_volume_list.next,
4622                     struct _tr_list, list);
4623                 mpt3sas_base_free_smid(ioc, smid);
4624                 _scsih_tm_tr_volume_send(ioc, delayed_tr->handle);
4625                 list_del(&delayed_tr->list);
4626                 kfree(delayed_tr);
4627                 return 0;
4628         }
4629
4630         if (!list_empty(&ioc->delayed_tr_list)) {
4631                 delayed_tr = list_entry(ioc->delayed_tr_list.next,
4632                     struct _tr_list, list);
4633                 mpt3sas_base_free_smid(ioc, smid);
4634                 _scsih_tm_tr_send(ioc, delayed_tr->handle);
4635                 list_del(&delayed_tr->list);
4636                 kfree(delayed_tr);
4637                 return 0;
4638         }
4639
4640         return 1;
4641 }
4642
4643 /**
4644  * _scsih_check_topo_delete_events - sanity check on topo events
4645  * @ioc: per adapter object
4646  * @event_data: the event data payload
4647  *
4648  * This routine added to better handle cable breaker.
4649  *
4650  * This handles the case where driver receives multiple expander
4651  * add and delete events in a single shot.  When there is a delete event
4652  * the routine will void any pending add events waiting in the event queue.
4653  */
4654 static void
4655 _scsih_check_topo_delete_events(struct MPT3SAS_ADAPTER *ioc,
4656         Mpi2EventDataSasTopologyChangeList_t *event_data)
4657 {
4658         struct fw_event_work *fw_event;
4659         Mpi2EventDataSasTopologyChangeList_t *local_event_data;
4660         u16 expander_handle;
4661         struct _sas_node *sas_expander;
4662         unsigned long flags;
4663         int i, reason_code;
4664         u16 handle;
4665
4666         for (i = 0 ; i < event_data->NumEntries; i++) {
4667                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
4668                 if (!handle)
4669                         continue;
4670                 reason_code = event_data->PHY[i].PhyStatus &
4671                     MPI2_EVENT_SAS_TOPO_RC_MASK;
4672                 if (reason_code == MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING)
4673                         _scsih_tm_tr_send(ioc, handle);
4674         }
4675
4676         expander_handle = le16_to_cpu(event_data->ExpanderDevHandle);
4677         if (expander_handle < ioc->sas_hba.num_phys) {
4678                 _scsih_block_io_to_children_attached_directly(ioc, event_data);
4679                 return;
4680         }
4681         if (event_data->ExpStatus ==
4682             MPI2_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING) {
4683                 /* put expander attached devices into blocking state */
4684                 spin_lock_irqsave(&ioc->sas_node_lock, flags);
4685                 sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,
4686                     expander_handle);
4687                 _scsih_block_io_to_children_attached_to_ex(ioc, sas_expander);
4688                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
4689                 do {
4690                         handle = find_first_bit(ioc->blocking_handles,
4691                             ioc->facts.MaxDevHandle);
4692                         if (handle < ioc->facts.MaxDevHandle)
4693                                 _scsih_block_io_device(ioc, handle);
4694                 } while (test_and_clear_bit(handle, ioc->blocking_handles));
4695         } else if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_RESPONDING)
4696                 _scsih_block_io_to_children_attached_directly(ioc, event_data);
4697
4698         if (event_data->ExpStatus != MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
4699                 return;
4700
4701         /* mark ignore flag for pending events */
4702         spin_lock_irqsave(&ioc->fw_event_lock, flags);
4703         list_for_each_entry(fw_event, &ioc->fw_event_list, list) {
4704                 if (fw_event->event != MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST ||
4705                     fw_event->ignore)
4706                         continue;
4707                 local_event_data = (Mpi2EventDataSasTopologyChangeList_t *)
4708                                    fw_event->event_data;
4709                 if (local_event_data->ExpStatus ==
4710                     MPI2_EVENT_SAS_TOPO_ES_ADDED ||
4711                     local_event_data->ExpStatus ==
4712                     MPI2_EVENT_SAS_TOPO_ES_RESPONDING) {
4713                         if (le16_to_cpu(local_event_data->ExpanderDevHandle) ==
4714                             expander_handle) {
4715                                 dewtprintk(ioc,
4716                                            ioc_info(ioc, "setting ignoring flag\n"));
4717                                 fw_event->ignore = 1;
4718                         }
4719                 }
4720         }
4721         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
4722 }
4723
4724 /**
4725  * _scsih_check_pcie_topo_remove_events - sanity check on topo
4726  * events
4727  * @ioc: per adapter object
4728  * @event_data: the event data payload
4729  *
4730  * This handles the case where driver receives multiple switch
4731  * or device add and delete events in a single shot.  When there
4732  * is a delete event the routine will void any pending add
4733  * events waiting in the event queue.
4734  */
4735 static void
4736 _scsih_check_pcie_topo_remove_events(struct MPT3SAS_ADAPTER *ioc,
4737         Mpi26EventDataPCIeTopologyChangeList_t *event_data)
4738 {
4739         struct fw_event_work *fw_event;
4740         Mpi26EventDataPCIeTopologyChangeList_t *local_event_data;
4741         unsigned long flags;
4742         int i, reason_code;
4743         u16 handle, switch_handle;
4744
4745         for (i = 0; i < event_data->NumEntries; i++) {
4746                 handle =
4747                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
4748                 if (!handle)
4749                         continue;
4750                 reason_code = event_data->PortEntry[i].PortStatus;
4751                 if (reason_code == MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING)
4752                         _scsih_tm_tr_send(ioc, handle);
4753         }
4754
4755         switch_handle = le16_to_cpu(event_data->SwitchDevHandle);
4756         if (!switch_handle) {
4757                 _scsih_block_io_to_pcie_children_attached_directly(
4758                                                         ioc, event_data);
4759                 return;
4760         }
4761     /* TODO We are not supporting cascaded PCIe Switch removal yet*/
4762         if ((event_data->SwitchStatus
4763                 == MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING) ||
4764                 (event_data->SwitchStatus ==
4765                                         MPI26_EVENT_PCIE_TOPO_SS_RESPONDING))
4766                 _scsih_block_io_to_pcie_children_attached_directly(
4767                                                         ioc, event_data);
4768
4769         if (event_data->SwitchStatus != MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
4770                 return;
4771
4772         /* mark ignore flag for pending events */
4773         spin_lock_irqsave(&ioc->fw_event_lock, flags);
4774         list_for_each_entry(fw_event, &ioc->fw_event_list, list) {
4775                 if (fw_event->event != MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST ||
4776                         fw_event->ignore)
4777                         continue;
4778                 local_event_data =
4779                         (Mpi26EventDataPCIeTopologyChangeList_t *)
4780                         fw_event->event_data;
4781                 if (local_event_data->SwitchStatus ==
4782                     MPI2_EVENT_SAS_TOPO_ES_ADDED ||
4783                     local_event_data->SwitchStatus ==
4784                     MPI2_EVENT_SAS_TOPO_ES_RESPONDING) {
4785                         if (le16_to_cpu(local_event_data->SwitchDevHandle) ==
4786                                 switch_handle) {
4787                                 dewtprintk(ioc,
4788                                            ioc_info(ioc, "setting ignoring flag for switch event\n"));
4789                                 fw_event->ignore = 1;
4790                         }
4791                 }
4792         }
4793         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
4794 }
4795
4796 /**
4797  * _scsih_set_volume_delete_flag - setting volume delete flag
4798  * @ioc: per adapter object
4799  * @handle: device handle
4800  *
4801  * This returns nothing.
4802  */
4803 static void
4804 _scsih_set_volume_delete_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4805 {
4806         struct _raid_device *raid_device;
4807         struct MPT3SAS_TARGET *sas_target_priv_data;
4808         unsigned long flags;
4809
4810         spin_lock_irqsave(&ioc->raid_device_lock, flags);
4811         raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
4812         if (raid_device && raid_device->starget &&
4813             raid_device->starget->hostdata) {
4814                 sas_target_priv_data =
4815                     raid_device->starget->hostdata;
4816                 sas_target_priv_data->deleted = 1;
4817                 dewtprintk(ioc,
4818                            ioc_info(ioc, "setting delete flag: handle(0x%04x), wwid(0x%016llx)\n",
4819                                     handle, (u64)raid_device->wwid));
4820         }
4821         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
4822 }
4823
4824 /**
4825  * _scsih_set_volume_handle_for_tr - set handle for target reset to volume
4826  * @handle: input handle
4827  * @a: handle for volume a
4828  * @b: handle for volume b
4829  *
4830  * IR firmware only supports two raid volumes.  The purpose of this
4831  * routine is to set the volume handle in either a or b. When the given
4832  * input handle is non-zero, or when a and b have not been set before.
4833  */
4834 static void
4835 _scsih_set_volume_handle_for_tr(u16 handle, u16 *a, u16 *b)
4836 {
4837         if (!handle || handle == *a || handle == *b)
4838                 return;
4839         if (!*a)
4840                 *a = handle;
4841         else if (!*b)
4842                 *b = handle;
4843 }
4844
4845 /**
4846  * _scsih_check_ir_config_unhide_events - check for UNHIDE events
4847  * @ioc: per adapter object
4848  * @event_data: the event data payload
4849  * Context: interrupt time.
4850  *
4851  * This routine will send target reset to volume, followed by target
4852  * resets to the PDs. This is called when a PD has been removed, or
4853  * volume has been deleted or removed. When the target reset is sent
4854  * to volume, the PD target resets need to be queued to start upon
4855  * completion of the volume target reset.
4856  */
4857 static void
4858 _scsih_check_ir_config_unhide_events(struct MPT3SAS_ADAPTER *ioc,
4859         Mpi2EventDataIrConfigChangeList_t *event_data)
4860 {
4861         Mpi2EventIrConfigElement_t *element;
4862         int i;
4863         u16 handle, volume_handle, a, b;
4864         struct _tr_list *delayed_tr;
4865
4866         a = 0;
4867         b = 0;
4868
4869         if (ioc->is_warpdrive)
4870                 return;
4871
4872         /* Volume Resets for Deleted or Removed */
4873         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4874         for (i = 0; i < event_data->NumElements; i++, element++) {
4875                 if (le32_to_cpu(event_data->Flags) &
4876                     MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG)
4877                         continue;
4878                 if (element->ReasonCode ==
4879                     MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED ||
4880                     element->ReasonCode ==
4881                     MPI2_EVENT_IR_CHANGE_RC_REMOVED) {
4882                         volume_handle = le16_to_cpu(element->VolDevHandle);
4883                         _scsih_set_volume_delete_flag(ioc, volume_handle);
4884                         _scsih_set_volume_handle_for_tr(volume_handle, &a, &b);
4885                 }
4886         }
4887
4888         /* Volume Resets for UNHIDE events */
4889         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4890         for (i = 0; i < event_data->NumElements; i++, element++) {
4891                 if (le32_to_cpu(event_data->Flags) &
4892                     MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG)
4893                         continue;
4894                 if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_UNHIDE) {
4895                         volume_handle = le16_to_cpu(element->VolDevHandle);
4896                         _scsih_set_volume_handle_for_tr(volume_handle, &a, &b);
4897                 }
4898         }
4899
4900         if (a)
4901                 _scsih_tm_tr_volume_send(ioc, a);
4902         if (b)
4903                 _scsih_tm_tr_volume_send(ioc, b);
4904
4905         /* PD target resets */
4906         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4907         for (i = 0; i < event_data->NumElements; i++, element++) {
4908                 if (element->ReasonCode != MPI2_EVENT_IR_CHANGE_RC_UNHIDE)
4909                         continue;
4910                 handle = le16_to_cpu(element->PhysDiskDevHandle);
4911                 volume_handle = le16_to_cpu(element->VolDevHandle);
4912                 clear_bit(handle, ioc->pd_handles);
4913                 if (!volume_handle)
4914                         _scsih_tm_tr_send(ioc, handle);
4915                 else if (volume_handle == a || volume_handle == b) {
4916                         delayed_tr = kzalloc(sizeof(*delayed_tr), GFP_ATOMIC);
4917                         BUG_ON(!delayed_tr);
4918                         INIT_LIST_HEAD(&delayed_tr->list);
4919                         delayed_tr->handle = handle;
4920                         list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);
4921                         dewtprintk(ioc,
4922                                    ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4923                                             handle));
4924                 } else
4925                         _scsih_tm_tr_send(ioc, handle);
4926         }
4927 }
4928
4929
4930 /**
4931  * _scsih_check_volume_delete_events - set delete flag for volumes
4932  * @ioc: per adapter object
4933  * @event_data: the event data payload
4934  * Context: interrupt time.
4935  *
4936  * This will handle the case when the cable connected to entire volume is
4937  * pulled. We will take care of setting the deleted flag so normal IO will
4938  * not be sent.
4939  */
4940 static void
4941 _scsih_check_volume_delete_events(struct MPT3SAS_ADAPTER *ioc,
4942         Mpi2EventDataIrVolume_t *event_data)
4943 {
4944         u32 state;
4945
4946         if (event_data->ReasonCode != MPI2_EVENT_IR_VOLUME_RC_STATE_CHANGED)
4947                 return;
4948         state = le32_to_cpu(event_data->NewValue);
4949         if (state == MPI2_RAID_VOL_STATE_MISSING || state ==
4950             MPI2_RAID_VOL_STATE_FAILED)
4951                 _scsih_set_volume_delete_flag(ioc,
4952                     le16_to_cpu(event_data->VolDevHandle));
4953 }
4954
4955 /**
4956  * _scsih_temp_threshold_events - display temperature threshold exceeded events
4957  * @ioc: per adapter object
4958  * @event_data: the temp threshold event data
4959  * Context: interrupt time.
4960  */
4961 static void
4962 _scsih_temp_threshold_events(struct MPT3SAS_ADAPTER *ioc,
4963         Mpi2EventDataTemperature_t *event_data)
4964 {
4965         u32 doorbell;
4966         if (ioc->temp_sensors_count >= event_data->SensorNum) {
4967                 ioc_err(ioc, "Temperature Threshold flags %s%s%s%s exceeded for Sensor: %d !!!\n",
4968                         le16_to_cpu(event_data->Status) & 0x1 ? "0 " : " ",
4969                         le16_to_cpu(event_data->Status) & 0x2 ? "1 " : " ",
4970                         le16_to_cpu(event_data->Status) & 0x4 ? "2 " : " ",
4971                         le16_to_cpu(event_data->Status) & 0x8 ? "3 " : " ",
4972                         event_data->SensorNum);
4973                 ioc_err(ioc, "Current Temp In Celsius: %d\n",
4974                         event_data->CurrentTemperature);
4975                 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
4976                         doorbell = mpt3sas_base_get_iocstate(ioc, 0);
4977                         if ((doorbell & MPI2_IOC_STATE_MASK) ==
4978                             MPI2_IOC_STATE_FAULT) {
4979                                 mpt3sas_print_fault_code(ioc,
4980                                     doorbell & MPI2_DOORBELL_DATA_MASK);
4981                         } else if ((doorbell & MPI2_IOC_STATE_MASK) ==
4982                             MPI2_IOC_STATE_COREDUMP) {
4983                                 mpt3sas_print_coredump_info(ioc,
4984                                     doorbell & MPI2_DOORBELL_DATA_MASK);
4985                         }
4986                 }
4987         }
4988 }
4989
4990 static int _scsih_set_satl_pending(struct scsi_cmnd *scmd, bool pending)
4991 {
4992         struct MPT3SAS_DEVICE *priv = scmd->device->hostdata;
4993
4994         if (scmd->cmnd[0] != ATA_12 && scmd->cmnd[0] != ATA_16)
4995                 return 0;
4996
4997         if (pending)
4998                 return test_and_set_bit(0, &priv->ata_command_pending);
4999
5000         clear_bit(0, &priv->ata_command_pending);
5001         return 0;
5002 }
5003
5004 /**
5005  * _scsih_flush_running_cmds - completing outstanding commands.
5006  * @ioc: per adapter object
5007  *
5008  * The flushing out of all pending scmd commands following host reset,
5009  * where all IO is dropped to the floor.
5010  */
5011 static void
5012 _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc)
5013 {
5014         struct scsi_cmnd *scmd;
5015         struct scsiio_tracker *st;
5016         u16 smid;
5017         int count = 0;
5018
5019         for (smid = 1; smid <= ioc->scsiio_depth; smid++) {
5020                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
5021                 if (!scmd)
5022                         continue;
5023                 count++;
5024                 _scsih_set_satl_pending(scmd, false);
5025                 st = scsi_cmd_priv(scmd);
5026                 mpt3sas_base_clear_st(ioc, st);
5027                 scsi_dma_unmap(scmd);
5028                 if (ioc->pci_error_recovery || ioc->remove_host)
5029                         scmd->result = DID_NO_CONNECT << 16;
5030                 else
5031                         scmd->result = DID_RESET << 16;
5032                 scsi_done(scmd);
5033         }
5034         dtmprintk(ioc, ioc_info(ioc, "completing %d cmds\n", count));
5035 }
5036
5037 /**
5038  * _scsih_setup_eedp - setup MPI request for EEDP transfer
5039  * @ioc: per adapter object
5040  * @scmd: pointer to scsi command object
5041  * @mpi_request: pointer to the SCSI_IO request message frame
5042  *
5043  * Supporting protection 1 and 3.
5044  */
5045 static void
5046 _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
5047         Mpi25SCSIIORequest_t *mpi_request)
5048 {
5049         u16 eedp_flags;
5050         Mpi25SCSIIORequest_t *mpi_request_3v =
5051            (Mpi25SCSIIORequest_t *)mpi_request;
5052
5053         switch (scsi_get_prot_op(scmd)) {
5054         case SCSI_PROT_READ_STRIP:
5055                 eedp_flags = MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP;
5056                 break;
5057         case SCSI_PROT_WRITE_INSERT:
5058                 eedp_flags = MPI2_SCSIIO_EEDPFLAGS_INSERT_OP;
5059                 break;
5060         default:
5061                 return;
5062         }
5063
5064         if (scmd->prot_flags & SCSI_PROT_GUARD_CHECK)
5065                 eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD;
5066
5067         if (scmd->prot_flags & SCSI_PROT_REF_CHECK)
5068                 eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG;
5069
5070         if (scmd->prot_flags & SCSI_PROT_REF_INCREMENT) {
5071                 eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG;
5072
5073                 mpi_request->CDB.EEDP32.PrimaryReferenceTag =
5074                         cpu_to_be32(scsi_prot_ref_tag(scmd));
5075         }
5076
5077         mpi_request_3v->EEDPBlockSize = cpu_to_le16(scsi_prot_interval(scmd));
5078
5079         if (ioc->is_gen35_ioc)
5080                 eedp_flags |= MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
5081         mpi_request->EEDPFlags = cpu_to_le16(eedp_flags);
5082 }
5083
5084 /**
5085  * _scsih_eedp_error_handling - return sense code for EEDP errors
5086  * @scmd: pointer to scsi command object
5087  * @ioc_status: ioc status
5088  */
5089 static void
5090 _scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
5091 {
5092         u8 ascq;
5093
5094         switch (ioc_status) {
5095         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5096                 ascq = 0x01;
5097                 break;
5098         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5099                 ascq = 0x02;
5100                 break;
5101         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5102                 ascq = 0x03;
5103                 break;
5104         default:
5105                 ascq = 0x00;
5106                 break;
5107         }
5108         scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x10, ascq);
5109         set_host_byte(scmd, DID_ABORT);
5110 }
5111
5112 /**
5113  * scsih_qcmd - main scsi request entry point
5114  * @shost: SCSI host pointer
5115  * @scmd: pointer to scsi command object
5116  *
5117  * The callback index is set inside `ioc->scsi_io_cb_idx`.
5118  *
5119  * Return: 0 on success.  If there's a failure, return either:
5120  * SCSI_MLQUEUE_DEVICE_BUSY if the device queue is full, or
5121  * SCSI_MLQUEUE_HOST_BUSY if the entire host queue is full
5122  */
5123 static int
5124 scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
5125 {
5126         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
5127         struct MPT3SAS_DEVICE *sas_device_priv_data;
5128         struct MPT3SAS_TARGET *sas_target_priv_data;
5129         struct _raid_device *raid_device;
5130         struct request *rq = scsi_cmd_to_rq(scmd);
5131         int class;
5132         Mpi25SCSIIORequest_t *mpi_request;
5133         struct _pcie_device *pcie_device = NULL;
5134         u32 mpi_control;
5135         u16 smid;
5136         u16 handle;
5137
5138         if (ioc->logging_level & MPT_DEBUG_SCSI)
5139                 scsi_print_command(scmd);
5140
5141         sas_device_priv_data = scmd->device->hostdata;
5142         if (!sas_device_priv_data || !sas_device_priv_data->sas_target) {
5143                 scmd->result = DID_NO_CONNECT << 16;
5144                 scsi_done(scmd);
5145                 return 0;
5146         }
5147
5148         if (!(_scsih_allow_scmd_to_device(ioc, scmd))) {
5149                 scmd->result = DID_NO_CONNECT << 16;
5150                 scsi_done(scmd);
5151                 return 0;
5152         }
5153
5154         sas_target_priv_data = sas_device_priv_data->sas_target;
5155
5156         /* invalid device handle */
5157         handle = sas_target_priv_data->handle;
5158
5159         /*
5160          * Avoid error handling escallation when device is disconnected
5161          */
5162         if (handle == MPT3SAS_INVALID_DEVICE_HANDLE || sas_device_priv_data->block) {
5163                 if (scmd->device->host->shost_state == SHOST_RECOVERY &&
5164                     scmd->cmnd[0] == TEST_UNIT_READY) {
5165                         scsi_build_sense(scmd, 0, UNIT_ATTENTION, 0x29, 0x07);
5166                         scsi_done(scmd);
5167                         return 0;
5168                 }
5169         }
5170
5171         if (handle == MPT3SAS_INVALID_DEVICE_HANDLE) {
5172                 scmd->result = DID_NO_CONNECT << 16;
5173                 scsi_done(scmd);
5174                 return 0;
5175         }
5176
5177
5178         if (ioc->shost_recovery || ioc->ioc_link_reset_in_progress) {
5179                 /* host recovery or link resets sent via IOCTLs */
5180                 return SCSI_MLQUEUE_HOST_BUSY;
5181         } else if (sas_target_priv_data->deleted) {
5182                 /* device has been deleted */
5183                 scmd->result = DID_NO_CONNECT << 16;
5184                 scsi_done(scmd);
5185                 return 0;
5186         } else if (sas_target_priv_data->tm_busy ||
5187                    sas_device_priv_data->block) {
5188                 /* device busy with task management */
5189                 return SCSI_MLQUEUE_DEVICE_BUSY;
5190         }
5191
5192         /*
5193          * Bug work around for firmware SATL handling.  The loop
5194          * is based on atomic operations and ensures consistency
5195          * since we're lockless at this point
5196          */
5197         do {
5198                 if (test_bit(0, &sas_device_priv_data->ata_command_pending))
5199                         return SCSI_MLQUEUE_DEVICE_BUSY;
5200         } while (_scsih_set_satl_pending(scmd, true));
5201
5202         if (scmd->sc_data_direction == DMA_FROM_DEVICE)
5203                 mpi_control = MPI2_SCSIIO_CONTROL_READ;
5204         else if (scmd->sc_data_direction == DMA_TO_DEVICE)
5205                 mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
5206         else
5207                 mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
5208
5209         /* set tags */
5210         mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
5211         /* NCQ Prio supported, make sure control indicated high priority */
5212         if (sas_device_priv_data->ncq_prio_enable) {
5213                 class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
5214                 if (class == IOPRIO_CLASS_RT)
5215                         mpi_control |= 1 << MPI2_SCSIIO_CONTROL_CMDPRI_SHIFT;
5216         }
5217         /* Make sure Device is not raid volume.
5218          * We do not expose raid functionality to upper layer for warpdrive.
5219          */
5220         if (((!ioc->is_warpdrive && !scsih_is_raid(&scmd->device->sdev_gendev))
5221                 && !scsih_is_nvme(&scmd->device->sdev_gendev))
5222                 && sas_is_tlr_enabled(scmd->device) && scmd->cmd_len != 32)
5223                 mpi_control |= MPI2_SCSIIO_CONTROL_TLR_ON;
5224
5225         smid = mpt3sas_base_get_smid_scsiio(ioc, ioc->scsi_io_cb_idx, scmd);
5226         if (!smid) {
5227                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
5228                 _scsih_set_satl_pending(scmd, false);
5229                 goto out;
5230         }
5231         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5232         memset(mpi_request, 0, ioc->request_sz);
5233         _scsih_setup_eedp(ioc, scmd, mpi_request);
5234
5235         if (scmd->cmd_len == 32)
5236                 mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
5237         mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
5238         if (sas_device_priv_data->sas_target->flags &
5239             MPT_TARGET_FLAGS_RAID_COMPONENT)
5240                 mpi_request->Function = MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH;
5241         else
5242                 mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
5243         mpi_request->DevHandle = cpu_to_le16(handle);
5244         mpi_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
5245         mpi_request->Control = cpu_to_le32(mpi_control);
5246         mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len);
5247         mpi_request->MsgFlags = MPI2_SCSIIO_MSGFLAGS_SYSTEM_SENSE_ADDR;
5248         mpi_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
5249         mpi_request->SenseBufferLowAddress =
5250             mpt3sas_base_get_sense_buffer_dma(ioc, smid);
5251         mpi_request->SGLOffset0 = offsetof(Mpi25SCSIIORequest_t, SGL) / 4;
5252         int_to_scsilun(sas_device_priv_data->lun, (struct scsi_lun *)
5253             mpi_request->LUN);
5254         memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
5255
5256         if (mpi_request->DataLength) {
5257                 pcie_device = sas_target_priv_data->pcie_dev;
5258                 if (ioc->build_sg_scmd(ioc, scmd, smid, pcie_device)) {
5259                         mpt3sas_base_free_smid(ioc, smid);
5260                         _scsih_set_satl_pending(scmd, false);
5261                         goto out;
5262                 }
5263         } else
5264                 ioc->build_zero_len_sge(ioc, &mpi_request->SGL);
5265
5266         raid_device = sas_target_priv_data->raid_device;
5267         if (raid_device && raid_device->direct_io_enabled)
5268                 mpt3sas_setup_direct_io(ioc, scmd,
5269                         raid_device, mpi_request);
5270
5271         if (likely(mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST)) {
5272                 if (sas_target_priv_data->flags & MPT_TARGET_FASTPATH_IO) {
5273                         mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len |
5274                             MPI25_SCSIIO_IOFLAGS_FAST_PATH);
5275                         ioc->put_smid_fast_path(ioc, smid, handle);
5276                 } else
5277                         ioc->put_smid_scsi_io(ioc, smid,
5278                             le16_to_cpu(mpi_request->DevHandle));
5279         } else
5280                 ioc->put_smid_default(ioc, smid);
5281         return 0;
5282
5283  out:
5284         return SCSI_MLQUEUE_HOST_BUSY;
5285 }
5286
5287 /**
5288  * _scsih_normalize_sense - normalize descriptor and fixed format sense data
5289  * @sense_buffer: sense data returned by target
5290  * @data: normalized skey/asc/ascq
5291  */
5292 static void
5293 _scsih_normalize_sense(char *sense_buffer, struct sense_info *data)
5294 {
5295         if ((sense_buffer[0] & 0x7F) >= 0x72) {
5296                 /* descriptor format */
5297                 data->skey = sense_buffer[1] & 0x0F;
5298                 data->asc = sense_buffer[2];
5299                 data->ascq = sense_buffer[3];
5300         } else {
5301                 /* fixed format */
5302                 data->skey = sense_buffer[2] & 0x0F;
5303                 data->asc = sense_buffer[12];
5304                 data->ascq = sense_buffer[13];
5305         }
5306 }
5307
5308 /**
5309  * _scsih_scsi_ioc_info - translated non-successful SCSI_IO request
5310  * @ioc: per adapter object
5311  * @scmd: pointer to scsi command object
5312  * @mpi_reply: reply mf payload returned from firmware
5313  * @smid: ?
5314  *
5315  * scsi_status - SCSI Status code returned from target device
5316  * scsi_state - state info associated with SCSI_IO determined by ioc
5317  * ioc_status - ioc supplied status info
5318  */
5319 static void
5320 _scsih_scsi_ioc_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
5321         Mpi2SCSIIOReply_t *mpi_reply, u16 smid)
5322 {
5323         u32 response_info;
5324         u8 *response_bytes;
5325         u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
5326             MPI2_IOCSTATUS_MASK;
5327         u8 scsi_state = mpi_reply->SCSIState;
5328         u8 scsi_status = mpi_reply->SCSIStatus;
5329         char *desc_ioc_state = NULL;
5330         char *desc_scsi_status = NULL;
5331         char *desc_scsi_state = ioc->tmp_string;
5332         u32 log_info = le32_to_cpu(mpi_reply->IOCLogInfo);
5333         struct _sas_device *sas_device = NULL;
5334         struct _pcie_device *pcie_device = NULL;
5335         struct scsi_target *starget = scmd->device->sdev_target;
5336         struct MPT3SAS_TARGET *priv_target = starget->hostdata;
5337         char *device_str = NULL;
5338
5339         if (!priv_target)
5340                 return;
5341         if (ioc->hide_ir_msg)
5342                 device_str = "WarpDrive";
5343         else
5344                 device_str = "volume";
5345
5346         if (log_info == 0x31170000)
5347                 return;
5348
5349         switch (ioc_status) {
5350         case MPI2_IOCSTATUS_SUCCESS:
5351                 desc_ioc_state = "success";
5352                 break;
5353         case MPI2_IOCSTATUS_INVALID_FUNCTION:
5354                 desc_ioc_state = "invalid function";
5355                 break;
5356         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
5357                 desc_ioc_state = "scsi recovered error";
5358                 break;
5359         case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
5360                 desc_ioc_state = "scsi invalid dev handle";
5361                 break;
5362         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5363                 desc_ioc_state = "scsi device not there";
5364                 break;
5365         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5366                 desc_ioc_state = "scsi data overrun";
5367                 break;
5368         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
5369                 desc_ioc_state = "scsi data underrun";
5370                 break;
5371         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5372                 desc_ioc_state = "scsi io data error";
5373                 break;
5374         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5375                 desc_ioc_state = "scsi protocol error";
5376                 break;
5377         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5378                 desc_ioc_state = "scsi task terminated";
5379                 break;
5380         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
5381                 desc_ioc_state = "scsi residual mismatch";
5382                 break;
5383         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
5384                 desc_ioc_state = "scsi task mgmt failed";
5385                 break;
5386         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5387                 desc_ioc_state = "scsi ioc terminated";
5388                 break;
5389         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5390                 desc_ioc_state = "scsi ext terminated";
5391                 break;
5392         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5393                 desc_ioc_state = "eedp guard error";
5394                 break;
5395         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5396                 desc_ioc_state = "eedp ref tag error";
5397                 break;
5398         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5399                 desc_ioc_state = "eedp app tag error";
5400                 break;
5401         case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
5402                 desc_ioc_state = "insufficient power";
5403                 break;
5404         default:
5405                 desc_ioc_state = "unknown";
5406                 break;
5407         }
5408
5409         switch (scsi_status) {
5410         case MPI2_SCSI_STATUS_GOOD:
5411                 desc_scsi_status = "good";
5412                 break;
5413         case MPI2_SCSI_STATUS_CHECK_CONDITION:
5414                 desc_scsi_status = "check condition";
5415                 break;
5416         case MPI2_SCSI_STATUS_CONDITION_MET:
5417                 desc_scsi_status = "condition met";
5418                 break;
5419         case MPI2_SCSI_STATUS_BUSY:
5420                 desc_scsi_status = "busy";
5421                 break;
5422         case MPI2_SCSI_STATUS_INTERMEDIATE:
5423                 desc_scsi_status = "intermediate";
5424                 break;
5425         case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
5426                 desc_scsi_status = "intermediate condmet";
5427                 break;
5428         case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
5429                 desc_scsi_status = "reservation conflict";
5430                 break;
5431         case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
5432                 desc_scsi_status = "command terminated";
5433                 break;
5434         case MPI2_SCSI_STATUS_TASK_SET_FULL:
5435                 desc_scsi_status = "task set full";
5436                 break;
5437         case MPI2_SCSI_STATUS_ACA_ACTIVE:
5438                 desc_scsi_status = "aca active";
5439                 break;
5440         case MPI2_SCSI_STATUS_TASK_ABORTED:
5441                 desc_scsi_status = "task aborted";
5442                 break;
5443         default:
5444                 desc_scsi_status = "unknown";
5445                 break;
5446         }
5447
5448         desc_scsi_state[0] = '\0';
5449         if (!scsi_state)
5450                 desc_scsi_state = " ";
5451         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
5452                 strcat(desc_scsi_state, "response info ");
5453         if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5454                 strcat(desc_scsi_state, "state terminated ");
5455         if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
5456                 strcat(desc_scsi_state, "no status ");
5457         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
5458                 strcat(desc_scsi_state, "autosense failed ");
5459         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
5460                 strcat(desc_scsi_state, "autosense valid ");
5461
5462         scsi_print_command(scmd);
5463
5464         if (priv_target->flags & MPT_TARGET_FLAGS_VOLUME) {
5465                 ioc_warn(ioc, "\t%s wwid(0x%016llx)\n",
5466                          device_str, (u64)priv_target->sas_address);
5467         } else if (priv_target->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
5468                 pcie_device = mpt3sas_get_pdev_from_target(ioc, priv_target);
5469                 if (pcie_device) {
5470                         ioc_info(ioc, "\twwid(0x%016llx), port(%d)\n",
5471                                  (u64)pcie_device->wwid, pcie_device->port_num);
5472                         if (pcie_device->enclosure_handle != 0)
5473                                 ioc_info(ioc, "\tenclosure logical id(0x%016llx), slot(%d)\n",
5474                                          (u64)pcie_device->enclosure_logical_id,
5475                                          pcie_device->slot);
5476                         if (pcie_device->connector_name[0])
5477                                 ioc_info(ioc, "\tenclosure level(0x%04x), connector name( %s)\n",
5478                                          pcie_device->enclosure_level,
5479                                          pcie_device->connector_name);
5480                         pcie_device_put(pcie_device);
5481                 }
5482         } else {
5483                 sas_device = mpt3sas_get_sdev_from_target(ioc, priv_target);
5484                 if (sas_device) {
5485                         ioc_warn(ioc, "\tsas_address(0x%016llx), phy(%d)\n",
5486                                  (u64)sas_device->sas_address, sas_device->phy);
5487
5488                         _scsih_display_enclosure_chassis_info(ioc, sas_device,
5489                             NULL, NULL);
5490
5491                         sas_device_put(sas_device);
5492                 }
5493         }
5494
5495         ioc_warn(ioc, "\thandle(0x%04x), ioc_status(%s)(0x%04x), smid(%d)\n",
5496                  le16_to_cpu(mpi_reply->DevHandle),
5497                  desc_ioc_state, ioc_status, smid);
5498         ioc_warn(ioc, "\trequest_len(%d), underflow(%d), resid(%d)\n",
5499                  scsi_bufflen(scmd), scmd->underflow, scsi_get_resid(scmd));
5500         ioc_warn(ioc, "\ttag(%d), transfer_count(%d), sc->result(0x%08x)\n",
5501                  le16_to_cpu(mpi_reply->TaskTag),
5502                  le32_to_cpu(mpi_reply->TransferCount), scmd->result);
5503         ioc_warn(ioc, "\tscsi_status(%s)(0x%02x), scsi_state(%s)(0x%02x)\n",
5504                  desc_scsi_status, scsi_status, desc_scsi_state, scsi_state);
5505
5506         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5507                 struct sense_info data;
5508                 _scsih_normalize_sense(scmd->sense_buffer, &data);
5509                 ioc_warn(ioc, "\t[sense_key,asc,ascq]: [0x%02x,0x%02x,0x%02x], count(%d)\n",
5510                          data.skey, data.asc, data.ascq,
5511                          le32_to_cpu(mpi_reply->SenseCount));
5512         }
5513         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
5514                 response_info = le32_to_cpu(mpi_reply->ResponseInfo);
5515                 response_bytes = (u8 *)&response_info;
5516                 _scsih_response_code(ioc, response_bytes[0]);
5517         }
5518 }
5519
5520 /**
5521  * _scsih_turn_on_pfa_led - illuminate PFA LED
5522  * @ioc: per adapter object
5523  * @handle: device handle
5524  * Context: process
5525  */
5526 static void
5527 _scsih_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5528 {
5529         Mpi2SepReply_t mpi_reply;
5530         Mpi2SepRequest_t mpi_request;
5531         struct _sas_device *sas_device;
5532
5533         sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
5534         if (!sas_device)
5535                 return;
5536
5537         memset(&mpi_request, 0, sizeof(Mpi2SepRequest_t));
5538         mpi_request.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
5539         mpi_request.Action = MPI2_SEP_REQ_ACTION_WRITE_STATUS;
5540         mpi_request.SlotStatus =
5541             cpu_to_le32(MPI2_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT);
5542         mpi_request.DevHandle = cpu_to_le16(handle);
5543         mpi_request.Flags = MPI2_SEP_REQ_FLAGS_DEVHANDLE_ADDRESS;
5544         if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,
5545             &mpi_request)) != 0) {
5546                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5547                         __FILE__, __LINE__, __func__);
5548                 goto out;
5549         }
5550         sas_device->pfa_led_on = 1;
5551
5552         if (mpi_reply.IOCStatus || mpi_reply.IOCLogInfo) {
5553                 dewtprintk(ioc,
5554                            ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",
5555                                     le16_to_cpu(mpi_reply.IOCStatus),
5556                                     le32_to_cpu(mpi_reply.IOCLogInfo)));
5557                 goto out;
5558         }
5559 out:
5560         sas_device_put(sas_device);
5561 }
5562
5563 /**
5564  * _scsih_turn_off_pfa_led - turn off Fault LED
5565  * @ioc: per adapter object
5566  * @sas_device: sas device whose PFA LED has to turned off
5567  * Context: process
5568  */
5569 static void
5570 _scsih_turn_off_pfa_led(struct MPT3SAS_ADAPTER *ioc,
5571         struct _sas_device *sas_device)
5572 {
5573         Mpi2SepReply_t mpi_reply;
5574         Mpi2SepRequest_t mpi_request;
5575
5576         memset(&mpi_request, 0, sizeof(Mpi2SepRequest_t));
5577         mpi_request.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
5578         mpi_request.Action = MPI2_SEP_REQ_ACTION_WRITE_STATUS;
5579         mpi_request.SlotStatus = 0;
5580         mpi_request.Slot = cpu_to_le16(sas_device->slot);
5581         mpi_request.DevHandle = 0;
5582         mpi_request.EnclosureHandle = cpu_to_le16(sas_device->enclosure_handle);
5583         mpi_request.Flags = MPI2_SEP_REQ_FLAGS_ENCLOSURE_SLOT_ADDRESS;
5584         if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,
5585                 &mpi_request)) != 0) {
5586                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5587                         __FILE__, __LINE__, __func__);
5588                 return;
5589         }
5590
5591         if (mpi_reply.IOCStatus || mpi_reply.IOCLogInfo) {
5592                 dewtprintk(ioc,
5593                            ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",
5594                                     le16_to_cpu(mpi_reply.IOCStatus),
5595                                     le32_to_cpu(mpi_reply.IOCLogInfo)));
5596                 return;
5597         }
5598 }
5599
5600 /**
5601  * _scsih_send_event_to_turn_on_pfa_led - fire delayed event
5602  * @ioc: per adapter object
5603  * @handle: device handle
5604  * Context: interrupt.
5605  */
5606 static void
5607 _scsih_send_event_to_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5608 {
5609         struct fw_event_work *fw_event;
5610
5611         fw_event = alloc_fw_event_work(0);
5612         if (!fw_event)
5613                 return;
5614         fw_event->event = MPT3SAS_TURN_ON_PFA_LED;
5615         fw_event->device_handle = handle;
5616         fw_event->ioc = ioc;
5617         _scsih_fw_event_add(ioc, fw_event);
5618         fw_event_work_put(fw_event);
5619 }
5620
5621 /**
5622  * _scsih_smart_predicted_fault - process smart errors
5623  * @ioc: per adapter object
5624  * @handle: device handle
5625  * Context: interrupt.
5626  */
5627 static void
5628 _scsih_smart_predicted_fault(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5629 {
5630         struct scsi_target *starget;
5631         struct MPT3SAS_TARGET *sas_target_priv_data;
5632         Mpi2EventNotificationReply_t *event_reply;
5633         Mpi2EventDataSasDeviceStatusChange_t *event_data;
5634         struct _sas_device *sas_device;
5635         ssize_t sz;
5636         unsigned long flags;
5637
5638         /* only handle non-raid devices */
5639         spin_lock_irqsave(&ioc->sas_device_lock, flags);
5640         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
5641         if (!sas_device)
5642                 goto out_unlock;
5643
5644         starget = sas_device->starget;
5645         sas_target_priv_data = starget->hostdata;
5646
5647         if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT) ||
5648            ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)))
5649                 goto out_unlock;
5650
5651         _scsih_display_enclosure_chassis_info(NULL, sas_device, NULL, starget);
5652
5653         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
5654
5655         if (ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM)
5656                 _scsih_send_event_to_turn_on_pfa_led(ioc, handle);
5657
5658         /* insert into event log */
5659         sz = offsetof(Mpi2EventNotificationReply_t, EventData) +
5660              sizeof(Mpi2EventDataSasDeviceStatusChange_t);
5661         event_reply = kzalloc(sz, GFP_ATOMIC);
5662         if (!event_reply) {
5663                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5664                         __FILE__, __LINE__, __func__);
5665                 goto out;
5666         }
5667
5668         event_reply->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
5669         event_reply->Event =
5670             cpu_to_le16(MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
5671         event_reply->MsgLength = sz/4;
5672         event_reply->EventDataLength =
5673             cpu_to_le16(sizeof(Mpi2EventDataSasDeviceStatusChange_t)/4);
5674         event_data = (Mpi2EventDataSasDeviceStatusChange_t *)
5675             event_reply->EventData;
5676         event_data->ReasonCode = MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA;
5677         event_data->ASC = 0x5D;
5678         event_data->DevHandle = cpu_to_le16(handle);
5679         event_data->SASAddress = cpu_to_le64(sas_target_priv_data->sas_address);
5680         mpt3sas_ctl_add_to_event_log(ioc, event_reply);
5681         kfree(event_reply);
5682 out:
5683         if (sas_device)
5684                 sas_device_put(sas_device);
5685         return;
5686
5687 out_unlock:
5688         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
5689         goto out;
5690 }
5691
5692 /**
5693  * _scsih_io_done - scsi request callback
5694  * @ioc: per adapter object
5695  * @smid: system request message index
5696  * @msix_index: MSIX table index supplied by the OS
5697  * @reply: reply message frame(lower 32bit addr)
5698  *
5699  * Callback handler when using _scsih_qcmd.
5700  *
5701  * Return: 1 meaning mf should be freed from _base_interrupt
5702  *         0 means the mf is freed from this function.
5703  */
5704 static u8
5705 _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
5706 {
5707         Mpi25SCSIIORequest_t *mpi_request;
5708         Mpi2SCSIIOReply_t *mpi_reply;
5709         struct scsi_cmnd *scmd;
5710         struct scsiio_tracker *st;
5711         u16 ioc_status;
5712         u32 xfer_cnt;
5713         u8 scsi_state;
5714         u8 scsi_status;
5715         u32 log_info;
5716         struct MPT3SAS_DEVICE *sas_device_priv_data;
5717         u32 response_code = 0;
5718
5719         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
5720
5721         scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
5722         if (scmd == NULL)
5723                 return 1;
5724
5725         _scsih_set_satl_pending(scmd, false);
5726
5727         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5728
5729         if (mpi_reply == NULL) {
5730                 scmd->result = DID_OK << 16;
5731                 goto out;
5732         }
5733
5734         sas_device_priv_data = scmd->device->hostdata;
5735         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
5736              sas_device_priv_data->sas_target->deleted) {
5737                 scmd->result = DID_NO_CONNECT << 16;
5738                 goto out;
5739         }
5740         ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
5741
5742         /*
5743          * WARPDRIVE: If direct_io is set then it is directIO,
5744          * the failed direct I/O should be redirected to volume
5745          */
5746         st = scsi_cmd_priv(scmd);
5747         if (st->direct_io &&
5748              ((ioc_status & MPI2_IOCSTATUS_MASK)
5749               != MPI2_IOCSTATUS_SCSI_TASK_TERMINATED)) {
5750                 st->direct_io = 0;
5751                 st->scmd = scmd;
5752                 memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
5753                 mpi_request->DevHandle =
5754                     cpu_to_le16(sas_device_priv_data->sas_target->handle);
5755                 ioc->put_smid_scsi_io(ioc, smid,
5756                     sas_device_priv_data->sas_target->handle);
5757                 return 0;
5758         }
5759         /* turning off TLR */
5760         scsi_state = mpi_reply->SCSIState;
5761         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
5762                 response_code =
5763                     le32_to_cpu(mpi_reply->ResponseInfo) & 0xFF;
5764         if (!sas_device_priv_data->tlr_snoop_check) {
5765                 sas_device_priv_data->tlr_snoop_check++;
5766                 if ((!ioc->is_warpdrive &&
5767                     !scsih_is_raid(&scmd->device->sdev_gendev) &&
5768                     !scsih_is_nvme(&scmd->device->sdev_gendev))
5769                     && sas_is_tlr_enabled(scmd->device) &&
5770                     response_code == MPI2_SCSITASKMGMT_RSP_INVALID_FRAME) {
5771                         sas_disable_tlr(scmd->device);
5772                         sdev_printk(KERN_INFO, scmd->device, "TLR disabled\n");
5773                 }
5774         }
5775
5776         xfer_cnt = le32_to_cpu(mpi_reply->TransferCount);
5777         scsi_set_resid(scmd, scsi_bufflen(scmd) - xfer_cnt);
5778         if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
5779                 log_info =  le32_to_cpu(mpi_reply->IOCLogInfo);
5780         else
5781                 log_info = 0;
5782         ioc_status &= MPI2_IOCSTATUS_MASK;
5783         scsi_status = mpi_reply->SCSIStatus;
5784
5785         if (ioc_status == MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN && xfer_cnt == 0 &&
5786             (scsi_status == MPI2_SCSI_STATUS_BUSY ||
5787              scsi_status == MPI2_SCSI_STATUS_RESERVATION_CONFLICT ||
5788              scsi_status == MPI2_SCSI_STATUS_TASK_SET_FULL)) {
5789                 ioc_status = MPI2_IOCSTATUS_SUCCESS;
5790         }
5791
5792         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5793                 struct sense_info data;
5794                 const void *sense_data = mpt3sas_base_get_sense_buffer(ioc,
5795                     smid);
5796                 u32 sz = min_t(u32, SCSI_SENSE_BUFFERSIZE,
5797                     le32_to_cpu(mpi_reply->SenseCount));
5798                 memcpy(scmd->sense_buffer, sense_data, sz);
5799                 _scsih_normalize_sense(scmd->sense_buffer, &data);
5800                 /* failure prediction threshold exceeded */
5801                 if (data.asc == 0x5D)
5802                         _scsih_smart_predicted_fault(ioc,
5803                             le16_to_cpu(mpi_reply->DevHandle));
5804                 mpt3sas_trigger_scsi(ioc, data.skey, data.asc, data.ascq);
5805
5806                 if ((ioc->logging_level & MPT_DEBUG_REPLY) &&
5807                      ((scmd->sense_buffer[2] == UNIT_ATTENTION) ||
5808                      (scmd->sense_buffer[2] == MEDIUM_ERROR) ||
5809                      (scmd->sense_buffer[2] == HARDWARE_ERROR)))
5810                         _scsih_scsi_ioc_info(ioc, scmd, mpi_reply, smid);
5811         }
5812         switch (ioc_status) {
5813         case MPI2_IOCSTATUS_BUSY:
5814         case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
5815                 scmd->result = SAM_STAT_BUSY;
5816                 break;
5817
5818         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5819                 scmd->result = DID_NO_CONNECT << 16;
5820                 break;
5821
5822         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5823                 if (sas_device_priv_data->block) {
5824                         scmd->result = DID_TRANSPORT_DISRUPTED << 16;
5825                         goto out;
5826                 }
5827                 if (log_info == 0x31110630) {
5828                         if (scmd->retries > 2) {
5829                                 scmd->result = DID_NO_CONNECT << 16;
5830                                 scsi_device_set_state(scmd->device,
5831                                     SDEV_OFFLINE);
5832                         } else {
5833                                 scmd->result = DID_SOFT_ERROR << 16;
5834                                 scmd->device->expecting_cc_ua = 1;
5835                         }
5836                         break;
5837                 } else if (log_info == VIRTUAL_IO_FAILED_RETRY) {
5838                         scmd->result = DID_RESET << 16;
5839                         break;
5840                 } else if ((scmd->device->channel == RAID_CHANNEL) &&
5841                    (scsi_state == (MPI2_SCSI_STATE_TERMINATED |
5842                    MPI2_SCSI_STATE_NO_SCSI_STATUS))) {
5843                         scmd->result = DID_RESET << 16;
5844                         break;
5845                 }
5846                 scmd->result = DID_SOFT_ERROR << 16;
5847                 break;
5848         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5849         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5850                 scmd->result = DID_RESET << 16;
5851                 break;
5852
5853         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
5854                 if ((xfer_cnt == 0) || (scmd->underflow > xfer_cnt))
5855                         scmd->result = DID_SOFT_ERROR << 16;
5856                 else
5857                         scmd->result = (DID_OK << 16) | scsi_status;
5858                 break;
5859
5860         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
5861                 scmd->result = (DID_OK << 16) | scsi_status;
5862
5863                 if ((scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID))
5864                         break;
5865
5866                 if (xfer_cnt < scmd->underflow) {
5867                         if (scsi_status == SAM_STAT_BUSY)
5868                                 scmd->result = SAM_STAT_BUSY;
5869                         else
5870                                 scmd->result = DID_SOFT_ERROR << 16;
5871                 } else if (scsi_state & (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
5872                      MPI2_SCSI_STATE_NO_SCSI_STATUS))
5873                         scmd->result = DID_SOFT_ERROR << 16;
5874                 else if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5875                         scmd->result = DID_RESET << 16;
5876                 else if (!xfer_cnt && scmd->cmnd[0] == REPORT_LUNS) {
5877                         mpi_reply->SCSIState = MPI2_SCSI_STATE_AUTOSENSE_VALID;
5878                         mpi_reply->SCSIStatus = SAM_STAT_CHECK_CONDITION;
5879                         scsi_build_sense(scmd, 0, ILLEGAL_REQUEST,
5880                                          0x20, 0);
5881                 }
5882                 break;
5883
5884         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5885                 scsi_set_resid(scmd, 0);
5886                 fallthrough;
5887         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
5888         case MPI2_IOCSTATUS_SUCCESS:
5889                 scmd->result = (DID_OK << 16) | scsi_status;
5890                 if (response_code ==
5891                     MPI2_SCSITASKMGMT_RSP_INVALID_FRAME ||
5892                     (scsi_state & (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
5893                      MPI2_SCSI_STATE_NO_SCSI_STATUS)))
5894                         scmd->result = DID_SOFT_ERROR << 16;
5895                 else if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5896                         scmd->result = DID_RESET << 16;
5897                 break;
5898
5899         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5900         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5901         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5902                 _scsih_eedp_error_handling(scmd, ioc_status);
5903                 break;
5904
5905         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5906         case MPI2_IOCSTATUS_INVALID_FUNCTION:
5907         case MPI2_IOCSTATUS_INVALID_SGL:
5908         case MPI2_IOCSTATUS_INTERNAL_ERROR:
5909         case MPI2_IOCSTATUS_INVALID_FIELD:
5910         case MPI2_IOCSTATUS_INVALID_STATE:
5911         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5912         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
5913         case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
5914         default:
5915                 scmd->result = DID_SOFT_ERROR << 16;
5916                 break;
5917
5918         }
5919
5920         if (scmd->result && (ioc->logging_level & MPT_DEBUG_REPLY))
5921                 _scsih_scsi_ioc_info(ioc , scmd, mpi_reply, smid);
5922
5923  out:
5924
5925         scsi_dma_unmap(scmd);
5926         mpt3sas_base_free_smid(ioc, smid);
5927         scsi_done(scmd);
5928         return 0;
5929 }
5930
5931 /**
5932  * _scsih_update_vphys_after_reset - update the Port's
5933  *                      vphys_list after reset
5934  * @ioc: per adapter object
5935  *
5936  * Returns nothing.
5937  */
5938 static void
5939 _scsih_update_vphys_after_reset(struct MPT3SAS_ADAPTER *ioc)
5940 {
5941         u16 sz, ioc_status;
5942         int i;
5943         Mpi2ConfigReply_t mpi_reply;
5944         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
5945         u16 attached_handle;
5946         u64 attached_sas_addr;
5947         u8 found = 0, port_id;
5948         Mpi2SasPhyPage0_t phy_pg0;
5949         struct hba_port *port, *port_next, *mport;
5950         struct virtual_phy *vphy, *vphy_next;
5951         struct _sas_device *sas_device;
5952
5953         /*
5954          * Mark all the vphys objects as dirty.
5955          */
5956         list_for_each_entry_safe(port, port_next,
5957             &ioc->port_table_list, list) {
5958                 if (!port->vphys_mask)
5959                         continue;
5960                 list_for_each_entry_safe(vphy, vphy_next,
5961                     &port->vphys_list, list) {
5962                         vphy->flags |= MPT_VPHY_FLAG_DIRTY_PHY;
5963                 }
5964         }
5965
5966         /*
5967          * Read SASIOUnitPage0 to get each HBA Phy's data.
5968          */
5969         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) +
5970             (ioc->sas_hba.num_phys * sizeof(Mpi2SasIOUnit0PhyData_t));
5971         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
5972         if (!sas_iounit_pg0) {
5973                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5974                     __FILE__, __LINE__, __func__);
5975                 return;
5976         }
5977         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
5978             sas_iounit_pg0, sz)) != 0)
5979                 goto out;
5980         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
5981         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5982                 goto out;
5983         /*
5984          * Loop over each HBA Phy.
5985          */
5986         for (i = 0; i < ioc->sas_hba.num_phys; i++) {
5987                 /*
5988                  * Check whether Phy's Negotiation Link Rate is > 1.5G or not.
5989                  */
5990                 if ((sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4) <
5991                     MPI2_SAS_NEG_LINK_RATE_1_5)
5992                         continue;
5993                 /*
5994                  * Check whether Phy is connected to SEP device or not,
5995                  * if it is SEP device then read the Phy's SASPHYPage0 data to
5996                  * determine whether Phy is a virtual Phy or not. if it is
5997                  * virtual phy then it is conformed that the attached remote
5998                  * device is a HBA's vSES device.
5999                  */
6000                 if (!(le32_to_cpu(
6001                     sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
6002                     MPI2_SAS_DEVICE_INFO_SEP))
6003                         continue;
6004
6005                 if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,
6006                     i))) {
6007                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6008                             __FILE__, __LINE__, __func__);
6009                         continue;
6010                 }
6011
6012                 if (!(le32_to_cpu(phy_pg0.PhyInfo) &
6013                     MPI2_SAS_PHYINFO_VIRTUAL_PHY))
6014                         continue;
6015                 /*
6016                  * Get the vSES device's SAS Address.
6017                  */
6018                 attached_handle = le16_to_cpu(
6019                     sas_iounit_pg0->PhyData[i].AttachedDevHandle);
6020                 if (_scsih_get_sas_address(ioc, attached_handle,
6021                     &attached_sas_addr) != 0) {
6022                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6023                             __FILE__, __LINE__, __func__);
6024                         continue;
6025                 }
6026
6027                 found = 0;
6028                 port = port_next = NULL;
6029                 /*
6030                  * Loop over each virtual_phy object from
6031                  * each port's vphys_list.
6032                  */
6033                 list_for_each_entry_safe(port,
6034                     port_next, &ioc->port_table_list, list) {
6035                         if (!port->vphys_mask)
6036                                 continue;
6037                         list_for_each_entry_safe(vphy, vphy_next,
6038                             &port->vphys_list, list) {
6039                                 /*
6040                                  * Continue with next virtual_phy object
6041                                  * if the object is not marked as dirty.
6042                                  */
6043                                 if (!(vphy->flags & MPT_VPHY_FLAG_DIRTY_PHY))
6044                                         continue;
6045
6046                                 /*
6047                                  * Continue with next virtual_phy object
6048                                  * if the object's SAS Address is not equals
6049                                  * to current Phy's vSES device SAS Address.
6050                                  */
6051                                 if (vphy->sas_address != attached_sas_addr)
6052                                         continue;
6053                                 /*
6054                                  * Enable current Phy number bit in object's
6055                                  * phy_mask field.
6056                                  */
6057                                 if (!(vphy->phy_mask & (1 << i)))
6058                                         vphy->phy_mask = (1 << i);
6059                                 /*
6060                                  * Get hba_port object from hba_port table
6061                                  * corresponding to current phy's Port ID.
6062                                  * if there is no hba_port object corresponding
6063                                  * to Phy's Port ID then create a new hba_port
6064                                  * object & add to hba_port table.
6065                                  */
6066                                 port_id = sas_iounit_pg0->PhyData[i].Port;
6067                                 mport = mpt3sas_get_port_by_id(ioc, port_id, 1);
6068                                 if (!mport) {
6069                                         mport = kzalloc(
6070                                             sizeof(struct hba_port), GFP_KERNEL);
6071                                         if (!mport)
6072                                                 break;
6073                                         mport->port_id = port_id;
6074                                         ioc_info(ioc,
6075                                             "%s: hba_port entry: %p, port: %d is added to hba_port list\n",
6076                                             __func__, mport, mport->port_id);
6077                                         list_add_tail(&mport->list,
6078                                                 &ioc->port_table_list);
6079                                 }
6080                                 /*
6081                                  * If mport & port pointers are not pointing to
6082                                  * same hba_port object then it means that vSES
6083                                  * device's Port ID got changed after reset and
6084                                  * hence move current virtual_phy object from
6085                                  * port's vphys_list to mport's vphys_list.
6086                                  */
6087                                 if (port != mport) {
6088                                         if (!mport->vphys_mask)
6089                                                 INIT_LIST_HEAD(
6090                                                     &mport->vphys_list);
6091                                         mport->vphys_mask |= (1 << i);
6092                                         port->vphys_mask &= ~(1 << i);
6093                                         list_move(&vphy->list,
6094                                             &mport->vphys_list);
6095                                         sas_device = mpt3sas_get_sdev_by_addr(
6096                                             ioc, attached_sas_addr, port);
6097                                         if (sas_device)
6098                                                 sas_device->port = mport;
6099                                 }
6100                                 /*
6101                                  * Earlier while updating the hba_port table,
6102                                  * it is determined that there is no other
6103                                  * direct attached device with mport's Port ID,
6104                                  * Hence mport was marked as dirty. Only vSES
6105                                  * device has this Port ID, so unmark the mport
6106                                  * as dirt.
6107                                  */
6108                                 if (mport->flags & HBA_PORT_FLAG_DIRTY_PORT) {
6109                                         mport->sas_address = 0;
6110                                         mport->phy_mask = 0;
6111                                         mport->flags &=
6112                                             ~HBA_PORT_FLAG_DIRTY_PORT;
6113                                 }
6114                                 /*
6115                                  * Unmark current virtual_phy object as dirty.
6116                                  */
6117                                 vphy->flags &= ~MPT_VPHY_FLAG_DIRTY_PHY;
6118                                 found = 1;
6119                                 break;
6120                         }
6121                         if (found)
6122                                 break;
6123                 }
6124         }
6125 out:
6126         kfree(sas_iounit_pg0);
6127 }
6128
6129 /**
6130  * _scsih_get_port_table_after_reset - Construct temporary port table
6131  * @ioc: per adapter object
6132  * @port_table: address where port table needs to be constructed
6133  *
6134  * return number of HBA port entries available after reset.
6135  */
6136 static int
6137 _scsih_get_port_table_after_reset(struct MPT3SAS_ADAPTER *ioc,
6138         struct hba_port *port_table)
6139 {
6140         u16 sz, ioc_status;
6141         int i, j;
6142         Mpi2ConfigReply_t mpi_reply;
6143         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6144         u16 attached_handle;
6145         u64 attached_sas_addr;
6146         u8 found = 0, port_count = 0, port_id;
6147
6148         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
6149             * sizeof(Mpi2SasIOUnit0PhyData_t));
6150         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
6151         if (!sas_iounit_pg0) {
6152                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6153                     __FILE__, __LINE__, __func__);
6154                 return port_count;
6155         }
6156
6157         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
6158             sas_iounit_pg0, sz)) != 0)
6159                 goto out;
6160         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6161         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6162                 goto out;
6163         for (i = 0; i < ioc->sas_hba.num_phys; i++) {
6164                 found = 0;
6165                 if ((sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4) <
6166                     MPI2_SAS_NEG_LINK_RATE_1_5)
6167                         continue;
6168                 attached_handle =
6169                     le16_to_cpu(sas_iounit_pg0->PhyData[i].AttachedDevHandle);
6170                 if (_scsih_get_sas_address(
6171                     ioc, attached_handle, &attached_sas_addr) != 0) {
6172                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6173                             __FILE__, __LINE__, __func__);
6174                         continue;
6175                 }
6176
6177                 for (j = 0; j < port_count; j++) {
6178                         port_id = sas_iounit_pg0->PhyData[i].Port;
6179                         if (port_table[j].port_id == port_id &&
6180                             port_table[j].sas_address == attached_sas_addr) {
6181                                 port_table[j].phy_mask |= (1 << i);
6182                                 found = 1;
6183                                 break;
6184                         }
6185                 }
6186
6187                 if (found)
6188                         continue;
6189
6190                 port_id = sas_iounit_pg0->PhyData[i].Port;
6191                 port_table[port_count].port_id = port_id;
6192                 port_table[port_count].phy_mask = (1 << i);
6193                 port_table[port_count].sas_address = attached_sas_addr;
6194                 port_count++;
6195         }
6196 out:
6197         kfree(sas_iounit_pg0);
6198         return port_count;
6199 }
6200
6201 enum hba_port_matched_codes {
6202         NOT_MATCHED = 0,
6203         MATCHED_WITH_ADDR_AND_PHYMASK,
6204         MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT,
6205         MATCHED_WITH_ADDR_AND_SUBPHYMASK,
6206         MATCHED_WITH_ADDR,
6207 };
6208
6209 /**
6210  * _scsih_look_and_get_matched_port_entry - Get matched hba port entry
6211  *                                      from HBA port table
6212  * @ioc: per adapter object
6213  * @port_entry: hba port entry from temporary port table which needs to be
6214  *              searched for matched entry in the HBA port table
6215  * @matched_port_entry: save matched hba port entry here
6216  * @count: count of matched entries
6217  *
6218  * return type of matched entry found.
6219  */
6220 static enum hba_port_matched_codes
6221 _scsih_look_and_get_matched_port_entry(struct MPT3SAS_ADAPTER *ioc,
6222         struct hba_port *port_entry,
6223         struct hba_port **matched_port_entry, int *count)
6224 {
6225         struct hba_port *port_table_entry, *matched_port = NULL;
6226         enum hba_port_matched_codes matched_code = NOT_MATCHED;
6227         int lcount = 0;
6228         *matched_port_entry = NULL;
6229
6230         list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {
6231                 if (!(port_table_entry->flags & HBA_PORT_FLAG_DIRTY_PORT))
6232                         continue;
6233
6234                 if ((port_table_entry->sas_address == port_entry->sas_address)
6235                     && (port_table_entry->phy_mask == port_entry->phy_mask)) {
6236                         matched_code = MATCHED_WITH_ADDR_AND_PHYMASK;
6237                         matched_port = port_table_entry;
6238                         break;
6239                 }
6240
6241                 if ((port_table_entry->sas_address == port_entry->sas_address)
6242                     && (port_table_entry->phy_mask & port_entry->phy_mask)
6243                     && (port_table_entry->port_id == port_entry->port_id)) {
6244                         matched_code = MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT;
6245                         matched_port = port_table_entry;
6246                         continue;
6247                 }
6248
6249                 if ((port_table_entry->sas_address == port_entry->sas_address)
6250                     && (port_table_entry->phy_mask & port_entry->phy_mask)) {
6251                         if (matched_code ==
6252                             MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT)
6253                                 continue;
6254                         matched_code = MATCHED_WITH_ADDR_AND_SUBPHYMASK;
6255                         matched_port = port_table_entry;
6256                         continue;
6257                 }
6258
6259                 if (port_table_entry->sas_address == port_entry->sas_address) {
6260                         if (matched_code ==
6261                             MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT)
6262                                 continue;
6263                         if (matched_code == MATCHED_WITH_ADDR_AND_SUBPHYMASK)
6264                                 continue;
6265                         matched_code = MATCHED_WITH_ADDR;
6266                         matched_port = port_table_entry;
6267                         lcount++;
6268                 }
6269         }
6270
6271         *matched_port_entry = matched_port;
6272         if (matched_code ==  MATCHED_WITH_ADDR)
6273                 *count = lcount;
6274         return matched_code;
6275 }
6276
6277 /**
6278  * _scsih_del_phy_part_of_anther_port - remove phy if it
6279  *                              is a part of anther port
6280  *@ioc: per adapter object
6281  *@port_table: port table after reset
6282  *@index: hba port entry index
6283  *@port_count: number of ports available after host reset
6284  *@offset: HBA phy bit offset
6285  *
6286  */
6287 static void
6288 _scsih_del_phy_part_of_anther_port(struct MPT3SAS_ADAPTER *ioc,
6289         struct hba_port *port_table,
6290         int index, u8 port_count, int offset)
6291 {
6292         struct _sas_node *sas_node = &ioc->sas_hba;
6293         u32 i, found = 0;
6294
6295         for (i = 0; i < port_count; i++) {
6296                 if (i == index)
6297                         continue;
6298
6299                 if (port_table[i].phy_mask & (1 << offset)) {
6300                         mpt3sas_transport_del_phy_from_an_existing_port(
6301                             ioc, sas_node, &sas_node->phy[offset]);
6302                         found = 1;
6303                         break;
6304                 }
6305         }
6306         if (!found)
6307                 port_table[index].phy_mask |= (1 << offset);
6308 }
6309
6310 /**
6311  * _scsih_add_or_del_phys_from_existing_port - add/remove phy to/from
6312  *                                              right port
6313  *@ioc: per adapter object
6314  *@hba_port_entry: hba port table entry
6315  *@port_table: temporary port table
6316  *@index: hba port entry index
6317  *@port_count: number of ports available after host reset
6318  *
6319  */
6320 static void
6321 _scsih_add_or_del_phys_from_existing_port(struct MPT3SAS_ADAPTER *ioc,
6322         struct hba_port *hba_port_entry, struct hba_port *port_table,
6323         int index, int port_count)
6324 {
6325         u32 phy_mask, offset = 0;
6326         struct _sas_node *sas_node = &ioc->sas_hba;
6327
6328         phy_mask = hba_port_entry->phy_mask ^ port_table[index].phy_mask;
6329
6330         for (offset = 0; offset < ioc->sas_hba.num_phys; offset++) {
6331                 if (phy_mask & (1 << offset)) {
6332                         if (!(port_table[index].phy_mask & (1 << offset))) {
6333                                 _scsih_del_phy_part_of_anther_port(
6334                                     ioc, port_table, index, port_count,
6335                                     offset);
6336                                 continue;
6337                         }
6338                         if (sas_node->phy[offset].phy_belongs_to_port)
6339                                 mpt3sas_transport_del_phy_from_an_existing_port(
6340                                     ioc, sas_node, &sas_node->phy[offset]);
6341                         mpt3sas_transport_add_phy_to_an_existing_port(
6342                             ioc, sas_node, &sas_node->phy[offset],
6343                             hba_port_entry->sas_address,
6344                             hba_port_entry);
6345                 }
6346         }
6347 }
6348
6349 /**
6350  * _scsih_del_dirty_vphy - delete virtual_phy objects marked as dirty.
6351  * @ioc: per adapter object
6352  *
6353  * Returns nothing.
6354  */
6355 static void
6356 _scsih_del_dirty_vphy(struct MPT3SAS_ADAPTER *ioc)
6357 {
6358         struct hba_port *port, *port_next;
6359         struct virtual_phy *vphy, *vphy_next;
6360
6361         list_for_each_entry_safe(port, port_next,
6362             &ioc->port_table_list, list) {
6363                 if (!port->vphys_mask)
6364                         continue;
6365                 list_for_each_entry_safe(vphy, vphy_next,
6366                     &port->vphys_list, list) {
6367                         if (vphy->flags & MPT_VPHY_FLAG_DIRTY_PHY) {
6368                                 drsprintk(ioc, ioc_info(ioc,
6369                                     "Deleting vphy %p entry from port id: %d\t, Phy_mask 0x%08x\n",
6370                                     vphy, port->port_id,
6371                                     vphy->phy_mask));
6372                                 port->vphys_mask &= ~vphy->phy_mask;
6373                                 list_del(&vphy->list);
6374                                 kfree(vphy);
6375                         }
6376                 }
6377                 if (!port->vphys_mask && !port->sas_address)
6378                         port->flags |= HBA_PORT_FLAG_DIRTY_PORT;
6379         }
6380 }
6381
6382 /**
6383  * _scsih_del_dirty_port_entries - delete dirty port entries from port list
6384  *                                      after host reset
6385  *@ioc: per adapter object
6386  *
6387  */
6388 static void
6389 _scsih_del_dirty_port_entries(struct MPT3SAS_ADAPTER *ioc)
6390 {
6391         struct hba_port *port, *port_next;
6392
6393         list_for_each_entry_safe(port, port_next,
6394             &ioc->port_table_list, list) {
6395                 if (!(port->flags & HBA_PORT_FLAG_DIRTY_PORT) ||
6396                     port->flags & HBA_PORT_FLAG_NEW_PORT)
6397                         continue;
6398
6399                 drsprintk(ioc, ioc_info(ioc,
6400                     "Deleting port table entry %p having Port: %d\t Phy_mask 0x%08x\n",
6401                     port, port->port_id, port->phy_mask));
6402                 list_del(&port->list);
6403                 kfree(port);
6404         }
6405 }
6406
6407 /**
6408  * _scsih_sas_port_refresh - Update HBA port table after host reset
6409  * @ioc: per adapter object
6410  */
6411 static void
6412 _scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)
6413 {
6414         u32 port_count = 0;
6415         struct hba_port *port_table;
6416         struct hba_port *port_table_entry;
6417         struct hba_port *port_entry = NULL;
6418         int i, j, count = 0, lcount = 0;
6419         int ret;
6420         u64 sas_addr;
6421         u8 num_phys;
6422
6423         drsprintk(ioc, ioc_info(ioc,
6424             "updating ports for sas_host(0x%016llx)\n",
6425             (unsigned long long)ioc->sas_hba.sas_address));
6426
6427         mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
6428         if (!num_phys) {
6429                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6430                     __FILE__, __LINE__, __func__);
6431                 return;
6432         }
6433
6434         if (num_phys > ioc->sas_hba.nr_phys_allocated) {
6435                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6436                    __FILE__, __LINE__, __func__);
6437                 return;
6438         }
6439         ioc->sas_hba.num_phys = num_phys;
6440
6441         port_table = kcalloc(ioc->sas_hba.num_phys,
6442             sizeof(struct hba_port), GFP_KERNEL);
6443         if (!port_table)
6444                 return;
6445
6446         port_count = _scsih_get_port_table_after_reset(ioc, port_table);
6447         if (!port_count)
6448                 return;
6449
6450         drsprintk(ioc, ioc_info(ioc, "New Port table\n"));
6451         for (j = 0; j < port_count; j++)
6452                 drsprintk(ioc, ioc_info(ioc,
6453                     "Port: %d\t Phy_mask 0x%08x\t sas_addr(0x%016llx)\n",
6454                     port_table[j].port_id,
6455                     port_table[j].phy_mask, port_table[j].sas_address));
6456
6457         list_for_each_entry(port_table_entry, &ioc->port_table_list, list)
6458                 port_table_entry->flags |= HBA_PORT_FLAG_DIRTY_PORT;
6459
6460         drsprintk(ioc, ioc_info(ioc, "Old Port table\n"));
6461         port_table_entry = NULL;
6462         list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {
6463                 drsprintk(ioc, ioc_info(ioc,
6464                     "Port: %d\t Phy_mask 0x%08x\t sas_addr(0x%016llx)\n",
6465                     port_table_entry->port_id,
6466                     port_table_entry->phy_mask,
6467                     port_table_entry->sas_address));
6468         }
6469
6470         for (j = 0; j < port_count; j++) {
6471                 ret = _scsih_look_and_get_matched_port_entry(ioc,
6472                     &port_table[j], &port_entry, &count);
6473                 if (!port_entry) {
6474                         drsprintk(ioc, ioc_info(ioc,
6475                             "No Matched entry for sas_addr(0x%16llx), Port:%d\n",
6476                             port_table[j].sas_address,
6477                             port_table[j].port_id));
6478                         continue;
6479                 }
6480
6481                 switch (ret) {
6482                 case MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT:
6483                 case MATCHED_WITH_ADDR_AND_SUBPHYMASK:
6484                         _scsih_add_or_del_phys_from_existing_port(ioc,
6485                             port_entry, port_table, j, port_count);
6486                         break;
6487                 case MATCHED_WITH_ADDR:
6488                         sas_addr = port_table[j].sas_address;
6489                         for (i = 0; i < port_count; i++) {
6490                                 if (port_table[i].sas_address == sas_addr)
6491                                         lcount++;
6492                         }
6493
6494                         if (count > 1 || lcount > 1)
6495                                 port_entry = NULL;
6496                         else
6497                                 _scsih_add_or_del_phys_from_existing_port(ioc,
6498                                     port_entry, port_table, j, port_count);
6499                 }
6500
6501                 if (!port_entry)
6502                         continue;
6503
6504                 if (port_entry->port_id != port_table[j].port_id)
6505                         port_entry->port_id = port_table[j].port_id;
6506                 port_entry->flags &= ~HBA_PORT_FLAG_DIRTY_PORT;
6507                 port_entry->phy_mask = port_table[j].phy_mask;
6508         }
6509
6510         port_table_entry = NULL;
6511 }
6512
6513 /**
6514  * _scsih_alloc_vphy - allocate virtual_phy object
6515  * @ioc: per adapter object
6516  * @port_id: Port ID number
6517  * @phy_num: HBA Phy number
6518  *
6519  * Returns allocated virtual_phy object.
6520  */
6521 static struct virtual_phy *
6522 _scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num)
6523 {
6524         struct virtual_phy *vphy;
6525         struct hba_port *port;
6526
6527         port = mpt3sas_get_port_by_id(ioc, port_id, 0);
6528         if (!port)
6529                 return NULL;
6530
6531         vphy = mpt3sas_get_vphy_by_phy(ioc, port, phy_num);
6532         if (!vphy) {
6533                 vphy = kzalloc(sizeof(struct virtual_phy), GFP_KERNEL);
6534                 if (!vphy)
6535                         return NULL;
6536
6537                 if (!port->vphys_mask)
6538                         INIT_LIST_HEAD(&port->vphys_list);
6539
6540                 /*
6541                  * Enable bit corresponding to HBA phy number on its
6542                  * parent hba_port object's vphys_mask field.
6543                  */
6544                 port->vphys_mask |= (1 << phy_num);
6545                 vphy->phy_mask |= (1 << phy_num);
6546
6547                 list_add_tail(&vphy->list, &port->vphys_list);
6548
6549                 ioc_info(ioc,
6550                     "vphy entry: %p, port id: %d, phy:%d is added to port's vphys_list\n",
6551                     vphy, port->port_id, phy_num);
6552         }
6553         return vphy;
6554 }
6555
6556 /**
6557  * _scsih_sas_host_refresh - refreshing sas host object contents
6558  * @ioc: per adapter object
6559  * Context: user
6560  *
6561  * During port enable, fw will send topology events for every device. Its
6562  * possible that the handles may change from the previous setting, so this
6563  * code keeping handles updating if changed.
6564  */
6565 static void
6566 _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
6567 {
6568         u16 sz;
6569         u16 ioc_status;
6570         int i;
6571         Mpi2ConfigReply_t mpi_reply;
6572         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6573         u16 attached_handle;
6574         u8 link_rate, port_id;
6575         struct hba_port *port;
6576         Mpi2SasPhyPage0_t phy_pg0;
6577
6578         dtmprintk(ioc,
6579                   ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",
6580                            (u64)ioc->sas_hba.sas_address));
6581
6582         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
6583             * sizeof(Mpi2SasIOUnit0PhyData_t));
6584         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
6585         if (!sas_iounit_pg0) {
6586                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6587                         __FILE__, __LINE__, __func__);
6588                 return;
6589         }
6590
6591         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
6592             sas_iounit_pg0, sz)) != 0)
6593                 goto out;
6594         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6595         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6596                 goto out;
6597         for (i = 0; i < ioc->sas_hba.num_phys ; i++) {
6598                 link_rate = sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4;
6599                 if (i == 0)
6600                         ioc->sas_hba.handle = le16_to_cpu(
6601                             sas_iounit_pg0->PhyData[0].ControllerDevHandle);
6602                 port_id = sas_iounit_pg0->PhyData[i].Port;
6603                 if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {
6604                         port = kzalloc(sizeof(struct hba_port), GFP_KERNEL);
6605                         if (!port)
6606                                 goto out;
6607
6608                         port->port_id = port_id;
6609                         ioc_info(ioc,
6610                             "hba_port entry: %p, port: %d is added to hba_port list\n",
6611                             port, port->port_id);
6612                         if (ioc->shost_recovery)
6613                                 port->flags = HBA_PORT_FLAG_NEW_PORT;
6614                         list_add_tail(&port->list, &ioc->port_table_list);
6615                 }
6616                 /*
6617                  * Check whether current Phy belongs to HBA vSES device or not.
6618                  */
6619                 if (le32_to_cpu(sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
6620                     MPI2_SAS_DEVICE_INFO_SEP &&
6621                     (link_rate >=  MPI2_SAS_NEG_LINK_RATE_1_5)) {
6622                         if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,
6623                             &phy_pg0, i))) {
6624                                 ioc_err(ioc,
6625                                     "failure at %s:%d/%s()!\n",
6626                                      __FILE__, __LINE__, __func__);
6627                                 goto out;
6628                         }
6629                         if (!(le32_to_cpu(phy_pg0.PhyInfo) &
6630                             MPI2_SAS_PHYINFO_VIRTUAL_PHY))
6631                                 continue;
6632                         /*
6633                          * Allocate a virtual_phy object for vSES device, if
6634                          * this vSES device is hot added.
6635                          */
6636                         if (!_scsih_alloc_vphy(ioc, port_id, i))
6637                                 goto out;
6638                         ioc->sas_hba.phy[i].hba_vphy = 1;
6639                 }
6640
6641                 /*
6642                  * Add new HBA phys to STL if these new phys got added as part
6643                  * of HBA Firmware upgrade/downgrade operation.
6644                  */
6645                 if (!ioc->sas_hba.phy[i].phy) {
6646                         if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,
6647                                                         &phy_pg0, i))) {
6648                                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6649                                         __FILE__, __LINE__, __func__);
6650                                 continue;
6651                         }
6652                         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6653                                 MPI2_IOCSTATUS_MASK;
6654                         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6655                                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6656                                         __FILE__, __LINE__, __func__);
6657                                 continue;
6658                         }
6659                         ioc->sas_hba.phy[i].phy_id = i;
6660                         mpt3sas_transport_add_host_phy(ioc,
6661                                 &ioc->sas_hba.phy[i], phy_pg0,
6662                                 ioc->sas_hba.parent_dev);
6663                         continue;
6664                 }
6665                 ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
6666                 attached_handle = le16_to_cpu(sas_iounit_pg0->PhyData[i].
6667                     AttachedDevHandle);
6668                 if (attached_handle && link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
6669                         link_rate = MPI2_SAS_NEG_LINK_RATE_1_5;
6670                 ioc->sas_hba.phy[i].port =
6671                     mpt3sas_get_port_by_id(ioc, port_id, 0);
6672                 mpt3sas_transport_update_links(ioc, ioc->sas_hba.sas_address,
6673                     attached_handle, i, link_rate,
6674                     ioc->sas_hba.phy[i].port);
6675         }
6676         /*
6677          * Clear the phy details if this phy got disabled as part of
6678          * HBA Firmware upgrade/downgrade operation.
6679          */
6680         for (i = ioc->sas_hba.num_phys;
6681              i < ioc->sas_hba.nr_phys_allocated; i++) {
6682                 if (ioc->sas_hba.phy[i].phy &&
6683                     ioc->sas_hba.phy[i].phy->negotiated_linkrate >=
6684                     SAS_LINK_RATE_1_5_GBPS)
6685                         mpt3sas_transport_update_links(ioc,
6686                                 ioc->sas_hba.sas_address, 0, i,
6687                                 MPI2_SAS_NEG_LINK_RATE_PHY_DISABLED, NULL);
6688         }
6689  out:
6690         kfree(sas_iounit_pg0);
6691 }
6692
6693 /**
6694  * _scsih_sas_host_add - create sas host object
6695  * @ioc: per adapter object
6696  *
6697  * Creating host side data object, stored in ioc->sas_hba
6698  */
6699 static void
6700 _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
6701 {
6702         int i;
6703         Mpi2ConfigReply_t mpi_reply;
6704         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6705         Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
6706         Mpi2SasPhyPage0_t phy_pg0;
6707         Mpi2SasDevicePage0_t sas_device_pg0;
6708         Mpi2SasEnclosurePage0_t enclosure_pg0;
6709         u16 ioc_status;
6710         u16 sz;
6711         u8 device_missing_delay;
6712         u8 num_phys, port_id;
6713         struct hba_port *port;
6714
6715         mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
6716         if (!num_phys) {
6717                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6718                         __FILE__, __LINE__, __func__);
6719                 return;
6720         }
6721
6722         ioc->sas_hba.nr_phys_allocated = max_t(u8,
6723             MPT_MAX_HBA_NUM_PHYS, num_phys);
6724         ioc->sas_hba.phy = kcalloc(ioc->sas_hba.nr_phys_allocated,
6725             sizeof(struct _sas_phy), GFP_KERNEL);
6726         if (!ioc->sas_hba.phy) {
6727                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6728                         __FILE__, __LINE__, __func__);
6729                 goto out;
6730         }
6731         ioc->sas_hba.num_phys = num_phys;
6732
6733         /* sas_iounit page 0 */
6734         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *
6735             sizeof(Mpi2SasIOUnit0PhyData_t));
6736         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
6737         if (!sas_iounit_pg0) {
6738                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6739                         __FILE__, __LINE__, __func__);
6740                 return;
6741         }
6742         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
6743             sas_iounit_pg0, sz))) {
6744                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6745                         __FILE__, __LINE__, __func__);
6746                 goto out;
6747         }
6748         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6749             MPI2_IOCSTATUS_MASK;
6750         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6751                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6752                         __FILE__, __LINE__, __func__);
6753                 goto out;
6754         }
6755
6756         /* sas_iounit page 1 */
6757         sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *
6758             sizeof(Mpi2SasIOUnit1PhyData_t));
6759         sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
6760         if (!sas_iounit_pg1) {
6761                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6762                         __FILE__, __LINE__, __func__);
6763                 goto out;
6764         }
6765         if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
6766             sas_iounit_pg1, sz))) {
6767                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6768                         __FILE__, __LINE__, __func__);
6769                 goto out;
6770         }
6771         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6772             MPI2_IOCSTATUS_MASK;
6773         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6774                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6775                         __FILE__, __LINE__, __func__);
6776                 goto out;
6777         }
6778
6779         ioc->io_missing_delay =
6780             sas_iounit_pg1->IODeviceMissingDelay;
6781         device_missing_delay =
6782             sas_iounit_pg1->ReportDeviceMissingDelay;
6783         if (device_missing_delay & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
6784                 ioc->device_missing_delay = (device_missing_delay &
6785                     MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
6786         else
6787                 ioc->device_missing_delay = device_missing_delay &
6788                     MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
6789
6790         ioc->sas_hba.parent_dev = &ioc->shost->shost_gendev;
6791         for (i = 0; i < ioc->sas_hba.num_phys ; i++) {
6792                 if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,
6793                     i))) {
6794                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6795                                 __FILE__, __LINE__, __func__);
6796                         goto out;
6797                 }
6798                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6799                     MPI2_IOCSTATUS_MASK;
6800                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6801                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6802                                 __FILE__, __LINE__, __func__);
6803                         goto out;
6804                 }
6805
6806                 if (i == 0)
6807                         ioc->sas_hba.handle = le16_to_cpu(sas_iounit_pg0->
6808                             PhyData[0].ControllerDevHandle);
6809
6810                 port_id = sas_iounit_pg0->PhyData[i].Port;
6811                 if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {
6812                         port = kzalloc(sizeof(struct hba_port), GFP_KERNEL);
6813                         if (!port)
6814                                 goto out;
6815
6816                         port->port_id = port_id;
6817                         ioc_info(ioc,
6818                            "hba_port entry: %p, port: %d is added to hba_port list\n",
6819                            port, port->port_id);
6820                         list_add_tail(&port->list,
6821                             &ioc->port_table_list);
6822                 }
6823
6824                 /*
6825                  * Check whether current Phy belongs to HBA vSES device or not.
6826                  */
6827                 if ((le32_to_cpu(phy_pg0.PhyInfo) &
6828                     MPI2_SAS_PHYINFO_VIRTUAL_PHY) &&
6829                     (phy_pg0.NegotiatedLinkRate >> 4) >=
6830                     MPI2_SAS_NEG_LINK_RATE_1_5) {
6831                         /*
6832                          * Allocate a virtual_phy object for vSES device.
6833                          */
6834                         if (!_scsih_alloc_vphy(ioc, port_id, i))
6835                                 goto out;
6836                         ioc->sas_hba.phy[i].hba_vphy = 1;
6837                 }
6838
6839                 ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
6840                 ioc->sas_hba.phy[i].phy_id = i;
6841                 ioc->sas_hba.phy[i].port =
6842                     mpt3sas_get_port_by_id(ioc, port_id, 0);
6843                 mpt3sas_transport_add_host_phy(ioc, &ioc->sas_hba.phy[i],
6844                     phy_pg0, ioc->sas_hba.parent_dev);
6845         }
6846         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
6847             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, ioc->sas_hba.handle))) {
6848                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6849                         __FILE__, __LINE__, __func__);
6850                 goto out;
6851         }
6852         ioc->sas_hba.enclosure_handle =
6853             le16_to_cpu(sas_device_pg0.EnclosureHandle);
6854         ioc->sas_hba.sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
6855         ioc_info(ioc, "host_add: handle(0x%04x), sas_addr(0x%016llx), phys(%d)\n",
6856                  ioc->sas_hba.handle,
6857                  (u64)ioc->sas_hba.sas_address,
6858                  ioc->sas_hba.num_phys);
6859
6860         if (ioc->sas_hba.enclosure_handle) {
6861                 if (!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
6862                     &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
6863                    ioc->sas_hba.enclosure_handle)))
6864                         ioc->sas_hba.enclosure_logical_id =
6865                             le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
6866         }
6867
6868  out:
6869         kfree(sas_iounit_pg1);
6870         kfree(sas_iounit_pg0);
6871 }
6872
6873 /**
6874  * _scsih_expander_add -  creating expander object
6875  * @ioc: per adapter object
6876  * @handle: expander handle
6877  *
6878  * Creating expander object, stored in ioc->sas_expander_list.
6879  *
6880  * Return: 0 for success, else error.
6881  */
6882 static int
6883 _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
6884 {
6885         struct _sas_node *sas_expander;
6886         struct _enclosure_node *enclosure_dev;
6887         Mpi2ConfigReply_t mpi_reply;
6888         Mpi2ExpanderPage0_t expander_pg0;
6889         Mpi2ExpanderPage1_t expander_pg1;
6890         u32 ioc_status;
6891         u16 parent_handle;
6892         u64 sas_address, sas_address_parent = 0;
6893         int i;
6894         unsigned long flags;
6895         struct _sas_port *mpt3sas_port = NULL;
6896         u8 port_id;
6897
6898         int rc = 0;
6899
6900         if (!handle)
6901                 return -1;
6902
6903         if (ioc->shost_recovery || ioc->pci_error_recovery)
6904                 return -1;
6905
6906         if ((mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
6907             MPI2_SAS_EXPAND_PGAD_FORM_HNDL, handle))) {
6908                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6909                         __FILE__, __LINE__, __func__);
6910                 return -1;
6911         }
6912
6913         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6914             MPI2_IOCSTATUS_MASK;
6915         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6916                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6917                         __FILE__, __LINE__, __func__);
6918                 return -1;
6919         }
6920
6921         /* handle out of order topology events */
6922         parent_handle = le16_to_cpu(expander_pg0.ParentDevHandle);
6923         if (_scsih_get_sas_address(ioc, parent_handle, &sas_address_parent)
6924             != 0) {
6925                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6926                         __FILE__, __LINE__, __func__);
6927                 return -1;
6928         }
6929
6930         port_id = expander_pg0.PhysicalPort;
6931         if (sas_address_parent != ioc->sas_hba.sas_address) {
6932                 spin_lock_irqsave(&ioc->sas_node_lock, flags);
6933                 sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6934                     sas_address_parent,
6935                     mpt3sas_get_port_by_id(ioc, port_id, 0));
6936                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
6937                 if (!sas_expander) {
6938                         rc = _scsih_expander_add(ioc, parent_handle);
6939                         if (rc != 0)
6940                                 return rc;
6941                 }
6942         }
6943
6944         spin_lock_irqsave(&ioc->sas_node_lock, flags);
6945         sas_address = le64_to_cpu(expander_pg0.SASAddress);
6946         sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6947             sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));
6948         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
6949
6950         if (sas_expander)
6951                 return 0;
6952
6953         sas_expander = kzalloc(sizeof(struct _sas_node),
6954             GFP_KERNEL);
6955         if (!sas_expander) {
6956                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6957                         __FILE__, __LINE__, __func__);
6958                 return -1;
6959         }
6960
6961         sas_expander->handle = handle;
6962         sas_expander->num_phys = expander_pg0.NumPhys;
6963         sas_expander->sas_address_parent = sas_address_parent;
6964         sas_expander->sas_address = sas_address;
6965         sas_expander->port = mpt3sas_get_port_by_id(ioc, port_id, 0);
6966         if (!sas_expander->port) {
6967                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6968                     __FILE__, __LINE__, __func__);
6969                 rc = -1;
6970                 goto out_fail;
6971         }
6972
6973         ioc_info(ioc, "expander_add: handle(0x%04x), parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n",
6974                  handle, parent_handle,
6975                  (u64)sas_expander->sas_address, sas_expander->num_phys);
6976
6977         if (!sas_expander->num_phys) {
6978                 rc = -1;
6979                 goto out_fail;
6980         }
6981         sas_expander->phy = kcalloc(sas_expander->num_phys,
6982             sizeof(struct _sas_phy), GFP_KERNEL);
6983         if (!sas_expander->phy) {
6984                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6985                         __FILE__, __LINE__, __func__);
6986                 rc = -1;
6987                 goto out_fail;
6988         }
6989
6990         INIT_LIST_HEAD(&sas_expander->sas_port_list);
6991         mpt3sas_port = mpt3sas_transport_port_add(ioc, handle,
6992             sas_address_parent, sas_expander->port);
6993         if (!mpt3sas_port) {
6994                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6995                         __FILE__, __LINE__, __func__);
6996                 rc = -1;
6997                 goto out_fail;
6998         }
6999         sas_expander->parent_dev = &mpt3sas_port->rphy->dev;
7000         sas_expander->rphy = mpt3sas_port->rphy;
7001
7002         for (i = 0 ; i < sas_expander->num_phys ; i++) {
7003                 if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,
7004                     &expander_pg1, i, handle))) {
7005                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
7006                                 __FILE__, __LINE__, __func__);
7007                         rc = -1;
7008                         goto out_fail;
7009                 }
7010                 sas_expander->phy[i].handle = handle;
7011                 sas_expander->phy[i].phy_id = i;
7012                 sas_expander->phy[i].port =
7013                     mpt3sas_get_port_by_id(ioc, port_id, 0);
7014
7015                 if ((mpt3sas_transport_add_expander_phy(ioc,
7016                     &sas_expander->phy[i], expander_pg1,
7017                     sas_expander->parent_dev))) {
7018                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
7019                                 __FILE__, __LINE__, __func__);
7020                         rc = -1;
7021                         goto out_fail;
7022                 }
7023         }
7024
7025         if (sas_expander->enclosure_handle) {
7026                 enclosure_dev =
7027                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
7028                                                 sas_expander->enclosure_handle);
7029                 if (enclosure_dev)
7030                         sas_expander->enclosure_logical_id =
7031                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7032         }
7033
7034         _scsih_expander_node_add(ioc, sas_expander);
7035         return 0;
7036
7037  out_fail:
7038
7039         if (mpt3sas_port)
7040                 mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,
7041                     sas_address_parent, sas_expander->port);
7042         kfree(sas_expander);
7043         return rc;
7044 }
7045
7046 /**
7047  * mpt3sas_expander_remove - removing expander object
7048  * @ioc: per adapter object
7049  * @sas_address: expander sas_address
7050  * @port: hba port entry
7051  */
7052 void
7053 mpt3sas_expander_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
7054         struct hba_port *port)
7055 {
7056         struct _sas_node *sas_expander;
7057         unsigned long flags;
7058
7059         if (ioc->shost_recovery)
7060                 return;
7061
7062         if (!port)
7063                 return;
7064
7065         spin_lock_irqsave(&ioc->sas_node_lock, flags);
7066         sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
7067             sas_address, port);
7068         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
7069         if (sas_expander)
7070                 _scsih_expander_node_remove(ioc, sas_expander);
7071 }
7072
7073 /**
7074  * _scsih_done -  internal SCSI_IO callback handler.
7075  * @ioc: per adapter object
7076  * @smid: system request message index
7077  * @msix_index: MSIX table index supplied by the OS
7078  * @reply: reply message frame(lower 32bit addr)
7079  *
7080  * Callback handler when sending internal generated SCSI_IO.
7081  * The callback index passed is `ioc->scsih_cb_idx`
7082  *
7083  * Return: 1 meaning mf should be freed from _base_interrupt
7084  *         0 means the mf is freed from this function.
7085  */
7086 static u8
7087 _scsih_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
7088 {
7089         MPI2DefaultReply_t *mpi_reply;
7090
7091         mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);
7092         if (ioc->scsih_cmds.status == MPT3_CMD_NOT_USED)
7093                 return 1;
7094         if (ioc->scsih_cmds.smid != smid)
7095                 return 1;
7096         ioc->scsih_cmds.status |= MPT3_CMD_COMPLETE;
7097         if (mpi_reply) {
7098                 memcpy(ioc->scsih_cmds.reply, mpi_reply,
7099                     mpi_reply->MsgLength*4);
7100                 ioc->scsih_cmds.status |= MPT3_CMD_REPLY_VALID;
7101         }
7102         ioc->scsih_cmds.status &= ~MPT3_CMD_PENDING;
7103         complete(&ioc->scsih_cmds.done);
7104         return 1;
7105 }
7106
7107
7108
7109
7110 #define MPT3_MAX_LUNS (255)
7111
7112
7113 /**
7114  * _scsih_check_access_status - check access flags
7115  * @ioc: per adapter object
7116  * @sas_address: sas address
7117  * @handle: sas device handle
7118  * @access_status: errors returned during discovery of the device
7119  *
7120  * Return: 0 for success, else failure
7121  */
7122 static u8
7123 _scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
7124         u16 handle, u8 access_status)
7125 {
7126         u8 rc = 1;
7127         char *desc = NULL;
7128
7129         switch (access_status) {
7130         case MPI2_SAS_DEVICE0_ASTATUS_NO_ERRORS:
7131         case MPI2_SAS_DEVICE0_ASTATUS_SATA_NEEDS_INITIALIZATION:
7132                 rc = 0;
7133                 break;
7134         case MPI2_SAS_DEVICE0_ASTATUS_SATA_CAPABILITY_FAILED:
7135                 desc = "sata capability failed";
7136                 break;
7137         case MPI2_SAS_DEVICE0_ASTATUS_SATA_AFFILIATION_CONFLICT:
7138                 desc = "sata affiliation conflict";
7139                 break;
7140         case MPI2_SAS_DEVICE0_ASTATUS_ROUTE_NOT_ADDRESSABLE:
7141                 desc = "route not addressable";
7142                 break;
7143         case MPI2_SAS_DEVICE0_ASTATUS_SMP_ERROR_NOT_ADDRESSABLE:
7144                 desc = "smp error not addressable";
7145                 break;
7146         case MPI2_SAS_DEVICE0_ASTATUS_DEVICE_BLOCKED:
7147                 desc = "device blocked";
7148                 break;
7149         case MPI2_SAS_DEVICE0_ASTATUS_SATA_INIT_FAILED:
7150         case MPI2_SAS_DEVICE0_ASTATUS_SIF_UNKNOWN:
7151         case MPI2_SAS_DEVICE0_ASTATUS_SIF_AFFILIATION_CONFLICT:
7152         case MPI2_SAS_DEVICE0_ASTATUS_SIF_DIAG:
7153         case MPI2_SAS_DEVICE0_ASTATUS_SIF_IDENTIFICATION:
7154         case MPI2_SAS_DEVICE0_ASTATUS_SIF_CHECK_POWER:
7155         case MPI2_SAS_DEVICE0_ASTATUS_SIF_PIO_SN:
7156         case MPI2_SAS_DEVICE0_ASTATUS_SIF_MDMA_SN:
7157         case MPI2_SAS_DEVICE0_ASTATUS_SIF_UDMA_SN:
7158         case MPI2_SAS_DEVICE0_ASTATUS_SIF_ZONING_VIOLATION:
7159         case MPI2_SAS_DEVICE0_ASTATUS_SIF_NOT_ADDRESSABLE:
7160         case MPI2_SAS_DEVICE0_ASTATUS_SIF_MAX:
7161                 desc = "sata initialization failed";
7162                 break;
7163         default:
7164                 desc = "unknown";
7165                 break;
7166         }
7167
7168         if (!rc)
7169                 return 0;
7170
7171         ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",
7172                 desc, (u64)sas_address, handle);
7173         return rc;
7174 }
7175
7176 /**
7177  * _scsih_check_device - checking device responsiveness
7178  * @ioc: per adapter object
7179  * @parent_sas_address: sas address of parent expander or sas host
7180  * @handle: attached device handle
7181  * @phy_number: phy number
7182  * @link_rate: new link rate
7183  */
7184 static void
7185 _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
7186         u64 parent_sas_address, u16 handle, u8 phy_number, u8 link_rate)
7187 {
7188         Mpi2ConfigReply_t mpi_reply;
7189         Mpi2SasDevicePage0_t sas_device_pg0;
7190         struct _sas_device *sas_device = NULL;
7191         struct _enclosure_node *enclosure_dev = NULL;
7192         u32 ioc_status;
7193         unsigned long flags;
7194         u64 sas_address;
7195         struct scsi_target *starget;
7196         struct MPT3SAS_TARGET *sas_target_priv_data;
7197         u32 device_info;
7198         struct hba_port *port;
7199
7200         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
7201             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle)))
7202                 return;
7203
7204         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
7205         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
7206                 return;
7207
7208         /* wide port handling ~ we need only handle device once for the phy that
7209          * is matched in sas device page zero
7210          */
7211         if (phy_number != sas_device_pg0.PhyNum)
7212                 return;
7213
7214         /* check if this is end device */
7215         device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
7216         if (!(_scsih_is_end_device(device_info)))
7217                 return;
7218
7219         spin_lock_irqsave(&ioc->sas_device_lock, flags);
7220         sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
7221         port = mpt3sas_get_port_by_id(ioc, sas_device_pg0.PhysicalPort, 0);
7222         if (!port)
7223                 goto out_unlock;
7224         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
7225             sas_address, port);
7226
7227         if (!sas_device)
7228                 goto out_unlock;
7229
7230         if (unlikely(sas_device->handle != handle)) {
7231                 starget = sas_device->starget;
7232                 sas_target_priv_data = starget->hostdata;
7233                 starget_printk(KERN_INFO, starget,
7234                         "handle changed from(0x%04x) to (0x%04x)!!!\n",
7235                         sas_device->handle, handle);
7236                 sas_target_priv_data->handle = handle;
7237                 sas_device->handle = handle;
7238                 if (le16_to_cpu(sas_device_pg0.Flags) &
7239                      MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
7240                         sas_device->enclosure_level =
7241                                 sas_device_pg0.EnclosureLevel;
7242                         memcpy(sas_device->connector_name,
7243                                 sas_device_pg0.ConnectorName, 4);
7244                         sas_device->connector_name[4] = '\0';
7245                 } else {
7246                         sas_device->enclosure_level = 0;
7247                         sas_device->connector_name[0] = '\0';
7248                 }
7249
7250                 sas_device->enclosure_handle =
7251                                 le16_to_cpu(sas_device_pg0.EnclosureHandle);
7252                 sas_device->is_chassis_slot_valid = 0;
7253                 enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,
7254                                                 sas_device->enclosure_handle);
7255                 if (enclosure_dev) {
7256                         sas_device->enclosure_logical_id =
7257                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7258                         if (le16_to_cpu(enclosure_dev->pg0.Flags) &
7259                             MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
7260                                 sas_device->is_chassis_slot_valid = 1;
7261                                 sas_device->chassis_slot =
7262                                         enclosure_dev->pg0.ChassisSlot;
7263                         }
7264                 }
7265         }
7266
7267         /* check if device is present */
7268         if (!(le16_to_cpu(sas_device_pg0.Flags) &
7269             MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
7270                 ioc_err(ioc, "device is not present handle(0x%04x), flags!!!\n",
7271                         handle);
7272                 goto out_unlock;
7273         }
7274
7275         /* check if there were any issues with discovery */
7276         if (_scsih_check_access_status(ioc, sas_address, handle,
7277             sas_device_pg0.AccessStatus))
7278                 goto out_unlock;
7279
7280         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
7281         _scsih_ublock_io_device(ioc, sas_address, port);
7282
7283         if (sas_device)
7284                 sas_device_put(sas_device);
7285         return;
7286
7287 out_unlock:
7288         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
7289         if (sas_device)
7290                 sas_device_put(sas_device);
7291 }
7292
7293 /**
7294  * _scsih_add_device -  creating sas device object
7295  * @ioc: per adapter object
7296  * @handle: sas device handle
7297  * @phy_num: phy number end device attached to
7298  * @is_pd: is this hidden raid component
7299  *
7300  * Creating end device object, stored in ioc->sas_device_list.
7301  *
7302  * Return: 0 for success, non-zero for failure.
7303  */
7304 static int
7305 _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
7306         u8 is_pd)
7307 {
7308         Mpi2ConfigReply_t mpi_reply;
7309         Mpi2SasDevicePage0_t sas_device_pg0;
7310         struct _sas_device *sas_device;
7311         struct _enclosure_node *enclosure_dev = NULL;
7312         u32 ioc_status;
7313         u64 sas_address;
7314         u32 device_info;
7315         u8 port_id;
7316
7317         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
7318             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
7319                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7320                         __FILE__, __LINE__, __func__);
7321                 return -1;
7322         }
7323
7324         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
7325             MPI2_IOCSTATUS_MASK;
7326         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
7327                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7328                         __FILE__, __LINE__, __func__);
7329                 return -1;
7330         }
7331
7332         /* check if this is end device */
7333         device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
7334         if (!(_scsih_is_end_device(device_info)))
7335                 return -1;
7336         set_bit(handle, ioc->pend_os_device_add);
7337         sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
7338
7339         /* check if device is present */
7340         if (!(le16_to_cpu(sas_device_pg0.Flags) &
7341             MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
7342                 ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",
7343                         handle);
7344                 return -1;
7345         }
7346
7347         /* check if there were any issues with discovery */
7348         if (_scsih_check_access_status(ioc, sas_address, handle,
7349             sas_device_pg0.AccessStatus))
7350                 return -1;
7351
7352         port_id = sas_device_pg0.PhysicalPort;
7353         sas_device = mpt3sas_get_sdev_by_addr(ioc,
7354             sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));
7355         if (sas_device) {
7356                 clear_bit(handle, ioc->pend_os_device_add);
7357                 sas_device_put(sas_device);
7358                 return -1;
7359         }
7360
7361         if (sas_device_pg0.EnclosureHandle) {
7362                 enclosure_dev =
7363                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
7364                             le16_to_cpu(sas_device_pg0.EnclosureHandle));
7365                 if (enclosure_dev == NULL)
7366                         ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",
7367                                  sas_device_pg0.EnclosureHandle);
7368         }
7369
7370         sas_device = kzalloc(sizeof(struct _sas_device),
7371             GFP_KERNEL);
7372         if (!sas_device) {
7373                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7374                         __FILE__, __LINE__, __func__);
7375                 return 0;
7376         }
7377
7378         kref_init(&sas_device->refcount);
7379         sas_device->handle = handle;
7380         if (_scsih_get_sas_address(ioc,
7381             le16_to_cpu(sas_device_pg0.ParentDevHandle),
7382             &sas_device->sas_address_parent) != 0)
7383                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7384                         __FILE__, __LINE__, __func__);
7385         sas_device->enclosure_handle =
7386             le16_to_cpu(sas_device_pg0.EnclosureHandle);
7387         if (sas_device->enclosure_handle != 0)
7388                 sas_device->slot =
7389                     le16_to_cpu(sas_device_pg0.Slot);
7390         sas_device->device_info = device_info;
7391         sas_device->sas_address = sas_address;
7392         sas_device->phy = sas_device_pg0.PhyNum;
7393         sas_device->fast_path = (le16_to_cpu(sas_device_pg0.Flags) &
7394             MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
7395         sas_device->port = mpt3sas_get_port_by_id(ioc, port_id, 0);
7396         if (!sas_device->port) {
7397                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7398                     __FILE__, __LINE__, __func__);
7399                 goto out;
7400         }
7401
7402         if (le16_to_cpu(sas_device_pg0.Flags)
7403                 & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
7404                 sas_device->enclosure_level =
7405                         sas_device_pg0.EnclosureLevel;
7406                 memcpy(sas_device->connector_name,
7407                         sas_device_pg0.ConnectorName, 4);
7408                 sas_device->connector_name[4] = '\0';
7409         } else {
7410                 sas_device->enclosure_level = 0;
7411                 sas_device->connector_name[0] = '\0';
7412         }
7413         /* get enclosure_logical_id & chassis_slot*/
7414         sas_device->is_chassis_slot_valid = 0;
7415         if (enclosure_dev) {
7416                 sas_device->enclosure_logical_id =
7417                     le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7418                 if (le16_to_cpu(enclosure_dev->pg0.Flags) &
7419                     MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
7420                         sas_device->is_chassis_slot_valid = 1;
7421                         sas_device->chassis_slot =
7422                                         enclosure_dev->pg0.ChassisSlot;
7423                 }
7424         }
7425
7426         /* get device name */
7427         sas_device->device_name = le64_to_cpu(sas_device_pg0.DeviceName);
7428         sas_device->port_type = sas_device_pg0.MaxPortConnections;
7429         ioc_info(ioc,
7430             "handle(0x%0x) sas_address(0x%016llx) port_type(0x%0x)\n",
7431             handle, sas_device->sas_address, sas_device->port_type);
7432
7433         if (ioc->wait_for_discovery_to_complete)
7434                 _scsih_sas_device_init_add(ioc, sas_device);
7435         else
7436                 _scsih_sas_device_add(ioc, sas_device);
7437
7438 out:
7439         sas_device_put(sas_device);
7440         return 0;
7441 }
7442
7443 /**
7444  * _scsih_remove_device -  removing sas device object
7445  * @ioc: per adapter object
7446  * @sas_device: the sas_device object
7447  */
7448 static void
7449 _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
7450         struct _sas_device *sas_device)
7451 {
7452         struct MPT3SAS_TARGET *sas_target_priv_data;
7453
7454         if ((ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM) &&
7455              (sas_device->pfa_led_on)) {
7456                 _scsih_turn_off_pfa_led(ioc, sas_device);
7457                 sas_device->pfa_led_on = 0;
7458         }
7459
7460         dewtprintk(ioc,
7461                    ioc_info(ioc, "%s: enter: handle(0x%04x), sas_addr(0x%016llx)\n",
7462                             __func__,
7463                             sas_device->handle, (u64)sas_device->sas_address));
7464
7465         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
7466             NULL, NULL));
7467
7468         if (sas_device->starget && sas_device->starget->hostdata) {
7469                 sas_target_priv_data = sas_device->starget->hostdata;
7470                 sas_target_priv_data->deleted = 1;
7471                 _scsih_ublock_io_device(ioc, sas_device->sas_address,
7472                     sas_device->port);
7473                 sas_target_priv_data->handle =
7474                      MPT3SAS_INVALID_DEVICE_HANDLE;
7475         }
7476
7477         if (!ioc->hide_drives)
7478                 mpt3sas_transport_port_remove(ioc,
7479                     sas_device->sas_address,
7480                     sas_device->sas_address_parent,
7481                     sas_device->port);
7482
7483         ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",
7484                  sas_device->handle, (u64)sas_device->sas_address);
7485
7486         _scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);
7487
7488         dewtprintk(ioc,
7489                    ioc_info(ioc, "%s: exit: handle(0x%04x), sas_addr(0x%016llx)\n",
7490                             __func__,
7491                             sas_device->handle, (u64)sas_device->sas_address));
7492         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
7493             NULL, NULL));
7494 }
7495
7496 /**
7497  * _scsih_sas_topology_change_event_debug - debug for topology event
7498  * @ioc: per adapter object
7499  * @event_data: event data payload
7500  * Context: user.
7501  */
7502 static void
7503 _scsih_sas_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
7504         Mpi2EventDataSasTopologyChangeList_t *event_data)
7505 {
7506         int i;
7507         u16 handle;
7508         u16 reason_code;
7509         u8 phy_number;
7510         char *status_str = NULL;
7511         u8 link_rate, prev_link_rate;
7512
7513         switch (event_data->ExpStatus) {
7514         case MPI2_EVENT_SAS_TOPO_ES_ADDED:
7515                 status_str = "add";
7516                 break;
7517         case MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING:
7518                 status_str = "remove";
7519                 break;
7520         case MPI2_EVENT_SAS_TOPO_ES_RESPONDING:
7521         case 0:
7522                 status_str =  "responding";
7523                 break;
7524         case MPI2_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING:
7525                 status_str = "remove delay";
7526                 break;
7527         default:
7528                 status_str = "unknown status";
7529                 break;
7530         }
7531         ioc_info(ioc, "sas topology change: (%s)\n", status_str);
7532         pr_info("\thandle(0x%04x), enclosure_handle(0x%04x) " \
7533             "start_phy(%02d), count(%d)\n",
7534             le16_to_cpu(event_data->ExpanderDevHandle),
7535             le16_to_cpu(event_data->EnclosureHandle),
7536             event_data->StartPhyNum, event_data->NumEntries);
7537         for (i = 0; i < event_data->NumEntries; i++) {
7538                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
7539                 if (!handle)
7540                         continue;
7541                 phy_number = event_data->StartPhyNum + i;
7542                 reason_code = event_data->PHY[i].PhyStatus &
7543                     MPI2_EVENT_SAS_TOPO_RC_MASK;
7544                 switch (reason_code) {
7545                 case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
7546                         status_str = "target add";
7547                         break;
7548                 case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
7549                         status_str = "target remove";
7550                         break;
7551                 case MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING:
7552                         status_str = "delay target remove";
7553                         break;
7554                 case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
7555                         status_str = "link rate change";
7556                         break;
7557                 case MPI2_EVENT_SAS_TOPO_RC_NO_CHANGE:
7558                         status_str = "target responding";
7559                         break;
7560                 default:
7561                         status_str = "unknown";
7562                         break;
7563                 }
7564                 link_rate = event_data->PHY[i].LinkRate >> 4;
7565                 prev_link_rate = event_data->PHY[i].LinkRate & 0xF;
7566                 pr_info("\tphy(%02d), attached_handle(0x%04x): %s:" \
7567                     " link rate: new(0x%02x), old(0x%02x)\n", phy_number,
7568                     handle, status_str, link_rate, prev_link_rate);
7569
7570         }
7571 }
7572
7573 /**
7574  * _scsih_sas_topology_change_event - handle topology changes
7575  * @ioc: per adapter object
7576  * @fw_event: The fw_event_work object
7577  * Context: user.
7578  *
7579  */
7580 static int
7581 _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
7582         struct fw_event_work *fw_event)
7583 {
7584         int i;
7585         u16 parent_handle, handle;
7586         u16 reason_code;
7587         u8 phy_number, max_phys;
7588         struct _sas_node *sas_expander;
7589         u64 sas_address;
7590         unsigned long flags;
7591         u8 link_rate, prev_link_rate;
7592         struct hba_port *port;
7593         Mpi2EventDataSasTopologyChangeList_t *event_data =
7594                 (Mpi2EventDataSasTopologyChangeList_t *)
7595                 fw_event->event_data;
7596
7597         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
7598                 _scsih_sas_topology_change_event_debug(ioc, event_data);
7599
7600         if (ioc->shost_recovery || ioc->remove_host || ioc->pci_error_recovery)
7601                 return 0;
7602
7603         if (!ioc->sas_hba.num_phys)
7604                 _scsih_sas_host_add(ioc);
7605         else
7606                 _scsih_sas_host_refresh(ioc);
7607
7608         if (fw_event->ignore) {
7609                 dewtprintk(ioc, ioc_info(ioc, "ignoring expander event\n"));
7610                 return 0;
7611         }
7612
7613         parent_handle = le16_to_cpu(event_data->ExpanderDevHandle);
7614         port = mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0);
7615
7616         /* handle expander add */
7617         if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_ADDED)
7618                 if (_scsih_expander_add(ioc, parent_handle) != 0)
7619                         return 0;
7620
7621         spin_lock_irqsave(&ioc->sas_node_lock, flags);
7622         sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,
7623             parent_handle);
7624         if (sas_expander) {
7625                 sas_address = sas_expander->sas_address;
7626                 max_phys = sas_expander->num_phys;
7627                 port = sas_expander->port;
7628         } else if (parent_handle < ioc->sas_hba.num_phys) {
7629                 sas_address = ioc->sas_hba.sas_address;
7630                 max_phys = ioc->sas_hba.num_phys;
7631         } else {
7632                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
7633                 return 0;
7634         }
7635         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
7636
7637         /* handle siblings events */
7638         for (i = 0; i < event_data->NumEntries; i++) {
7639                 if (fw_event->ignore) {
7640                         dewtprintk(ioc,
7641                                    ioc_info(ioc, "ignoring expander event\n"));
7642                         return 0;
7643                 }
7644                 if (ioc->remove_host || ioc->pci_error_recovery)
7645                         return 0;
7646                 phy_number = event_data->StartPhyNum + i;
7647                 if (phy_number >= max_phys)
7648                         continue;
7649                 reason_code = event_data->PHY[i].PhyStatus &
7650                     MPI2_EVENT_SAS_TOPO_RC_MASK;
7651                 if ((event_data->PHY[i].PhyStatus &
7652                     MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT) && (reason_code !=
7653                     MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING))
7654                                 continue;
7655                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
7656                 if (!handle)
7657                         continue;
7658                 link_rate = event_data->PHY[i].LinkRate >> 4;
7659                 prev_link_rate = event_data->PHY[i].LinkRate & 0xF;
7660                 switch (reason_code) {
7661                 case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
7662
7663                         if (ioc->shost_recovery)
7664                                 break;
7665
7666                         if (link_rate == prev_link_rate)
7667                                 break;
7668
7669                         mpt3sas_transport_update_links(ioc, sas_address,
7670                             handle, phy_number, link_rate, port);
7671
7672                         if (link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
7673                                 break;
7674
7675                         _scsih_check_device(ioc, sas_address, handle,
7676                             phy_number, link_rate);
7677
7678                         if (!test_bit(handle, ioc->pend_os_device_add))
7679                                 break;
7680
7681                         fallthrough;
7682
7683                 case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
7684
7685                         if (ioc->shost_recovery)
7686                                 break;
7687
7688                         mpt3sas_transport_update_links(ioc, sas_address,
7689                             handle, phy_number, link_rate, port);
7690
7691                         _scsih_add_device(ioc, handle, phy_number, 0);
7692
7693                         break;
7694                 case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
7695
7696                         _scsih_device_remove_by_handle(ioc, handle);
7697                         break;
7698                 }
7699         }
7700
7701         /* handle expander removal */
7702         if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING &&
7703             sas_expander)
7704                 mpt3sas_expander_remove(ioc, sas_address, port);
7705
7706         return 0;
7707 }
7708
7709 /**
7710  * _scsih_sas_device_status_change_event_debug - debug for device event
7711  * @ioc: ?
7712  * @event_data: event data payload
7713  * Context: user.
7714  */
7715 static void
7716 _scsih_sas_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
7717         Mpi2EventDataSasDeviceStatusChange_t *event_data)
7718 {
7719         char *reason_str = NULL;
7720
7721         switch (event_data->ReasonCode) {
7722         case MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA:
7723                 reason_str = "smart data";
7724                 break;
7725         case MPI2_EVENT_SAS_DEV_STAT_RC_UNSUPPORTED:
7726                 reason_str = "unsupported device discovered";
7727                 break;
7728         case MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET:
7729                 reason_str = "internal device reset";
7730                 break;
7731         case MPI2_EVENT_SAS_DEV_STAT_RC_TASK_ABORT_INTERNAL:
7732                 reason_str = "internal task abort";
7733                 break;
7734         case MPI2_EVENT_SAS_DEV_STAT_RC_ABORT_TASK_SET_INTERNAL:
7735                 reason_str = "internal task abort set";
7736                 break;
7737         case MPI2_EVENT_SAS_DEV_STAT_RC_CLEAR_TASK_SET_INTERNAL:
7738                 reason_str = "internal clear task set";
7739                 break;
7740         case MPI2_EVENT_SAS_DEV_STAT_RC_QUERY_TASK_INTERNAL:
7741                 reason_str = "internal query task";
7742                 break;
7743         case MPI2_EVENT_SAS_DEV_STAT_RC_SATA_INIT_FAILURE:
7744                 reason_str = "sata init failure";
7745                 break;
7746         case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
7747                 reason_str = "internal device reset complete";
7748                 break;
7749         case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_TASK_ABORT_INTERNAL:
7750                 reason_str = "internal task abort complete";
7751                 break;
7752         case MPI2_EVENT_SAS_DEV_STAT_RC_ASYNC_NOTIFICATION:
7753                 reason_str = "internal async notification";
7754                 break;
7755         case MPI2_EVENT_SAS_DEV_STAT_RC_EXPANDER_REDUCED_FUNCTIONALITY:
7756                 reason_str = "expander reduced functionality";
7757                 break;
7758         case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_EXPANDER_REDUCED_FUNCTIONALITY:
7759                 reason_str = "expander reduced functionality complete";
7760                 break;
7761         default:
7762                 reason_str = "unknown reason";
7763                 break;
7764         }
7765         ioc_info(ioc, "device status change: (%s)\thandle(0x%04x), sas address(0x%016llx), tag(%d)",
7766                  reason_str, le16_to_cpu(event_data->DevHandle),
7767                  (u64)le64_to_cpu(event_data->SASAddress),
7768                  le16_to_cpu(event_data->TaskTag));
7769         if (event_data->ReasonCode == MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA)
7770                 pr_cont(", ASC(0x%x), ASCQ(0x%x)\n",
7771                         event_data->ASC, event_data->ASCQ);
7772         pr_cont("\n");
7773 }
7774
7775 /**
7776  * _scsih_sas_device_status_change_event - handle device status change
7777  * @ioc: per adapter object
7778  * @event_data: The fw event
7779  * Context: user.
7780  */
7781 static void
7782 _scsih_sas_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,
7783         Mpi2EventDataSasDeviceStatusChange_t *event_data)
7784 {
7785         struct MPT3SAS_TARGET *target_priv_data;
7786         struct _sas_device *sas_device;
7787         u64 sas_address;
7788         unsigned long flags;
7789
7790         /* In MPI Revision K (0xC), the internal device reset complete was
7791          * implemented, so avoid setting tm_busy flag for older firmware.
7792          */
7793         if ((ioc->facts.HeaderVersion >> 8) < 0xC)
7794                 return;
7795
7796         if (event_data->ReasonCode !=
7797             MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET &&
7798            event_data->ReasonCode !=
7799             MPI2_EVENT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET)
7800                 return;
7801
7802         spin_lock_irqsave(&ioc->sas_device_lock, flags);
7803         sas_address = le64_to_cpu(event_data->SASAddress);
7804         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
7805             sas_address,
7806             mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0));
7807
7808         if (!sas_device || !sas_device->starget)
7809                 goto out;
7810
7811         target_priv_data = sas_device->starget->hostdata;
7812         if (!target_priv_data)
7813                 goto out;
7814
7815         if (event_data->ReasonCode ==
7816             MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET)
7817                 target_priv_data->tm_busy = 1;
7818         else
7819                 target_priv_data->tm_busy = 0;
7820
7821         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
7822                 ioc_info(ioc,
7823                     "%s tm_busy flag for handle(0x%04x)\n",
7824                     (target_priv_data->tm_busy == 1) ? "Enable" : "Disable",
7825                     target_priv_data->handle);
7826
7827 out:
7828         if (sas_device)
7829                 sas_device_put(sas_device);
7830
7831         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
7832 }
7833
7834
7835 /**
7836  * _scsih_check_pcie_access_status - check access flags
7837  * @ioc: per adapter object
7838  * @wwid: wwid
7839  * @handle: sas device handle
7840  * @access_status: errors returned during discovery of the device
7841  *
7842  * Return: 0 for success, else failure
7843  */
7844 static u8
7845 _scsih_check_pcie_access_status(struct MPT3SAS_ADAPTER *ioc, u64 wwid,
7846         u16 handle, u8 access_status)
7847 {
7848         u8 rc = 1;
7849         char *desc = NULL;
7850
7851         switch (access_status) {
7852         case MPI26_PCIEDEV0_ASTATUS_NO_ERRORS:
7853         case MPI26_PCIEDEV0_ASTATUS_NEEDS_INITIALIZATION:
7854                 rc = 0;
7855                 break;
7856         case MPI26_PCIEDEV0_ASTATUS_CAPABILITY_FAILED:
7857                 desc = "PCIe device capability failed";
7858                 break;
7859         case MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED:
7860                 desc = "PCIe device blocked";
7861                 ioc_info(ioc,
7862                     "Device with Access Status (%s): wwid(0x%016llx), "
7863                     "handle(0x%04x)\n ll only be added to the internal list",
7864                     desc, (u64)wwid, handle);
7865                 rc = 0;
7866                 break;
7867         case MPI26_PCIEDEV0_ASTATUS_MEMORY_SPACE_ACCESS_FAILED:
7868                 desc = "PCIe device mem space access failed";
7869                 break;
7870         case MPI26_PCIEDEV0_ASTATUS_UNSUPPORTED_DEVICE:
7871                 desc = "PCIe device unsupported";
7872                 break;
7873         case MPI26_PCIEDEV0_ASTATUS_MSIX_REQUIRED:
7874                 desc = "PCIe device MSIx Required";
7875                 break;
7876         case MPI26_PCIEDEV0_ASTATUS_INIT_FAIL_MAX:
7877                 desc = "PCIe device init fail max";
7878                 break;
7879         case MPI26_PCIEDEV0_ASTATUS_UNKNOWN:
7880                 desc = "PCIe device status unknown";
7881                 break;
7882         case MPI26_PCIEDEV0_ASTATUS_NVME_READY_TIMEOUT:
7883                 desc = "nvme ready timeout";
7884                 break;
7885         case MPI26_PCIEDEV0_ASTATUS_NVME_DEVCFG_UNSUPPORTED:
7886                 desc = "nvme device configuration unsupported";
7887                 break;
7888         case MPI26_PCIEDEV0_ASTATUS_NVME_IDENTIFY_FAILED:
7889                 desc = "nvme identify failed";
7890                 break;
7891         case MPI26_PCIEDEV0_ASTATUS_NVME_QCONFIG_FAILED:
7892                 desc = "nvme qconfig failed";
7893                 break;
7894         case MPI26_PCIEDEV0_ASTATUS_NVME_QCREATION_FAILED:
7895                 desc = "nvme qcreation failed";
7896                 break;
7897         case MPI26_PCIEDEV0_ASTATUS_NVME_EVENTCFG_FAILED:
7898                 desc = "nvme eventcfg failed";
7899                 break;
7900         case MPI26_PCIEDEV0_ASTATUS_NVME_GET_FEATURE_STAT_FAILED:
7901                 desc = "nvme get feature stat failed";
7902                 break;
7903         case MPI26_PCIEDEV0_ASTATUS_NVME_IDLE_TIMEOUT:
7904                 desc = "nvme idle timeout";
7905                 break;
7906         case MPI26_PCIEDEV0_ASTATUS_NVME_FAILURE_STATUS:
7907                 desc = "nvme failure status";
7908                 break;
7909         default:
7910                 ioc_err(ioc, "NVMe discovery error(0x%02x): wwid(0x%016llx), handle(0x%04x)\n",
7911                         access_status, (u64)wwid, handle);
7912                 return rc;
7913         }
7914
7915         if (!rc)
7916                 return rc;
7917
7918         ioc_info(ioc, "NVMe discovery error(%s): wwid(0x%016llx), handle(0x%04x)\n",
7919                  desc, (u64)wwid, handle);
7920         return rc;
7921 }
7922
7923 /**
7924  * _scsih_pcie_device_remove_from_sml -  removing pcie device
7925  * from SML and free up associated memory
7926  * @ioc: per adapter object
7927  * @pcie_device: the pcie_device object
7928  */
7929 static void
7930 _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
7931         struct _pcie_device *pcie_device)
7932 {
7933         struct MPT3SAS_TARGET *sas_target_priv_data;
7934
7935         dewtprintk(ioc,
7936                    ioc_info(ioc, "%s: enter: handle(0x%04x), wwid(0x%016llx)\n",
7937                             __func__,
7938                             pcie_device->handle, (u64)pcie_device->wwid));
7939         if (pcie_device->enclosure_handle != 0)
7940                 dewtprintk(ioc,
7941                            ioc_info(ioc, "%s: enter: enclosure logical id(0x%016llx), slot(%d)\n",
7942                                     __func__,
7943                                     (u64)pcie_device->enclosure_logical_id,
7944                                     pcie_device->slot));
7945         if (pcie_device->connector_name[0] != '\0')
7946                 dewtprintk(ioc,
7947                            ioc_info(ioc, "%s: enter: enclosure level(0x%04x), connector name(%s)\n",
7948                                     __func__,
7949                                     pcie_device->enclosure_level,
7950                                     pcie_device->connector_name));
7951
7952         if (pcie_device->starget && pcie_device->starget->hostdata) {
7953                 sas_target_priv_data = pcie_device->starget->hostdata;
7954                 sas_target_priv_data->deleted = 1;
7955                 _scsih_ublock_io_device(ioc, pcie_device->wwid, NULL);
7956                 sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
7957         }
7958
7959         ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
7960                  pcie_device->handle, (u64)pcie_device->wwid);
7961         if (pcie_device->enclosure_handle != 0)
7962                 ioc_info(ioc, "removing : enclosure logical id(0x%016llx), slot(%d)\n",
7963                          (u64)pcie_device->enclosure_logical_id,
7964                          pcie_device->slot);
7965         if (pcie_device->connector_name[0] != '\0')
7966                 ioc_info(ioc, "removing: enclosure level(0x%04x), connector name( %s)\n",
7967                          pcie_device->enclosure_level,
7968                          pcie_device->connector_name);
7969
7970         if (pcie_device->starget && (pcie_device->access_status !=
7971                                 MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED))
7972                 scsi_remove_target(&pcie_device->starget->dev);
7973         dewtprintk(ioc,
7974                    ioc_info(ioc, "%s: exit: handle(0x%04x), wwid(0x%016llx)\n",
7975                             __func__,
7976                             pcie_device->handle, (u64)pcie_device->wwid));
7977         if (pcie_device->enclosure_handle != 0)
7978                 dewtprintk(ioc,
7979                            ioc_info(ioc, "%s: exit: enclosure logical id(0x%016llx), slot(%d)\n",
7980                                     __func__,
7981                                     (u64)pcie_device->enclosure_logical_id,
7982                                     pcie_device->slot));
7983         if (pcie_device->connector_name[0] != '\0')
7984                 dewtprintk(ioc,
7985                            ioc_info(ioc, "%s: exit: enclosure level(0x%04x), connector name( %s)\n",
7986                                     __func__,
7987                                     pcie_device->enclosure_level,
7988                                     pcie_device->connector_name));
7989
7990         kfree(pcie_device->serial_number);
7991 }
7992
7993
7994 /**
7995  * _scsih_pcie_check_device - checking device responsiveness
7996  * @ioc: per adapter object
7997  * @handle: attached device handle
7998  */
7999 static void
8000 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
8001 {
8002         Mpi2ConfigReply_t mpi_reply;
8003         Mpi26PCIeDevicePage0_t pcie_device_pg0;
8004         u32 ioc_status;
8005         struct _pcie_device *pcie_device;
8006         u64 wwid;
8007         unsigned long flags;
8008         struct scsi_target *starget;
8009         struct MPT3SAS_TARGET *sas_target_priv_data;
8010         u32 device_info;
8011
8012         if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
8013                 &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle)))
8014                 return;
8015
8016         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
8017         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
8018                 return;
8019
8020         /* check if this is end device */
8021         device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
8022         if (!(_scsih_is_nvme_pciescsi_device(device_info)))
8023                 return;
8024
8025         wwid = le64_to_cpu(pcie_device_pg0.WWID);
8026         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8027         pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
8028
8029         if (!pcie_device) {
8030                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8031                 return;
8032         }
8033
8034         if (unlikely(pcie_device->handle != handle)) {
8035                 starget = pcie_device->starget;
8036                 sas_target_priv_data = starget->hostdata;
8037                 pcie_device->access_status = pcie_device_pg0.AccessStatus;
8038                 starget_printk(KERN_INFO, starget,
8039                     "handle changed from(0x%04x) to (0x%04x)!!!\n",
8040                     pcie_device->handle, handle);
8041                 sas_target_priv_data->handle = handle;
8042                 pcie_device->handle = handle;
8043
8044                 if (le32_to_cpu(pcie_device_pg0.Flags) &
8045                     MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
8046                         pcie_device->enclosure_level =
8047                             pcie_device_pg0.EnclosureLevel;
8048                         memcpy(&pcie_device->connector_name[0],
8049                             &pcie_device_pg0.ConnectorName[0], 4);
8050                 } else {
8051                         pcie_device->enclosure_level = 0;
8052                         pcie_device->connector_name[0] = '\0';
8053                 }
8054         }
8055
8056         /* check if device is present */
8057         if (!(le32_to_cpu(pcie_device_pg0.Flags) &
8058             MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT)) {
8059                 ioc_info(ioc, "device is not present handle(0x%04x), flags!!!\n",
8060                          handle);
8061                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8062                 pcie_device_put(pcie_device);
8063                 return;
8064         }
8065
8066         /* check if there were any issues with discovery */
8067         if (_scsih_check_pcie_access_status(ioc, wwid, handle,
8068             pcie_device_pg0.AccessStatus)) {
8069                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8070                 pcie_device_put(pcie_device);
8071                 return;
8072         }
8073
8074         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8075         pcie_device_put(pcie_device);
8076
8077         _scsih_ublock_io_device(ioc, wwid, NULL);
8078
8079         return;
8080 }
8081
8082 /**
8083  * _scsih_pcie_add_device -  creating pcie device object
8084  * @ioc: per adapter object
8085  * @handle: pcie device handle
8086  *
8087  * Creating end device object, stored in ioc->pcie_device_list.
8088  *
8089  * Return: 1 means queue the event later, 0 means complete the event
8090  */
8091 static int
8092 _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
8093 {
8094         Mpi26PCIeDevicePage0_t pcie_device_pg0;
8095         Mpi26PCIeDevicePage2_t pcie_device_pg2;
8096         Mpi2ConfigReply_t mpi_reply;
8097         struct _pcie_device *pcie_device;
8098         struct _enclosure_node *enclosure_dev;
8099         u32 ioc_status;
8100         u64 wwid;
8101
8102         if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
8103             &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle))) {
8104                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8105                         __FILE__, __LINE__, __func__);
8106                 return 0;
8107         }
8108         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
8109             MPI2_IOCSTATUS_MASK;
8110         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8111                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8112                         __FILE__, __LINE__, __func__);
8113                 return 0;
8114         }
8115
8116         set_bit(handle, ioc->pend_os_device_add);
8117         wwid = le64_to_cpu(pcie_device_pg0.WWID);
8118
8119         /* check if device is present */
8120         if (!(le32_to_cpu(pcie_device_pg0.Flags) &
8121                 MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT)) {
8122                 ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",
8123                         handle);
8124                 return 0;
8125         }
8126
8127         /* check if there were any issues with discovery */
8128         if (_scsih_check_pcie_access_status(ioc, wwid, handle,
8129             pcie_device_pg0.AccessStatus))
8130                 return 0;
8131
8132         if (!(_scsih_is_nvme_pciescsi_device(le32_to_cpu
8133             (pcie_device_pg0.DeviceInfo))))
8134                 return 0;
8135
8136         pcie_device = mpt3sas_get_pdev_by_wwid(ioc, wwid);
8137         if (pcie_device) {
8138                 clear_bit(handle, ioc->pend_os_device_add);
8139                 pcie_device_put(pcie_device);
8140                 return 0;
8141         }
8142
8143         /* PCIe Device Page 2 contains read-only information about a
8144          * specific NVMe device; therefore, this page is only
8145          * valid for NVMe devices and skip for pcie devices of type scsi.
8146          */
8147         if (!(mpt3sas_scsih_is_pcie_scsi_device(
8148                 le32_to_cpu(pcie_device_pg0.DeviceInfo)))) {
8149                 if (mpt3sas_config_get_pcie_device_pg2(ioc, &mpi_reply,
8150                     &pcie_device_pg2, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
8151                     handle)) {
8152                         ioc_err(ioc,
8153                             "failure at %s:%d/%s()!\n", __FILE__,
8154                             __LINE__, __func__);
8155                         return 0;
8156                 }
8157
8158                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
8159                                         MPI2_IOCSTATUS_MASK;
8160                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8161                         ioc_err(ioc,
8162                             "failure at %s:%d/%s()!\n", __FILE__,
8163                             __LINE__, __func__);
8164                         return 0;
8165                 }
8166         }
8167
8168         pcie_device = kzalloc(sizeof(struct _pcie_device), GFP_KERNEL);
8169         if (!pcie_device) {
8170                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8171                         __FILE__, __LINE__, __func__);
8172                 return 0;
8173         }
8174
8175         kref_init(&pcie_device->refcount);
8176         pcie_device->id = ioc->pcie_target_id++;
8177         pcie_device->channel = PCIE_CHANNEL;
8178         pcie_device->handle = handle;
8179         pcie_device->access_status = pcie_device_pg0.AccessStatus;
8180         pcie_device->device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
8181         pcie_device->wwid = wwid;
8182         pcie_device->port_num = pcie_device_pg0.PortNum;
8183         pcie_device->fast_path = (le32_to_cpu(pcie_device_pg0.Flags) &
8184             MPI26_PCIEDEV0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
8185
8186         pcie_device->enclosure_handle =
8187             le16_to_cpu(pcie_device_pg0.EnclosureHandle);
8188         if (pcie_device->enclosure_handle != 0)
8189                 pcie_device->slot = le16_to_cpu(pcie_device_pg0.Slot);
8190
8191         if (le32_to_cpu(pcie_device_pg0.Flags) &
8192             MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
8193                 pcie_device->enclosure_level = pcie_device_pg0.EnclosureLevel;
8194                 memcpy(&pcie_device->connector_name[0],
8195                     &pcie_device_pg0.ConnectorName[0], 4);
8196         } else {
8197                 pcie_device->enclosure_level = 0;
8198                 pcie_device->connector_name[0] = '\0';
8199         }
8200
8201         /* get enclosure_logical_id */
8202         if (pcie_device->enclosure_handle) {
8203                 enclosure_dev =
8204                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
8205                                                 pcie_device->enclosure_handle);
8206                 if (enclosure_dev)
8207                         pcie_device->enclosure_logical_id =
8208                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
8209         }
8210         /* TODO -- Add device name once FW supports it */
8211         if (!(mpt3sas_scsih_is_pcie_scsi_device(
8212             le32_to_cpu(pcie_device_pg0.DeviceInfo)))) {
8213                 pcie_device->nvme_mdts =
8214                     le32_to_cpu(pcie_device_pg2.MaximumDataTransferSize);
8215                 pcie_device->shutdown_latency =
8216                         le16_to_cpu(pcie_device_pg2.ShutdownLatency);
8217                 /*
8218                  * Set IOC's max_shutdown_latency to drive's RTD3 Entry Latency
8219                  * if drive's RTD3 Entry Latency is greater then IOC's
8220                  * max_shutdown_latency.
8221                  */
8222                 if (pcie_device->shutdown_latency > ioc->max_shutdown_latency)
8223                         ioc->max_shutdown_latency =
8224                                 pcie_device->shutdown_latency;
8225                 if (pcie_device_pg2.ControllerResetTO)
8226                         pcie_device->reset_timeout =
8227                             pcie_device_pg2.ControllerResetTO;
8228                 else
8229                         pcie_device->reset_timeout = 30;
8230         } else
8231                 pcie_device->reset_timeout = 30;
8232
8233         if (ioc->wait_for_discovery_to_complete)
8234                 _scsih_pcie_device_init_add(ioc, pcie_device);
8235         else
8236                 _scsih_pcie_device_add(ioc, pcie_device);
8237
8238         pcie_device_put(pcie_device);
8239         return 0;
8240 }
8241
8242 /**
8243  * _scsih_pcie_topology_change_event_debug - debug for topology
8244  * event
8245  * @ioc: per adapter object
8246  * @event_data: event data payload
8247  * Context: user.
8248  */
8249 static void
8250 _scsih_pcie_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8251         Mpi26EventDataPCIeTopologyChangeList_t *event_data)
8252 {
8253         int i;
8254         u16 handle;
8255         u16 reason_code;
8256         u8 port_number;
8257         char *status_str = NULL;
8258         u8 link_rate, prev_link_rate;
8259
8260         switch (event_data->SwitchStatus) {
8261         case MPI26_EVENT_PCIE_TOPO_SS_ADDED:
8262                 status_str = "add";
8263                 break;
8264         case MPI26_EVENT_PCIE_TOPO_SS_NOT_RESPONDING:
8265                 status_str = "remove";
8266                 break;
8267         case MPI26_EVENT_PCIE_TOPO_SS_RESPONDING:
8268         case 0:
8269                 status_str =  "responding";
8270                 break;
8271         case MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING:
8272                 status_str = "remove delay";
8273                 break;
8274         default:
8275                 status_str = "unknown status";
8276                 break;
8277         }
8278         ioc_info(ioc, "pcie topology change: (%s)\n", status_str);
8279         pr_info("\tswitch_handle(0x%04x), enclosure_handle(0x%04x)"
8280                 "start_port(%02d), count(%d)\n",
8281                 le16_to_cpu(event_data->SwitchDevHandle),
8282                 le16_to_cpu(event_data->EnclosureHandle),
8283                 event_data->StartPortNum, event_data->NumEntries);
8284         for (i = 0; i < event_data->NumEntries; i++) {
8285                 handle =
8286                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
8287                 if (!handle)
8288                         continue;
8289                 port_number = event_data->StartPortNum + i;
8290                 reason_code = event_data->PortEntry[i].PortStatus;
8291                 switch (reason_code) {
8292                 case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
8293                         status_str = "target add";
8294                         break;
8295                 case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
8296                         status_str = "target remove";
8297                         break;
8298                 case MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
8299                         status_str = "delay target remove";
8300                         break;
8301                 case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
8302                         status_str = "link rate change";
8303                         break;
8304                 case MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE:
8305                         status_str = "target responding";
8306                         break;
8307                 default:
8308                         status_str = "unknown";
8309                         break;
8310                 }
8311                 link_rate = event_data->PortEntry[i].CurrentPortInfo &
8312                         MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8313                 prev_link_rate = event_data->PortEntry[i].PreviousPortInfo &
8314                         MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8315                 pr_info("\tport(%02d), attached_handle(0x%04x): %s:"
8316                         " link rate: new(0x%02x), old(0x%02x)\n", port_number,
8317                         handle, status_str, link_rate, prev_link_rate);
8318         }
8319 }
8320
8321 /**
8322  * _scsih_pcie_topology_change_event - handle PCIe topology
8323  *  changes
8324  * @ioc: per adapter object
8325  * @fw_event: The fw_event_work object
8326  * Context: user.
8327  *
8328  */
8329 static void
8330 _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
8331         struct fw_event_work *fw_event)
8332 {
8333         int i;
8334         u16 handle;
8335         u16 reason_code;
8336         u8 link_rate, prev_link_rate;
8337         unsigned long flags;
8338         int rc;
8339         Mpi26EventDataPCIeTopologyChangeList_t *event_data =
8340                 (Mpi26EventDataPCIeTopologyChangeList_t *) fw_event->event_data;
8341         struct _pcie_device *pcie_device;
8342
8343         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8344                 _scsih_pcie_topology_change_event_debug(ioc, event_data);
8345
8346         if (ioc->shost_recovery || ioc->remove_host ||
8347                 ioc->pci_error_recovery)
8348                 return;
8349
8350         if (fw_event->ignore) {
8351                 dewtprintk(ioc, ioc_info(ioc, "ignoring switch event\n"));
8352                 return;
8353         }
8354
8355         /* handle siblings events */
8356         for (i = 0; i < event_data->NumEntries; i++) {
8357                 if (fw_event->ignore) {
8358                         dewtprintk(ioc,
8359                                    ioc_info(ioc, "ignoring switch event\n"));
8360                         return;
8361                 }
8362                 if (ioc->remove_host || ioc->pci_error_recovery)
8363                         return;
8364                 reason_code = event_data->PortEntry[i].PortStatus;
8365                 handle =
8366                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
8367                 if (!handle)
8368                         continue;
8369
8370                 link_rate = event_data->PortEntry[i].CurrentPortInfo
8371                         & MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8372                 prev_link_rate = event_data->PortEntry[i].PreviousPortInfo
8373                         & MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8374
8375                 switch (reason_code) {
8376                 case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
8377                         if (ioc->shost_recovery)
8378                                 break;
8379                         if (link_rate == prev_link_rate)
8380                                 break;
8381                         if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
8382                                 break;
8383
8384                         _scsih_pcie_check_device(ioc, handle);
8385
8386                         /* This code after this point handles the test case
8387                          * where a device has been added, however its returning
8388                          * BUSY for sometime.  Then before the Device Missing
8389                          * Delay expires and the device becomes READY, the
8390                          * device is removed and added back.
8391                          */
8392                         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8393                         pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
8394                         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8395
8396                         if (pcie_device) {
8397                                 pcie_device_put(pcie_device);
8398                                 break;
8399                         }
8400
8401                         if (!test_bit(handle, ioc->pend_os_device_add))
8402                                 break;
8403
8404                         dewtprintk(ioc,
8405                                    ioc_info(ioc, "handle(0x%04x) device not found: convert event to a device add\n",
8406                                             handle));
8407                         event_data->PortEntry[i].PortStatus &= 0xF0;
8408                         event_data->PortEntry[i].PortStatus |=
8409                                 MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED;
8410                         fallthrough;
8411                 case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
8412                         if (ioc->shost_recovery)
8413                                 break;
8414                         if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
8415                                 break;
8416
8417                         rc = _scsih_pcie_add_device(ioc, handle);
8418                         if (!rc) {
8419                                 /* mark entry vacant */
8420                                 /* TODO This needs to be reviewed and fixed,
8421                                  * we dont have an entry
8422                                  * to make an event void like vacant
8423                                  */
8424                                 event_data->PortEntry[i].PortStatus |=
8425                                         MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE;
8426                         }
8427                         break;
8428                 case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
8429                         _scsih_pcie_device_remove_by_handle(ioc, handle);
8430                         break;
8431                 }
8432         }
8433 }
8434
8435 /**
8436  * _scsih_pcie_device_status_change_event_debug - debug for device event
8437  * @ioc: ?
8438  * @event_data: event data payload
8439  * Context: user.
8440  */
8441 static void
8442 _scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8443         Mpi26EventDataPCIeDeviceStatusChange_t *event_data)
8444 {
8445         char *reason_str = NULL;
8446
8447         switch (event_data->ReasonCode) {
8448         case MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA:
8449                 reason_str = "smart data";
8450                 break;
8451         case MPI26_EVENT_PCIDEV_STAT_RC_UNSUPPORTED:
8452                 reason_str = "unsupported device discovered";
8453                 break;
8454         case MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET:
8455                 reason_str = "internal device reset";
8456                 break;
8457         case MPI26_EVENT_PCIDEV_STAT_RC_TASK_ABORT_INTERNAL:
8458                 reason_str = "internal task abort";
8459                 break;
8460         case MPI26_EVENT_PCIDEV_STAT_RC_ABORT_TASK_SET_INTERNAL:
8461                 reason_str = "internal task abort set";
8462                 break;
8463         case MPI26_EVENT_PCIDEV_STAT_RC_CLEAR_TASK_SET_INTERNAL:
8464                 reason_str = "internal clear task set";
8465                 break;
8466         case MPI26_EVENT_PCIDEV_STAT_RC_QUERY_TASK_INTERNAL:
8467                 reason_str = "internal query task";
8468                 break;
8469         case MPI26_EVENT_PCIDEV_STAT_RC_DEV_INIT_FAILURE:
8470                 reason_str = "device init failure";
8471                 break;
8472         case MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
8473                 reason_str = "internal device reset complete";
8474                 break;
8475         case MPI26_EVENT_PCIDEV_STAT_RC_CMP_TASK_ABORT_INTERNAL:
8476                 reason_str = "internal task abort complete";
8477                 break;
8478         case MPI26_EVENT_PCIDEV_STAT_RC_ASYNC_NOTIFICATION:
8479                 reason_str = "internal async notification";
8480                 break;
8481         case MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED:
8482                 reason_str = "pcie hot reset failed";
8483                 break;
8484         default:
8485                 reason_str = "unknown reason";
8486                 break;
8487         }
8488
8489         ioc_info(ioc, "PCIE device status change: (%s)\n"
8490                  "\thandle(0x%04x), WWID(0x%016llx), tag(%d)",
8491                  reason_str, le16_to_cpu(event_data->DevHandle),
8492                  (u64)le64_to_cpu(event_data->WWID),
8493                  le16_to_cpu(event_data->TaskTag));
8494         if (event_data->ReasonCode == MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA)
8495                 pr_cont(", ASC(0x%x), ASCQ(0x%x)\n",
8496                         event_data->ASC, event_data->ASCQ);
8497         pr_cont("\n");
8498 }
8499
8500 /**
8501  * _scsih_pcie_device_status_change_event - handle device status
8502  * change
8503  * @ioc: per adapter object
8504  * @fw_event: The fw_event_work object
8505  * Context: user.
8506  */
8507 static void
8508 _scsih_pcie_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,
8509         struct fw_event_work *fw_event)
8510 {
8511         struct MPT3SAS_TARGET *target_priv_data;
8512         struct _pcie_device *pcie_device;
8513         u64 wwid;
8514         unsigned long flags;
8515         Mpi26EventDataPCIeDeviceStatusChange_t *event_data =
8516                 (Mpi26EventDataPCIeDeviceStatusChange_t *)fw_event->event_data;
8517         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8518                 _scsih_pcie_device_status_change_event_debug(ioc,
8519                         event_data);
8520
8521         if (event_data->ReasonCode !=
8522                 MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET &&
8523                 event_data->ReasonCode !=
8524                 MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET)
8525                 return;
8526
8527         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8528         wwid = le64_to_cpu(event_data->WWID);
8529         pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
8530
8531         if (!pcie_device || !pcie_device->starget)
8532                 goto out;
8533
8534         target_priv_data = pcie_device->starget->hostdata;
8535         if (!target_priv_data)
8536                 goto out;
8537
8538         if (event_data->ReasonCode ==
8539                 MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET)
8540                 target_priv_data->tm_busy = 1;
8541         else
8542                 target_priv_data->tm_busy = 0;
8543 out:
8544         if (pcie_device)
8545                 pcie_device_put(pcie_device);
8546
8547         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8548 }
8549
8550 /**
8551  * _scsih_sas_enclosure_dev_status_change_event_debug - debug for enclosure
8552  * event
8553  * @ioc: per adapter object
8554  * @event_data: event data payload
8555  * Context: user.
8556  */
8557 static void
8558 _scsih_sas_enclosure_dev_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8559         Mpi2EventDataSasEnclDevStatusChange_t *event_data)
8560 {
8561         char *reason_str = NULL;
8562
8563         switch (event_data->ReasonCode) {
8564         case MPI2_EVENT_SAS_ENCL_RC_ADDED:
8565                 reason_str = "enclosure add";
8566                 break;
8567         case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
8568                 reason_str = "enclosure remove";
8569                 break;
8570         default:
8571                 reason_str = "unknown reason";
8572                 break;
8573         }
8574
8575         ioc_info(ioc, "enclosure status change: (%s)\n"
8576                  "\thandle(0x%04x), enclosure logical id(0x%016llx) number slots(%d)\n",
8577                  reason_str,
8578                  le16_to_cpu(event_data->EnclosureHandle),
8579                  (u64)le64_to_cpu(event_data->EnclosureLogicalID),
8580                  le16_to_cpu(event_data->StartSlot));
8581 }
8582
8583 /**
8584  * _scsih_sas_enclosure_dev_status_change_event - handle enclosure events
8585  * @ioc: per adapter object
8586  * @fw_event: The fw_event_work object
8587  * Context: user.
8588  */
8589 static void
8590 _scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,
8591         struct fw_event_work *fw_event)
8592 {
8593         Mpi2ConfigReply_t mpi_reply;
8594         struct _enclosure_node *enclosure_dev = NULL;
8595         Mpi2EventDataSasEnclDevStatusChange_t *event_data =
8596                 (Mpi2EventDataSasEnclDevStatusChange_t *)fw_event->event_data;
8597         int rc;
8598         u16 enclosure_handle = le16_to_cpu(event_data->EnclosureHandle);
8599
8600         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8601                 _scsih_sas_enclosure_dev_status_change_event_debug(ioc,
8602                      (Mpi2EventDataSasEnclDevStatusChange_t *)
8603                      fw_event->event_data);
8604         if (ioc->shost_recovery)
8605                 return;
8606
8607         if (enclosure_handle)
8608                 enclosure_dev =
8609                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
8610                                                 enclosure_handle);
8611         switch (event_data->ReasonCode) {
8612         case MPI2_EVENT_SAS_ENCL_RC_ADDED:
8613                 if (!enclosure_dev) {
8614                         enclosure_dev =
8615                                 kzalloc(sizeof(struct _enclosure_node),
8616                                         GFP_KERNEL);
8617                         if (!enclosure_dev) {
8618                                 ioc_info(ioc, "failure at %s:%d/%s()!\n",
8619                                          __FILE__, __LINE__, __func__);
8620                                 return;
8621                         }
8622                         rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
8623                                 &enclosure_dev->pg0,
8624                                 MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
8625                                 enclosure_handle);
8626
8627                         if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
8628                                                 MPI2_IOCSTATUS_MASK)) {
8629                                 kfree(enclosure_dev);
8630                                 return;
8631                         }
8632
8633                         list_add_tail(&enclosure_dev->list,
8634                                                         &ioc->enclosure_list);
8635                 }
8636                 break;
8637         case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
8638                 if (enclosure_dev) {
8639                         list_del(&enclosure_dev->list);
8640                         kfree(enclosure_dev);
8641                 }
8642                 break;
8643         default:
8644                 break;
8645         }
8646 }
8647
8648 /**
8649  * _scsih_sas_broadcast_primitive_event - handle broadcast events
8650  * @ioc: per adapter object
8651  * @fw_event: The fw_event_work object
8652  * Context: user.
8653  */
8654 static void
8655 _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
8656         struct fw_event_work *fw_event)
8657 {
8658         struct scsi_cmnd *scmd;
8659         struct scsi_device *sdev;
8660         struct scsiio_tracker *st;
8661         u16 smid, handle;
8662         u32 lun;
8663         struct MPT3SAS_DEVICE *sas_device_priv_data;
8664         u32 termination_count;
8665         u32 query_count;
8666         Mpi2SCSITaskManagementReply_t *mpi_reply;
8667         Mpi2EventDataSasBroadcastPrimitive_t *event_data =
8668                 (Mpi2EventDataSasBroadcastPrimitive_t *)
8669                 fw_event->event_data;
8670         u16 ioc_status;
8671         unsigned long flags;
8672         int r;
8673         u8 max_retries = 0;
8674         u8 task_abort_retries;
8675
8676         mutex_lock(&ioc->tm_cmds.mutex);
8677         ioc_info(ioc, "%s: enter: phy number(%d), width(%d)\n",
8678                  __func__, event_data->PhyNum, event_data->PortWidth);
8679
8680         _scsih_block_io_all_device(ioc);
8681
8682         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8683         mpi_reply = ioc->tm_cmds.reply;
8684  broadcast_aen_retry:
8685
8686         /* sanity checks for retrying this loop */
8687         if (max_retries++ == 5) {
8688                 dewtprintk(ioc, ioc_info(ioc, "%s: giving up\n", __func__));
8689                 goto out;
8690         } else if (max_retries > 1)
8691                 dewtprintk(ioc,
8692                            ioc_info(ioc, "%s: %d retry\n",
8693                                     __func__, max_retries - 1));
8694
8695         termination_count = 0;
8696         query_count = 0;
8697         for (smid = 1; smid <= ioc->scsiio_depth; smid++) {
8698                 if (ioc->shost_recovery)
8699                         goto out;
8700                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
8701                 if (!scmd)
8702                         continue;
8703                 st = scsi_cmd_priv(scmd);
8704                 sdev = scmd->device;
8705                 sas_device_priv_data = sdev->hostdata;
8706                 if (!sas_device_priv_data || !sas_device_priv_data->sas_target)
8707                         continue;
8708                  /* skip hidden raid components */
8709                 if (sas_device_priv_data->sas_target->flags &
8710                     MPT_TARGET_FLAGS_RAID_COMPONENT)
8711                         continue;
8712                  /* skip volumes */
8713                 if (sas_device_priv_data->sas_target->flags &
8714                     MPT_TARGET_FLAGS_VOLUME)
8715                         continue;
8716                  /* skip PCIe devices */
8717                 if (sas_device_priv_data->sas_target->flags &
8718                     MPT_TARGET_FLAGS_PCIE_DEVICE)
8719                         continue;
8720
8721                 handle = sas_device_priv_data->sas_target->handle;
8722                 lun = sas_device_priv_data->lun;
8723                 query_count++;
8724
8725                 if (ioc->shost_recovery)
8726                         goto out;
8727
8728                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
8729                 r = mpt3sas_scsih_issue_tm(ioc, handle, 0, 0, lun,
8730                         MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, st->smid,
8731                         st->msix_io, 30, 0);
8732                 if (r == FAILED) {
8733                         sdev_printk(KERN_WARNING, sdev,
8734                             "mpt3sas_scsih_issue_tm: FAILED when sending "
8735                             "QUERY_TASK: scmd(%p)\n", scmd);
8736                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8737                         goto broadcast_aen_retry;
8738                 }
8739                 ioc_status = le16_to_cpu(mpi_reply->IOCStatus)
8740                     & MPI2_IOCSTATUS_MASK;
8741                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8742                         sdev_printk(KERN_WARNING, sdev,
8743                                 "query task: FAILED with IOCSTATUS(0x%04x), scmd(%p)\n",
8744                                 ioc_status, scmd);
8745                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8746                         goto broadcast_aen_retry;
8747                 }
8748
8749                 /* see if IO is still owned by IOC and target */
8750                 if (mpi_reply->ResponseCode ==
8751                      MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED ||
8752                      mpi_reply->ResponseCode ==
8753                      MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC) {
8754                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8755                         continue;
8756                 }
8757                 task_abort_retries = 0;
8758  tm_retry:
8759                 if (task_abort_retries++ == 60) {
8760                         dewtprintk(ioc,
8761                                    ioc_info(ioc, "%s: ABORT_TASK: giving up\n",
8762                                             __func__));
8763                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8764                         goto broadcast_aen_retry;
8765                 }
8766
8767                 if (ioc->shost_recovery)
8768                         goto out_no_lock;
8769
8770                 r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->channel, sdev->id,
8771                         sdev->lun, MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
8772                         st->smid, st->msix_io, 30, 0);
8773                 if (r == FAILED || st->cb_idx != 0xFF) {
8774                         sdev_printk(KERN_WARNING, sdev,
8775                             "mpt3sas_scsih_issue_tm: ABORT_TASK: FAILED : "
8776                             "scmd(%p)\n", scmd);
8777                         goto tm_retry;
8778                 }
8779
8780                 if (task_abort_retries > 1)
8781                         sdev_printk(KERN_WARNING, sdev,
8782                             "mpt3sas_scsih_issue_tm: ABORT_TASK: RETRIES (%d):"
8783                             " scmd(%p)\n",
8784                             task_abort_retries - 1, scmd);
8785
8786                 termination_count += le32_to_cpu(mpi_reply->TerminationCount);
8787                 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8788         }
8789
8790         if (ioc->broadcast_aen_pending) {
8791                 dewtprintk(ioc,
8792                            ioc_info(ioc,
8793                                     "%s: loop back due to pending AEN\n",
8794                                     __func__));
8795                  ioc->broadcast_aen_pending = 0;
8796                  goto broadcast_aen_retry;
8797         }
8798
8799  out:
8800         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
8801  out_no_lock:
8802
8803         dewtprintk(ioc,
8804                    ioc_info(ioc, "%s - exit, query_count = %d termination_count = %d\n",
8805                             __func__, query_count, termination_count));
8806
8807         ioc->broadcast_aen_busy = 0;
8808         if (!ioc->shost_recovery)
8809                 _scsih_ublock_io_all_device(ioc);
8810         mutex_unlock(&ioc->tm_cmds.mutex);
8811 }
8812
8813 /**
8814  * _scsih_sas_discovery_event - handle discovery events
8815  * @ioc: per adapter object
8816  * @fw_event: The fw_event_work object
8817  * Context: user.
8818  */
8819 static void
8820 _scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,
8821         struct fw_event_work *fw_event)
8822 {
8823         Mpi2EventDataSasDiscovery_t *event_data =
8824                 (Mpi2EventDataSasDiscovery_t *) fw_event->event_data;
8825
8826         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) {
8827                 ioc_info(ioc, "discovery event: (%s)",
8828                          event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ?
8829                          "start" : "stop");
8830                 if (event_data->DiscoveryStatus)
8831                         pr_cont("discovery_status(0x%08x)",
8832                                 le32_to_cpu(event_data->DiscoveryStatus));
8833                 pr_cont("\n");
8834         }
8835
8836         if (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED &&
8837             !ioc->sas_hba.num_phys) {
8838                 if (disable_discovery > 0 && ioc->shost_recovery) {
8839                         /* Wait for the reset to complete */
8840                         while (ioc->shost_recovery)
8841                                 ssleep(1);
8842                 }
8843                 _scsih_sas_host_add(ioc);
8844         }
8845 }
8846
8847 /**
8848  * _scsih_sas_device_discovery_error_event - display SAS device discovery error
8849  *                                              events
8850  * @ioc: per adapter object
8851  * @fw_event: The fw_event_work object
8852  * Context: user.
8853  */
8854 static void
8855 _scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,
8856         struct fw_event_work *fw_event)
8857 {
8858         Mpi25EventDataSasDeviceDiscoveryError_t *event_data =
8859                 (Mpi25EventDataSasDeviceDiscoveryError_t *)fw_event->event_data;
8860
8861         switch (event_data->ReasonCode) {
8862         case MPI25_EVENT_SAS_DISC_ERR_SMP_FAILED:
8863                 ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_port:0x%02x) has failed\n",
8864                          le16_to_cpu(event_data->DevHandle),
8865                          (u64)le64_to_cpu(event_data->SASAddress),
8866                          event_data->PhysicalPort);
8867                 break;
8868         case MPI25_EVENT_SAS_DISC_ERR_SMP_TIMEOUT:
8869                 ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_port:0x%02x) has timed out\n",
8870                          le16_to_cpu(event_data->DevHandle),
8871                          (u64)le64_to_cpu(event_data->SASAddress),
8872                          event_data->PhysicalPort);
8873                 break;
8874         default:
8875                 break;
8876         }
8877 }
8878
8879 /**
8880  * _scsih_pcie_enumeration_event - handle enumeration events
8881  * @ioc: per adapter object
8882  * @fw_event: The fw_event_work object
8883  * Context: user.
8884  */
8885 static void
8886 _scsih_pcie_enumeration_event(struct MPT3SAS_ADAPTER *ioc,
8887         struct fw_event_work *fw_event)
8888 {
8889         Mpi26EventDataPCIeEnumeration_t *event_data =
8890                 (Mpi26EventDataPCIeEnumeration_t *)fw_event->event_data;
8891
8892         if (!(ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK))
8893                 return;
8894
8895         ioc_info(ioc, "pcie enumeration event: (%s) Flag 0x%02x",
8896                  (event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED) ?
8897                  "started" : "completed",
8898                  event_data->Flags);
8899         if (event_data->EnumerationStatus)
8900                 pr_cont("enumeration_status(0x%08x)",
8901                         le32_to_cpu(event_data->EnumerationStatus));
8902         pr_cont("\n");
8903 }
8904
8905 /**
8906  * _scsih_ir_fastpath - turn on fastpath for IR physdisk
8907  * @ioc: per adapter object
8908  * @handle: device handle for physical disk
8909  * @phys_disk_num: physical disk number
8910  *
8911  * Return: 0 for success, else failure.
8912  */
8913 static int
8914 _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)
8915 {
8916         Mpi2RaidActionRequest_t *mpi_request;
8917         Mpi2RaidActionReply_t *mpi_reply;
8918         u16 smid;
8919         u8 issue_reset = 0;
8920         int rc = 0;
8921         u16 ioc_status;
8922         u32 log_info;
8923
8924         if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
8925                 return rc;
8926
8927         mutex_lock(&ioc->scsih_cmds.mutex);
8928
8929         if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
8930                 ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
8931                 rc = -EAGAIN;
8932                 goto out;
8933         }
8934         ioc->scsih_cmds.status = MPT3_CMD_PENDING;
8935
8936         smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
8937         if (!smid) {
8938                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
8939                 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8940                 rc = -EAGAIN;
8941                 goto out;
8942         }
8943
8944         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
8945         ioc->scsih_cmds.smid = smid;
8946         memset(mpi_request, 0, sizeof(Mpi2RaidActionRequest_t));
8947
8948         mpi_request->Function = MPI2_FUNCTION_RAID_ACTION;
8949         mpi_request->Action = MPI2_RAID_ACTION_PHYSDISK_HIDDEN;
8950         mpi_request->PhysDiskNum = phys_disk_num;
8951
8952         dewtprintk(ioc,
8953                    ioc_info(ioc, "IR RAID_ACTION: turning fast path on for handle(0x%04x), phys_disk_num (0x%02x)\n",
8954                             handle, phys_disk_num));
8955
8956         init_completion(&ioc->scsih_cmds.done);
8957         ioc->put_smid_default(ioc, smid);
8958         wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);
8959
8960         if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
8961                 mpt3sas_check_cmd_timeout(ioc,
8962                     ioc->scsih_cmds.status, mpi_request,
8963                     sizeof(Mpi2RaidActionRequest_t)/4, issue_reset);
8964                 rc = -EFAULT;
8965                 goto out;
8966         }
8967
8968         if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
8969
8970                 mpi_reply = ioc->scsih_cmds.reply;
8971                 ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
8972                 if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
8973                         log_info =  le32_to_cpu(mpi_reply->IOCLogInfo);
8974                 else
8975                         log_info = 0;
8976                 ioc_status &= MPI2_IOCSTATUS_MASK;
8977                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8978                         dewtprintk(ioc,
8979                                    ioc_info(ioc, "IR RAID_ACTION: failed: ioc_status(0x%04x), loginfo(0x%08x)!!!\n",
8980                                             ioc_status, log_info));
8981                         rc = -EFAULT;
8982                 } else
8983                         dewtprintk(ioc,
8984                                    ioc_info(ioc, "IR RAID_ACTION: completed successfully\n"));
8985         }
8986
8987  out:
8988         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8989         mutex_unlock(&ioc->scsih_cmds.mutex);
8990
8991         if (issue_reset)
8992                 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
8993         return rc;
8994 }
8995
8996 /**
8997  * _scsih_reprobe_lun - reprobing lun
8998  * @sdev: scsi device struct
8999  * @no_uld_attach: sdev->no_uld_attach flag setting
9000  *
9001  **/
9002 static void
9003 _scsih_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
9004 {
9005         sdev->no_uld_attach = no_uld_attach ? 1 : 0;
9006         sdev_printk(KERN_INFO, sdev, "%s raid component\n",
9007             sdev->no_uld_attach ? "hiding" : "exposing");
9008         WARN_ON(scsi_device_reprobe(sdev));
9009 }
9010
9011 /**
9012  * _scsih_sas_volume_add - add new volume
9013  * @ioc: per adapter object
9014  * @element: IR config element data
9015  * Context: user.
9016  */
9017 static void
9018 _scsih_sas_volume_add(struct MPT3SAS_ADAPTER *ioc,
9019         Mpi2EventIrConfigElement_t *element)
9020 {
9021         struct _raid_device *raid_device;
9022         unsigned long flags;
9023         u64 wwid;
9024         u16 handle = le16_to_cpu(element->VolDevHandle);
9025         int rc;
9026
9027         mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);
9028         if (!wwid) {
9029                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
9030                         __FILE__, __LINE__, __func__);
9031                 return;
9032         }
9033
9034         spin_lock_irqsave(&ioc->raid_device_lock, flags);
9035         raid_device = _scsih_raid_device_find_by_wwid(ioc, wwid);
9036         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9037
9038         if (raid_device)
9039                 return;
9040
9041         raid_device = kzalloc(sizeof(struct _raid_device), GFP_KERNEL);
9042         if (!raid_device) {
9043                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
9044                         __FILE__, __LINE__, __func__);
9045                 return;
9046         }
9047
9048         raid_device->id = ioc->sas_id++;
9049         raid_device->channel = RAID_CHANNEL;
9050         raid_device->handle = handle;
9051         raid_device->wwid = wwid;
9052         _scsih_raid_device_add(ioc, raid_device);
9053         if (!ioc->wait_for_discovery_to_complete) {
9054                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
9055                     raid_device->id, 0);
9056                 if (rc)
9057                         _scsih_raid_device_remove(ioc, raid_device);
9058         } else {
9059                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
9060                 _scsih_determine_boot_device(ioc, raid_device, 1);
9061                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9062         }
9063 }
9064
9065 /**
9066  * _scsih_sas_volume_delete - delete volume
9067  * @ioc: per adapter object
9068  * @handle: volume device handle
9069  * Context: user.
9070  */
9071 static void
9072 _scsih_sas_volume_delete(struct MPT3SAS_ADAPTER *ioc, u16 handle)
9073 {
9074         struct _raid_device *raid_device;
9075         unsigned long flags;
9076         struct MPT3SAS_TARGET *sas_target_priv_data;
9077         struct scsi_target *starget = NULL;
9078
9079         spin_lock_irqsave(&ioc->raid_device_lock, flags);
9080         raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9081         if (raid_device) {
9082                 if (raid_device->starget) {
9083                         starget = raid_device->starget;
9084                         sas_target_priv_data = starget->hostdata;
9085                         sas_target_priv_data->deleted = 1;
9086                 }
9087                 ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
9088                          raid_device->handle, (u64)raid_device->wwid);
9089                 list_del(&raid_device->list);
9090                 kfree(raid_device);
9091         }
9092         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9093         if (starget)
9094                 scsi_remove_target(&starget->dev);
9095 }
9096
9097 /**
9098  * _scsih_sas_pd_expose - expose pd component to /dev/sdX
9099  * @ioc: per adapter object
9100  * @element: IR config element data
9101  * Context: user.
9102  */
9103 static void
9104 _scsih_sas_pd_expose(struct MPT3SAS_ADAPTER *ioc,
9105         Mpi2EventIrConfigElement_t *element)
9106 {
9107         struct _sas_device *sas_device;
9108         struct scsi_target *starget = NULL;
9109         struct MPT3SAS_TARGET *sas_target_priv_data;
9110         unsigned long flags;
9111         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9112
9113         spin_lock_irqsave(&ioc->sas_device_lock, flags);
9114         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
9115         if (sas_device) {
9116                 sas_device->volume_handle = 0;
9117                 sas_device->volume_wwid = 0;
9118                 clear_bit(handle, ioc->pd_handles);
9119                 if (sas_device->starget && sas_device->starget->hostdata) {
9120                         starget = sas_device->starget;
9121                         sas_target_priv_data = starget->hostdata;
9122                         sas_target_priv_data->flags &=
9123                             ~MPT_TARGET_FLAGS_RAID_COMPONENT;
9124                 }
9125         }
9126         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
9127         if (!sas_device)
9128                 return;
9129
9130         /* exposing raid component */
9131         if (starget)
9132                 starget_for_each_device(starget, NULL, _scsih_reprobe_lun);
9133
9134         sas_device_put(sas_device);
9135 }
9136
9137 /**
9138  * _scsih_sas_pd_hide - hide pd component from /dev/sdX
9139  * @ioc: per adapter object
9140  * @element: IR config element data
9141  * Context: user.
9142  */
9143 static void
9144 _scsih_sas_pd_hide(struct MPT3SAS_ADAPTER *ioc,
9145         Mpi2EventIrConfigElement_t *element)
9146 {
9147         struct _sas_device *sas_device;
9148         struct scsi_target *starget = NULL;
9149         struct MPT3SAS_TARGET *sas_target_priv_data;
9150         unsigned long flags;
9151         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9152         u16 volume_handle = 0;
9153         u64 volume_wwid = 0;
9154
9155         mpt3sas_config_get_volume_handle(ioc, handle, &volume_handle);
9156         if (volume_handle)
9157                 mpt3sas_config_get_volume_wwid(ioc, volume_handle,
9158                     &volume_wwid);
9159
9160         spin_lock_irqsave(&ioc->sas_device_lock, flags);
9161         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
9162         if (sas_device) {
9163                 set_bit(handle, ioc->pd_handles);
9164                 if (sas_device->starget && sas_device->starget->hostdata) {
9165                         starget = sas_device->starget;
9166                         sas_target_priv_data = starget->hostdata;
9167                         sas_target_priv_data->flags |=
9168                             MPT_TARGET_FLAGS_RAID_COMPONENT;
9169                         sas_device->volume_handle = volume_handle;
9170                         sas_device->volume_wwid = volume_wwid;
9171                 }
9172         }
9173         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
9174         if (!sas_device)
9175                 return;
9176
9177         /* hiding raid component */
9178         _scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);
9179
9180         if (starget)
9181                 starget_for_each_device(starget, (void *)1, _scsih_reprobe_lun);
9182
9183         sas_device_put(sas_device);
9184 }
9185
9186 /**
9187  * _scsih_sas_pd_delete - delete pd component
9188  * @ioc: per adapter object
9189  * @element: IR config element data
9190  * Context: user.
9191  */
9192 static void
9193 _scsih_sas_pd_delete(struct MPT3SAS_ADAPTER *ioc,
9194         Mpi2EventIrConfigElement_t *element)
9195 {
9196         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9197
9198         _scsih_device_remove_by_handle(ioc, handle);
9199 }
9200
9201 /**
9202  * _scsih_sas_pd_add - remove pd component
9203  * @ioc: per adapter object
9204  * @element: IR config element data
9205  * Context: user.
9206  */
9207 static void
9208 _scsih_sas_pd_add(struct MPT3SAS_ADAPTER *ioc,
9209         Mpi2EventIrConfigElement_t *element)
9210 {
9211         struct _sas_device *sas_device;
9212         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9213         Mpi2ConfigReply_t mpi_reply;
9214         Mpi2SasDevicePage0_t sas_device_pg0;
9215         u32 ioc_status;
9216         u64 sas_address;
9217         u16 parent_handle;
9218
9219         set_bit(handle, ioc->pd_handles);
9220
9221         sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
9222         if (sas_device) {
9223                 _scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);
9224                 sas_device_put(sas_device);
9225                 return;
9226         }
9227
9228         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
9229             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
9230                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
9231                         __FILE__, __LINE__, __func__);
9232                 return;
9233         }
9234
9235         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9236             MPI2_IOCSTATUS_MASK;
9237         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9238                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
9239                         __FILE__, __LINE__, __func__);
9240                 return;
9241         }
9242
9243         parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
9244         if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))
9245                 mpt3sas_transport_update_links(ioc, sas_address, handle,
9246                     sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
9247                     mpt3sas_get_port_by_id(ioc,
9248                     sas_device_pg0.PhysicalPort, 0));
9249
9250         _scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);
9251         _scsih_add_device(ioc, handle, 0, 1);
9252 }
9253
9254 /**
9255  * _scsih_sas_ir_config_change_event_debug - debug for IR Config Change events
9256  * @ioc: per adapter object
9257  * @event_data: event data payload
9258  * Context: user.
9259  */
9260 static void
9261 _scsih_sas_ir_config_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
9262         Mpi2EventDataIrConfigChangeList_t *event_data)
9263 {
9264         Mpi2EventIrConfigElement_t *element;
9265         u8 element_type;
9266         int i;
9267         char *reason_str = NULL, *element_str = NULL;
9268
9269         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
9270
9271         ioc_info(ioc, "raid config change: (%s), elements(%d)\n",
9272                  le32_to_cpu(event_data->Flags) & MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG ?
9273                  "foreign" : "native",
9274                  event_data->NumElements);
9275         for (i = 0; i < event_data->NumElements; i++, element++) {
9276                 switch (element->ReasonCode) {
9277                 case MPI2_EVENT_IR_CHANGE_RC_ADDED:
9278                         reason_str = "add";
9279                         break;
9280                 case MPI2_EVENT_IR_CHANGE_RC_REMOVED:
9281                         reason_str = "remove";
9282                         break;
9283                 case MPI2_EVENT_IR_CHANGE_RC_NO_CHANGE:
9284                         reason_str = "no change";
9285                         break;
9286                 case MPI2_EVENT_IR_CHANGE_RC_HIDE:
9287                         reason_str = "hide";
9288                         break;
9289                 case MPI2_EVENT_IR_CHANGE_RC_UNHIDE:
9290                         reason_str = "unhide";
9291                         break;
9292                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED:
9293                         reason_str = "volume_created";
9294                         break;
9295                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED:
9296                         reason_str = "volume_deleted";
9297                         break;
9298                 case MPI2_EVENT_IR_CHANGE_RC_PD_CREATED:
9299                         reason_str = "pd_created";
9300                         break;
9301                 case MPI2_EVENT_IR_CHANGE_RC_PD_DELETED:
9302                         reason_str = "pd_deleted";
9303                         break;
9304                 default:
9305                         reason_str = "unknown reason";
9306                         break;
9307                 }
9308                 element_type = le16_to_cpu(element->ElementFlags) &
9309                     MPI2_EVENT_IR_CHANGE_EFLAGS_ELEMENT_TYPE_MASK;
9310                 switch (element_type) {
9311                 case MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT:
9312                         element_str = "volume";
9313                         break;
9314                 case MPI2_EVENT_IR_CHANGE_EFLAGS_VOLPHYSDISK_ELEMENT:
9315                         element_str = "phys disk";
9316                         break;
9317                 case MPI2_EVENT_IR_CHANGE_EFLAGS_HOTSPARE_ELEMENT:
9318                         element_str = "hot spare";
9319                         break;
9320                 default:
9321                         element_str = "unknown element";
9322                         break;
9323                 }
9324                 pr_info("\t(%s:%s), vol handle(0x%04x), " \
9325                     "pd handle(0x%04x), pd num(0x%02x)\n", element_str,
9326                     reason_str, le16_to_cpu(element->VolDevHandle),
9327                     le16_to_cpu(element->PhysDiskDevHandle),
9328                     element->PhysDiskNum);
9329         }
9330 }
9331
9332 /**
9333  * _scsih_sas_ir_config_change_event - handle ir configuration change events
9334  * @ioc: per adapter object
9335  * @fw_event: The fw_event_work object
9336  * Context: user.
9337  */
9338 static void
9339 _scsih_sas_ir_config_change_event(struct MPT3SAS_ADAPTER *ioc,
9340         struct fw_event_work *fw_event)
9341 {
9342         Mpi2EventIrConfigElement_t *element;
9343         int i;
9344         u8 foreign_config;
9345         Mpi2EventDataIrConfigChangeList_t *event_data =
9346                 (Mpi2EventDataIrConfigChangeList_t *)
9347                 fw_event->event_data;
9348
9349         if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&
9350              (!ioc->hide_ir_msg))
9351                 _scsih_sas_ir_config_change_event_debug(ioc, event_data);
9352
9353         foreign_config = (le32_to_cpu(event_data->Flags) &
9354             MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG) ? 1 : 0;
9355
9356         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
9357         if (ioc->shost_recovery &&
9358             ioc->hba_mpi_version_belonged != MPI2_VERSION) {
9359                 for (i = 0; i < event_data->NumElements; i++, element++) {
9360                         if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_HIDE)
9361                                 _scsih_ir_fastpath(ioc,
9362                                         le16_to_cpu(element->PhysDiskDevHandle),
9363                                         element->PhysDiskNum);
9364                 }
9365                 return;
9366         }
9367
9368         for (i = 0; i < event_data->NumElements; i++, element++) {
9369
9370                 switch (element->ReasonCode) {
9371                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED:
9372                 case MPI2_EVENT_IR_CHANGE_RC_ADDED:
9373                         if (!foreign_config)
9374                                 _scsih_sas_volume_add(ioc, element);
9375                         break;
9376                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED:
9377                 case MPI2_EVENT_IR_CHANGE_RC_REMOVED:
9378                         if (!foreign_config)
9379                                 _scsih_sas_volume_delete(ioc,
9380                                     le16_to_cpu(element->VolDevHandle));
9381                         break;
9382                 case MPI2_EVENT_IR_CHANGE_RC_PD_CREATED:
9383                         if (!ioc->is_warpdrive)
9384                                 _scsih_sas_pd_hide(ioc, element);
9385                         break;
9386                 case MPI2_EVENT_IR_CHANGE_RC_PD_DELETED:
9387                         if (!ioc->is_warpdrive)
9388                                 _scsih_sas_pd_expose(ioc, element);
9389                         break;
9390                 case MPI2_EVENT_IR_CHANGE_RC_HIDE:
9391                         if (!ioc->is_warpdrive)
9392                                 _scsih_sas_pd_add(ioc, element);
9393                         break;
9394                 case MPI2_EVENT_IR_CHANGE_RC_UNHIDE:
9395                         if (!ioc->is_warpdrive)
9396                                 _scsih_sas_pd_delete(ioc, element);
9397                         break;
9398                 }
9399         }
9400 }
9401
9402 /**
9403  * _scsih_sas_ir_volume_event - IR volume event
9404  * @ioc: per adapter object
9405  * @fw_event: The fw_event_work object
9406  * Context: user.
9407  */
9408 static void
9409 _scsih_sas_ir_volume_event(struct MPT3SAS_ADAPTER *ioc,
9410         struct fw_event_work *fw_event)
9411 {
9412         u64 wwid;
9413         unsigned long flags;
9414         struct _raid_device *raid_device;
9415         u16 handle;
9416         u32 state;
9417         int rc;
9418         Mpi2EventDataIrVolume_t *event_data =
9419                 (Mpi2EventDataIrVolume_t *) fw_event->event_data;
9420
9421         if (ioc->shost_recovery)
9422                 return;
9423
9424         if (event_data->ReasonCode != MPI2_EVENT_IR_VOLUME_RC_STATE_CHANGED)
9425                 return;
9426
9427         handle = le16_to_cpu(event_data->VolDevHandle);
9428         state = le32_to_cpu(event_data->NewValue);
9429         if (!ioc->hide_ir_msg)
9430                 dewtprintk(ioc,
9431                            ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",
9432                                     __func__, handle,
9433                                     le32_to_cpu(event_data->PreviousValue),
9434                                     state));
9435         switch (state) {
9436         case MPI2_RAID_VOL_STATE_MISSING:
9437         case MPI2_RAID_VOL_STATE_FAILED:
9438                 _scsih_sas_volume_delete(ioc, handle);
9439                 break;
9440
9441         case MPI2_RAID_VOL_STATE_ONLINE:
9442         case MPI2_RAID_VOL_STATE_DEGRADED:
9443         case MPI2_RAID_VOL_STATE_OPTIMAL:
9444
9445                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
9446                 raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9447                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9448
9449                 if (raid_device)
9450                         break;
9451
9452                 mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);
9453                 if (!wwid) {
9454                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9455                                 __FILE__, __LINE__, __func__);
9456                         break;
9457                 }
9458
9459                 raid_device = kzalloc(sizeof(struct _raid_device), GFP_KERNEL);
9460                 if (!raid_device) {
9461                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9462                                 __FILE__, __LINE__, __func__);
9463                         break;
9464                 }
9465
9466                 raid_device->id = ioc->sas_id++;
9467                 raid_device->channel = RAID_CHANNEL;
9468                 raid_device->handle = handle;
9469                 raid_device->wwid = wwid;
9470                 _scsih_raid_device_add(ioc, raid_device);
9471                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
9472                     raid_device->id, 0);
9473                 if (rc)
9474                         _scsih_raid_device_remove(ioc, raid_device);
9475                 break;
9476
9477         case MPI2_RAID_VOL_STATE_INITIALIZING:
9478         default:
9479                 break;
9480         }
9481 }
9482
9483 /**
9484  * _scsih_sas_ir_physical_disk_event - PD event
9485  * @ioc: per adapter object
9486  * @fw_event: The fw_event_work object
9487  * Context: user.
9488  */
9489 static void
9490 _scsih_sas_ir_physical_disk_event(struct MPT3SAS_ADAPTER *ioc,
9491         struct fw_event_work *fw_event)
9492 {
9493         u16 handle, parent_handle;
9494         u32 state;
9495         struct _sas_device *sas_device;
9496         Mpi2ConfigReply_t mpi_reply;
9497         Mpi2SasDevicePage0_t sas_device_pg0;
9498         u32 ioc_status;
9499         Mpi2EventDataIrPhysicalDisk_t *event_data =
9500                 (Mpi2EventDataIrPhysicalDisk_t *) fw_event->event_data;
9501         u64 sas_address;
9502
9503         if (ioc->shost_recovery)
9504                 return;
9505
9506         if (event_data->ReasonCode != MPI2_EVENT_IR_PHYSDISK_RC_STATE_CHANGED)
9507                 return;
9508
9509         handle = le16_to_cpu(event_data->PhysDiskDevHandle);
9510         state = le32_to_cpu(event_data->NewValue);
9511
9512         if (!ioc->hide_ir_msg)
9513                 dewtprintk(ioc,
9514                            ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",
9515                                     __func__, handle,
9516                                     le32_to_cpu(event_data->PreviousValue),
9517                                     state));
9518
9519         switch (state) {
9520         case MPI2_RAID_PD_STATE_ONLINE:
9521         case MPI2_RAID_PD_STATE_DEGRADED:
9522         case MPI2_RAID_PD_STATE_REBUILDING:
9523         case MPI2_RAID_PD_STATE_OPTIMAL:
9524         case MPI2_RAID_PD_STATE_HOT_SPARE:
9525
9526                 if (!ioc->is_warpdrive)
9527                         set_bit(handle, ioc->pd_handles);
9528
9529                 sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
9530                 if (sas_device) {
9531                         sas_device_put(sas_device);
9532                         return;
9533                 }
9534
9535                 if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
9536                     &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
9537                     handle))) {
9538                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9539                                 __FILE__, __LINE__, __func__);
9540                         return;
9541                 }
9542
9543                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9544                     MPI2_IOCSTATUS_MASK;
9545                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9546                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9547                                 __FILE__, __LINE__, __func__);
9548                         return;
9549                 }
9550
9551                 parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
9552                 if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))
9553                         mpt3sas_transport_update_links(ioc, sas_address, handle,
9554                             sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
9555                             mpt3sas_get_port_by_id(ioc,
9556                             sas_device_pg0.PhysicalPort, 0));
9557
9558                 _scsih_add_device(ioc, handle, 0, 1);
9559
9560                 break;
9561
9562         case MPI2_RAID_PD_STATE_OFFLINE:
9563         case MPI2_RAID_PD_STATE_NOT_CONFIGURED:
9564         case MPI2_RAID_PD_STATE_NOT_COMPATIBLE:
9565         default:
9566                 break;
9567         }
9568 }
9569
9570 /**
9571  * _scsih_sas_ir_operation_status_event_debug - debug for IR op event
9572  * @ioc: per adapter object
9573  * @event_data: event data payload
9574  * Context: user.
9575  */
9576 static void
9577 _scsih_sas_ir_operation_status_event_debug(struct MPT3SAS_ADAPTER *ioc,
9578         Mpi2EventDataIrOperationStatus_t *event_data)
9579 {
9580         char *reason_str = NULL;
9581
9582         switch (event_data->RAIDOperation) {
9583         case MPI2_EVENT_IR_RAIDOP_RESYNC:
9584                 reason_str = "resync";
9585                 break;
9586         case MPI2_EVENT_IR_RAIDOP_ONLINE_CAP_EXPANSION:
9587                 reason_str = "online capacity expansion";
9588                 break;
9589         case MPI2_EVENT_IR_RAIDOP_CONSISTENCY_CHECK:
9590                 reason_str = "consistency check";
9591                 break;
9592         case MPI2_EVENT_IR_RAIDOP_BACKGROUND_INIT:
9593                 reason_str = "background init";
9594                 break;
9595         case MPI2_EVENT_IR_RAIDOP_MAKE_DATA_CONSISTENT:
9596                 reason_str = "make data consistent";
9597                 break;
9598         }
9599
9600         if (!reason_str)
9601                 return;
9602
9603         ioc_info(ioc, "raid operational status: (%s)\thandle(0x%04x), percent complete(%d)\n",
9604                  reason_str,
9605                  le16_to_cpu(event_data->VolDevHandle),
9606                  event_data->PercentComplete);
9607 }
9608
9609 /**
9610  * _scsih_sas_ir_operation_status_event - handle RAID operation events
9611  * @ioc: per adapter object
9612  * @fw_event: The fw_event_work object
9613  * Context: user.
9614  */
9615 static void
9616 _scsih_sas_ir_operation_status_event(struct MPT3SAS_ADAPTER *ioc,
9617         struct fw_event_work *fw_event)
9618 {
9619         Mpi2EventDataIrOperationStatus_t *event_data =
9620                 (Mpi2EventDataIrOperationStatus_t *)
9621                 fw_event->event_data;
9622         static struct _raid_device *raid_device;
9623         unsigned long flags;
9624         u16 handle;
9625
9626         if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&
9627             (!ioc->hide_ir_msg))
9628                 _scsih_sas_ir_operation_status_event_debug(ioc,
9629                      event_data);
9630
9631         /* code added for raid transport support */
9632         if (event_data->RAIDOperation == MPI2_EVENT_IR_RAIDOP_RESYNC) {
9633
9634                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
9635                 handle = le16_to_cpu(event_data->VolDevHandle);
9636                 raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9637                 if (raid_device)
9638                         raid_device->percent_complete =
9639                             event_data->PercentComplete;
9640                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9641         }
9642 }
9643
9644 /**
9645  * _scsih_prep_device_scan - initialize parameters prior to device scan
9646  * @ioc: per adapter object
9647  *
9648  * Set the deleted flag prior to device scan.  If the device is found during
9649  * the scan, then we clear the deleted flag.
9650  */
9651 static void
9652 _scsih_prep_device_scan(struct MPT3SAS_ADAPTER *ioc)
9653 {
9654         struct MPT3SAS_DEVICE *sas_device_priv_data;
9655         struct scsi_device *sdev;
9656
9657         shost_for_each_device(sdev, ioc->shost) {
9658                 sas_device_priv_data = sdev->hostdata;
9659                 if (sas_device_priv_data && sas_device_priv_data->sas_target)
9660                         sas_device_priv_data->sas_target->deleted = 1;
9661         }
9662 }
9663
9664 /**
9665  * _scsih_update_device_qdepth - Update QD during Reset.
9666  * @ioc: per adapter object
9667  *
9668  */
9669 static void
9670 _scsih_update_device_qdepth(struct MPT3SAS_ADAPTER *ioc)
9671 {
9672         struct MPT3SAS_DEVICE *sas_device_priv_data;
9673         struct MPT3SAS_TARGET *sas_target_priv_data;
9674         struct _sas_device *sas_device;
9675         struct scsi_device *sdev;
9676         u16 qdepth;
9677
9678         ioc_info(ioc, "Update devices with firmware reported queue depth\n");
9679         shost_for_each_device(sdev, ioc->shost) {
9680                 sas_device_priv_data = sdev->hostdata;
9681                 if (sas_device_priv_data && sas_device_priv_data->sas_target) {
9682                         sas_target_priv_data = sas_device_priv_data->sas_target;
9683                         sas_device = sas_device_priv_data->sas_target->sas_dev;
9684                         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE)
9685                                 qdepth = ioc->max_nvme_qd;
9686                         else if (sas_device &&
9687                             sas_device->device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET)
9688                                 qdepth = (sas_device->port_type > 1) ?
9689                                     ioc->max_wideport_qd : ioc->max_narrowport_qd;
9690                         else if (sas_device &&
9691                             sas_device->device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
9692                                 qdepth = ioc->max_sata_qd;
9693                         else
9694                                 continue;
9695                         mpt3sas_scsih_change_queue_depth(sdev, qdepth);
9696                 }
9697         }
9698 }
9699
9700 /**
9701  * _scsih_mark_responding_sas_device - mark a sas_devices as responding
9702  * @ioc: per adapter object
9703  * @sas_device_pg0: SAS Device page 0
9704  *
9705  * After host reset, find out whether devices are still responding.
9706  * Used in _scsih_remove_unresponsive_sas_devices.
9707  */
9708 static void
9709 _scsih_mark_responding_sas_device(struct MPT3SAS_ADAPTER *ioc,
9710 Mpi2SasDevicePage0_t *sas_device_pg0)
9711 {
9712         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9713         struct scsi_target *starget;
9714         struct _sas_device *sas_device = NULL;
9715         struct _enclosure_node *enclosure_dev = NULL;
9716         unsigned long flags;
9717         struct hba_port *port = mpt3sas_get_port_by_id(
9718             ioc, sas_device_pg0->PhysicalPort, 0);
9719
9720         if (sas_device_pg0->EnclosureHandle) {
9721                 enclosure_dev =
9722                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
9723                                 le16_to_cpu(sas_device_pg0->EnclosureHandle));
9724                 if (enclosure_dev == NULL)
9725                         ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",
9726                                  sas_device_pg0->EnclosureHandle);
9727         }
9728         spin_lock_irqsave(&ioc->sas_device_lock, flags);
9729         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
9730                 if (sas_device->sas_address != le64_to_cpu(
9731                     sas_device_pg0->SASAddress))
9732                         continue;
9733                 if (sas_device->slot != le16_to_cpu(sas_device_pg0->Slot))
9734                         continue;
9735                 if (sas_device->port != port)
9736                         continue;
9737                 sas_device->responding = 1;
9738                 starget = sas_device->starget;
9739                 if (starget && starget->hostdata) {
9740                         sas_target_priv_data = starget->hostdata;
9741                         sas_target_priv_data->tm_busy = 0;
9742                         sas_target_priv_data->deleted = 0;
9743                 } else
9744                         sas_target_priv_data = NULL;
9745                 if (starget) {
9746                         starget_printk(KERN_INFO, starget,
9747                             "handle(0x%04x), sas_addr(0x%016llx)\n",
9748                             le16_to_cpu(sas_device_pg0->DevHandle),
9749                             (unsigned long long)
9750                             sas_device->sas_address);
9751
9752                         if (sas_device->enclosure_handle != 0)
9753                                 starget_printk(KERN_INFO, starget,
9754                                  "enclosure logical id(0x%016llx), slot(%d)\n",
9755                                  (unsigned long long)
9756                                  sas_device->enclosure_logical_id,
9757                                  sas_device->slot);
9758                 }
9759                 if (le16_to_cpu(sas_device_pg0->Flags) &
9760                       MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
9761                         sas_device->enclosure_level =
9762                            sas_device_pg0->EnclosureLevel;
9763                         memcpy(&sas_device->connector_name[0],
9764                                 &sas_device_pg0->ConnectorName[0], 4);
9765                 } else {
9766                         sas_device->enclosure_level = 0;
9767                         sas_device->connector_name[0] = '\0';
9768                 }
9769
9770                 sas_device->enclosure_handle =
9771                         le16_to_cpu(sas_device_pg0->EnclosureHandle);
9772                 sas_device->is_chassis_slot_valid = 0;
9773                 if (enclosure_dev) {
9774                         sas_device->enclosure_logical_id = le64_to_cpu(
9775                                 enclosure_dev->pg0.EnclosureLogicalID);
9776                         if (le16_to_cpu(enclosure_dev->pg0.Flags) &
9777                             MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
9778                                 sas_device->is_chassis_slot_valid = 1;
9779                                 sas_device->chassis_slot =
9780                                         enclosure_dev->pg0.ChassisSlot;
9781                         }
9782                 }
9783
9784                 if (sas_device->handle == le16_to_cpu(
9785                     sas_device_pg0->DevHandle))
9786                         goto out;
9787                 pr_info("\thandle changed from(0x%04x)!!!\n",
9788                     sas_device->handle);
9789                 sas_device->handle = le16_to_cpu(
9790                     sas_device_pg0->DevHandle);
9791                 if (sas_target_priv_data)
9792                         sas_target_priv_data->handle =
9793                             le16_to_cpu(sas_device_pg0->DevHandle);
9794                 goto out;
9795         }
9796  out:
9797         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
9798 }
9799
9800 /**
9801  * _scsih_create_enclosure_list_after_reset - Free Existing list,
9802  *      And create enclosure list by scanning all Enclosure Page(0)s
9803  * @ioc: per adapter object
9804  */
9805 static void
9806 _scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)
9807 {
9808         struct _enclosure_node *enclosure_dev;
9809         Mpi2ConfigReply_t mpi_reply;
9810         u16 enclosure_handle;
9811         int rc;
9812
9813         /* Free existing enclosure list */
9814         mpt3sas_free_enclosure_list(ioc);
9815
9816         /* Re constructing enclosure list after reset*/
9817         enclosure_handle = 0xFFFF;
9818         do {
9819                 enclosure_dev =
9820                         kzalloc(sizeof(struct _enclosure_node), GFP_KERNEL);
9821                 if (!enclosure_dev) {
9822                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9823                                 __FILE__, __LINE__, __func__);
9824                         return;
9825                 }
9826                 rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
9827                                 &enclosure_dev->pg0,
9828                                 MPI2_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE,
9829                                 enclosure_handle);
9830
9831                 if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
9832                                                 MPI2_IOCSTATUS_MASK)) {
9833                         kfree(enclosure_dev);
9834                         return;
9835                 }
9836                 list_add_tail(&enclosure_dev->list,
9837                                                 &ioc->enclosure_list);
9838                 enclosure_handle =
9839                         le16_to_cpu(enclosure_dev->pg0.EnclosureHandle);
9840         } while (1);
9841 }
9842
9843 /**
9844  * _scsih_search_responding_sas_devices -
9845  * @ioc: per adapter object
9846  *
9847  * After host reset, find out whether devices are still responding.
9848  * If not remove.
9849  */
9850 static void
9851 _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)
9852 {
9853         Mpi2SasDevicePage0_t sas_device_pg0;
9854         Mpi2ConfigReply_t mpi_reply;
9855         u16 ioc_status;
9856         u16 handle;
9857         u32 device_info;
9858
9859         ioc_info(ioc, "search for end-devices: start\n");
9860
9861         if (list_empty(&ioc->sas_device_list))
9862                 goto out;
9863
9864         handle = 0xFFFF;
9865         while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
9866             &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
9867             handle))) {
9868                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9869                     MPI2_IOCSTATUS_MASK;
9870                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
9871                         break;
9872                 handle = le16_to_cpu(sas_device_pg0.DevHandle);
9873                 device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
9874                 if (!(_scsih_is_end_device(device_info)))
9875                         continue;
9876                 _scsih_mark_responding_sas_device(ioc, &sas_device_pg0);
9877         }
9878
9879  out:
9880         ioc_info(ioc, "search for end-devices: complete\n");
9881 }
9882
9883 /**
9884  * _scsih_mark_responding_pcie_device - mark a pcie_device as responding
9885  * @ioc: per adapter object
9886  * @pcie_device_pg0: PCIe Device page 0
9887  *
9888  * After host reset, find out whether devices are still responding.
9889  * Used in _scsih_remove_unresponding_devices.
9890  */
9891 static void
9892 _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
9893         Mpi26PCIeDevicePage0_t *pcie_device_pg0)
9894 {
9895         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9896         struct scsi_target *starget;
9897         struct _pcie_device *pcie_device;
9898         unsigned long flags;
9899
9900         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
9901         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
9902                 if ((pcie_device->wwid == le64_to_cpu(pcie_device_pg0->WWID))
9903                     && (pcie_device->slot == le16_to_cpu(
9904                     pcie_device_pg0->Slot))) {
9905                         pcie_device->access_status =
9906                                         pcie_device_pg0->AccessStatus;
9907                         pcie_device->responding = 1;
9908                         starget = pcie_device->starget;
9909                         if (starget && starget->hostdata) {
9910                                 sas_target_priv_data = starget->hostdata;
9911                                 sas_target_priv_data->tm_busy = 0;
9912                                 sas_target_priv_data->deleted = 0;
9913                         } else
9914                                 sas_target_priv_data = NULL;
9915                         if (starget) {
9916                                 starget_printk(KERN_INFO, starget,
9917                                     "handle(0x%04x), wwid(0x%016llx) ",
9918                                     pcie_device->handle,
9919                                     (unsigned long long)pcie_device->wwid);
9920                                 if (pcie_device->enclosure_handle != 0)
9921                                         starget_printk(KERN_INFO, starget,
9922                                             "enclosure logical id(0x%016llx), "
9923                                             "slot(%d)\n",
9924                                             (unsigned long long)
9925                                             pcie_device->enclosure_logical_id,
9926                                             pcie_device->slot);
9927                         }
9928
9929                         if (((le32_to_cpu(pcie_device_pg0->Flags)) &
9930                             MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) &&
9931                             (ioc->hba_mpi_version_belonged != MPI2_VERSION)) {
9932                                 pcie_device->enclosure_level =
9933                                     pcie_device_pg0->EnclosureLevel;
9934                                 memcpy(&pcie_device->connector_name[0],
9935                                     &pcie_device_pg0->ConnectorName[0], 4);
9936                         } else {
9937                                 pcie_device->enclosure_level = 0;
9938                                 pcie_device->connector_name[0] = '\0';
9939                         }
9940
9941                         if (pcie_device->handle == le16_to_cpu(
9942                             pcie_device_pg0->DevHandle))
9943                                 goto out;
9944                         pr_info("\thandle changed from(0x%04x)!!!\n",
9945                             pcie_device->handle);
9946                         pcie_device->handle = le16_to_cpu(
9947                             pcie_device_pg0->DevHandle);
9948                         if (sas_target_priv_data)
9949                                 sas_target_priv_data->handle =
9950                                     le16_to_cpu(pcie_device_pg0->DevHandle);
9951                         goto out;
9952                 }
9953         }
9954
9955  out:
9956         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
9957 }
9958
9959 /**
9960  * _scsih_search_responding_pcie_devices -
9961  * @ioc: per adapter object
9962  *
9963  * After host reset, find out whether devices are still responding.
9964  * If not remove.
9965  */
9966 static void
9967 _scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)
9968 {
9969         Mpi26PCIeDevicePage0_t pcie_device_pg0;
9970         Mpi2ConfigReply_t mpi_reply;
9971         u16 ioc_status;
9972         u16 handle;
9973         u32 device_info;
9974
9975         ioc_info(ioc, "search for end-devices: start\n");
9976
9977         if (list_empty(&ioc->pcie_device_list))
9978                 goto out;
9979
9980         handle = 0xFFFF;
9981         while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
9982                 &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
9983                 handle))) {
9984                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9985                     MPI2_IOCSTATUS_MASK;
9986                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9987                         ioc_info(ioc, "\tbreak from %s: ioc_status(0x%04x), loginfo(0x%08x)\n",
9988                                  __func__, ioc_status,
9989                                  le32_to_cpu(mpi_reply.IOCLogInfo));
9990                         break;
9991                 }
9992                 handle = le16_to_cpu(pcie_device_pg0.DevHandle);
9993                 device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
9994                 if (!(_scsih_is_nvme_pciescsi_device(device_info)))
9995                         continue;
9996                 _scsih_mark_responding_pcie_device(ioc, &pcie_device_pg0);
9997         }
9998 out:
9999         ioc_info(ioc, "search for PCIe end-devices: complete\n");
10000 }
10001
10002 /**
10003  * _scsih_mark_responding_raid_device - mark a raid_device as responding
10004  * @ioc: per adapter object
10005  * @wwid: world wide identifier for raid volume
10006  * @handle: device handle
10007  *
10008  * After host reset, find out whether devices are still responding.
10009  * Used in _scsih_remove_unresponsive_raid_devices.
10010  */
10011 static void
10012 _scsih_mark_responding_raid_device(struct MPT3SAS_ADAPTER *ioc, u64 wwid,
10013         u16 handle)
10014 {
10015         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
10016         struct scsi_target *starget;
10017         struct _raid_device *raid_device;
10018         unsigned long flags;
10019
10020         spin_lock_irqsave(&ioc->raid_device_lock, flags);
10021         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
10022                 if (raid_device->wwid == wwid && raid_device->starget) {
10023                         starget = raid_device->starget;
10024                         if (starget && starget->hostdata) {
10025                                 sas_target_priv_data = starget->hostdata;
10026                                 sas_target_priv_data->deleted = 0;
10027                         } else
10028                                 sas_target_priv_data = NULL;
10029                         raid_device->responding = 1;
10030                         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
10031                         starget_printk(KERN_INFO, raid_device->starget,
10032                             "handle(0x%04x), wwid(0x%016llx)\n", handle,
10033                             (unsigned long long)raid_device->wwid);
10034
10035                         /*
10036                          * WARPDRIVE: The handles of the PDs might have changed
10037                          * across the host reset so re-initialize the
10038                          * required data for Direct IO
10039                          */
10040                         mpt3sas_init_warpdrive_properties(ioc, raid_device);
10041                         spin_lock_irqsave(&ioc->raid_device_lock, flags);
10042                         if (raid_device->handle == handle) {
10043                                 spin_unlock_irqrestore(&ioc->raid_device_lock,
10044                                     flags);
10045                                 return;
10046                         }
10047                         pr_info("\thandle changed from(0x%04x)!!!\n",
10048                             raid_device->handle);
10049                         raid_device->handle = handle;
10050                         if (sas_target_priv_data)
10051                                 sas_target_priv_data->handle = handle;
10052                         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
10053                         return;
10054                 }
10055         }
10056         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
10057 }
10058
10059 /**
10060  * _scsih_search_responding_raid_devices -
10061  * @ioc: per adapter object
10062  *
10063  * After host reset, find out whether devices are still responding.
10064  * If not remove.
10065  */
10066 static void
10067 _scsih_search_responding_raid_devices(struct MPT3SAS_ADAPTER *ioc)
10068 {
10069         Mpi2RaidVolPage1_t volume_pg1;
10070         Mpi2RaidVolPage0_t volume_pg0;
10071         Mpi2RaidPhysDiskPage0_t pd_pg0;
10072         Mpi2ConfigReply_t mpi_reply;
10073         u16 ioc_status;
10074         u16 handle;
10075         u8 phys_disk_num;
10076
10077         if (!ioc->ir_firmware)
10078                 return;
10079
10080         ioc_info(ioc, "search for raid volumes: start\n");
10081
10082         if (list_empty(&ioc->raid_device_list))
10083                 goto out;
10084
10085         handle = 0xFFFF;
10086         while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,
10087             &volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
10088                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10089                     MPI2_IOCSTATUS_MASK;
10090                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10091                         break;
10092                 handle = le16_to_cpu(volume_pg1.DevHandle);
10093
10094                 if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,
10095                     &volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
10096                      sizeof(Mpi2RaidVolPage0_t)))
10097                         continue;
10098
10099                 if (volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
10100                     volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
10101                     volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_DEGRADED)
10102                         _scsih_mark_responding_raid_device(ioc,
10103                             le64_to_cpu(volume_pg1.WWID), handle);
10104         }
10105
10106         /* refresh the pd_handles */
10107         if (!ioc->is_warpdrive) {
10108                 phys_disk_num = 0xFF;
10109                 memset(ioc->pd_handles, 0, ioc->pd_handles_sz);
10110                 while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
10111                     &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_GET_NEXT_PHYSDISKNUM,
10112                     phys_disk_num))) {
10113                         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10114                             MPI2_IOCSTATUS_MASK;
10115                         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10116                                 break;
10117                         phys_disk_num = pd_pg0.PhysDiskNum;
10118                         handle = le16_to_cpu(pd_pg0.DevHandle);
10119                         set_bit(handle, ioc->pd_handles);
10120                 }
10121         }
10122  out:
10123         ioc_info(ioc, "search for responding raid volumes: complete\n");
10124 }
10125
10126 /**
10127  * _scsih_mark_responding_expander - mark a expander as responding
10128  * @ioc: per adapter object
10129  * @expander_pg0:SAS Expander Config Page0
10130  *
10131  * After host reset, find out whether devices are still responding.
10132  * Used in _scsih_remove_unresponsive_expanders.
10133  */
10134 static void
10135 _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
10136         Mpi2ExpanderPage0_t *expander_pg0)
10137 {
10138         struct _sas_node *sas_expander = NULL;
10139         unsigned long flags;
10140         int i;
10141         struct _enclosure_node *enclosure_dev = NULL;
10142         u16 handle = le16_to_cpu(expander_pg0->DevHandle);
10143         u16 enclosure_handle = le16_to_cpu(expander_pg0->EnclosureHandle);
10144         u64 sas_address = le64_to_cpu(expander_pg0->SASAddress);
10145         struct hba_port *port = mpt3sas_get_port_by_id(
10146             ioc, expander_pg0->PhysicalPort, 0);
10147
10148         if (enclosure_handle)
10149                 enclosure_dev =
10150                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
10151                                                         enclosure_handle);
10152
10153         spin_lock_irqsave(&ioc->sas_node_lock, flags);
10154         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
10155                 if (sas_expander->sas_address != sas_address)
10156                         continue;
10157                 if (sas_expander->port != port)
10158                         continue;
10159                 sas_expander->responding = 1;
10160
10161                 if (enclosure_dev) {
10162                         sas_expander->enclosure_logical_id =
10163                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
10164                         sas_expander->enclosure_handle =
10165                             le16_to_cpu(expander_pg0->EnclosureHandle);
10166                 }
10167
10168                 if (sas_expander->handle == handle)
10169                         goto out;
10170                 pr_info("\texpander(0x%016llx): handle changed" \
10171                     " from(0x%04x) to (0x%04x)!!!\n",
10172                     (unsigned long long)sas_expander->sas_address,
10173                     sas_expander->handle, handle);
10174                 sas_expander->handle = handle;
10175                 for (i = 0 ; i < sas_expander->num_phys ; i++)
10176                         sas_expander->phy[i].handle = handle;
10177                 goto out;
10178         }
10179  out:
10180         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
10181 }
10182
10183 /**
10184  * _scsih_search_responding_expanders -
10185  * @ioc: per adapter object
10186  *
10187  * After host reset, find out whether devices are still responding.
10188  * If not remove.
10189  */
10190 static void
10191 _scsih_search_responding_expanders(struct MPT3SAS_ADAPTER *ioc)
10192 {
10193         Mpi2ExpanderPage0_t expander_pg0;
10194         Mpi2ConfigReply_t mpi_reply;
10195         u16 ioc_status;
10196         u64 sas_address;
10197         u16 handle;
10198         u8 port;
10199
10200         ioc_info(ioc, "search for expanders: start\n");
10201
10202         if (list_empty(&ioc->sas_expander_list))
10203                 goto out;
10204
10205         handle = 0xFFFF;
10206         while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
10207             MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL, handle))) {
10208
10209                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10210                     MPI2_IOCSTATUS_MASK;
10211                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10212                         break;
10213
10214                 handle = le16_to_cpu(expander_pg0.DevHandle);
10215                 sas_address = le64_to_cpu(expander_pg0.SASAddress);
10216                 port = expander_pg0.PhysicalPort;
10217                 pr_info(
10218                     "\texpander present: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
10219                     handle, (unsigned long long)sas_address,
10220                     (ioc->multipath_on_hba ?
10221                     port : MULTIPATH_DISABLED_PORT_ID));
10222                 _scsih_mark_responding_expander(ioc, &expander_pg0);
10223         }
10224
10225  out:
10226         ioc_info(ioc, "search for expanders: complete\n");
10227 }
10228
10229 /**
10230  * _scsih_remove_unresponding_devices - removing unresponding devices
10231  * @ioc: per adapter object
10232  */
10233 static void
10234 _scsih_remove_unresponding_devices(struct MPT3SAS_ADAPTER *ioc)
10235 {
10236         struct _sas_device *sas_device, *sas_device_next;
10237         struct _sas_node *sas_expander, *sas_expander_next;
10238         struct _raid_device *raid_device, *raid_device_next;
10239         struct _pcie_device *pcie_device, *pcie_device_next;
10240         struct list_head tmp_list;
10241         unsigned long flags;
10242         LIST_HEAD(head);
10243
10244         ioc_info(ioc, "removing unresponding devices: start\n");
10245
10246         /* removing unresponding end devices */
10247         ioc_info(ioc, "removing unresponding devices: end-devices\n");
10248         /*
10249          * Iterate, pulling off devices marked as non-responding. We become the
10250          * owner for the reference the list had on any object we prune.
10251          */
10252         spin_lock_irqsave(&ioc->sas_device_lock, flags);
10253
10254         /*
10255          * Clean up the sas_device_init_list list as
10256          * driver goes for fresh scan as part of diag reset.
10257          */
10258         list_for_each_entry_safe(sas_device, sas_device_next,
10259             &ioc->sas_device_init_list, list) {
10260                 list_del_init(&sas_device->list);
10261                 sas_device_put(sas_device);
10262         }
10263
10264         list_for_each_entry_safe(sas_device, sas_device_next,
10265             &ioc->sas_device_list, list) {
10266                 if (!sas_device->responding)
10267                         list_move_tail(&sas_device->list, &head);
10268                 else
10269                         sas_device->responding = 0;
10270         }
10271         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
10272
10273         /*
10274          * Now, uninitialize and remove the unresponding devices we pruned.
10275          */
10276         list_for_each_entry_safe(sas_device, sas_device_next, &head, list) {
10277                 _scsih_remove_device(ioc, sas_device);
10278                 list_del_init(&sas_device->list);
10279                 sas_device_put(sas_device);
10280         }
10281
10282         ioc_info(ioc, "Removing unresponding devices: pcie end-devices\n");
10283         INIT_LIST_HEAD(&head);
10284         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
10285         /*
10286          * Clean up the pcie_device_init_list list as
10287          * driver goes for fresh scan as part of diag reset.
10288          */
10289         list_for_each_entry_safe(pcie_device, pcie_device_next,
10290             &ioc->pcie_device_init_list, list) {
10291                 list_del_init(&pcie_device->list);
10292                 pcie_device_put(pcie_device);
10293         }
10294
10295         list_for_each_entry_safe(pcie_device, pcie_device_next,
10296             &ioc->pcie_device_list, list) {
10297                 if (!pcie_device->responding)
10298                         list_move_tail(&pcie_device->list, &head);
10299                 else
10300                         pcie_device->responding = 0;
10301         }
10302         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
10303
10304         list_for_each_entry_safe(pcie_device, pcie_device_next, &head, list) {
10305                 _scsih_pcie_device_remove_from_sml(ioc, pcie_device);
10306                 list_del_init(&pcie_device->list);
10307                 pcie_device_put(pcie_device);
10308         }
10309
10310         /* removing unresponding volumes */
10311         if (ioc->ir_firmware) {
10312                 ioc_info(ioc, "removing unresponding devices: volumes\n");
10313                 list_for_each_entry_safe(raid_device, raid_device_next,
10314                     &ioc->raid_device_list, list) {
10315                         if (!raid_device->responding)
10316                                 _scsih_sas_volume_delete(ioc,
10317                                     raid_device->handle);
10318                         else
10319                                 raid_device->responding = 0;
10320                 }
10321         }
10322
10323         /* removing unresponding expanders */
10324         ioc_info(ioc, "removing unresponding devices: expanders\n");
10325         spin_lock_irqsave(&ioc->sas_node_lock, flags);
10326         INIT_LIST_HEAD(&tmp_list);
10327         list_for_each_entry_safe(sas_expander, sas_expander_next,
10328             &ioc->sas_expander_list, list) {
10329                 if (!sas_expander->responding)
10330                         list_move_tail(&sas_expander->list, &tmp_list);
10331                 else
10332                         sas_expander->responding = 0;
10333         }
10334         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
10335         list_for_each_entry_safe(sas_expander, sas_expander_next, &tmp_list,
10336             list) {
10337                 _scsih_expander_node_remove(ioc, sas_expander);
10338         }
10339
10340         ioc_info(ioc, "removing unresponding devices: complete\n");
10341
10342         /* unblock devices */
10343         _scsih_ublock_io_all_device(ioc);
10344 }
10345
10346 static void
10347 _scsih_refresh_expander_links(struct MPT3SAS_ADAPTER *ioc,
10348         struct _sas_node *sas_expander, u16 handle)
10349 {
10350         Mpi2ExpanderPage1_t expander_pg1;
10351         Mpi2ConfigReply_t mpi_reply;
10352         int i;
10353
10354         for (i = 0 ; i < sas_expander->num_phys ; i++) {
10355                 if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,
10356                     &expander_pg1, i, handle))) {
10357                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
10358                                 __FILE__, __LINE__, __func__);
10359                         return;
10360                 }
10361
10362                 mpt3sas_transport_update_links(ioc, sas_expander->sas_address,
10363                     le16_to_cpu(expander_pg1.AttachedDevHandle), i,
10364                     expander_pg1.NegotiatedLinkRate >> 4,
10365                     sas_expander->port);
10366         }
10367 }
10368
10369 /**
10370  * _scsih_scan_for_devices_after_reset - scan for devices after host reset
10371  * @ioc: per adapter object
10372  */
10373 static void
10374 _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
10375 {
10376         Mpi2ExpanderPage0_t expander_pg0;
10377         Mpi2SasDevicePage0_t sas_device_pg0;
10378         Mpi26PCIeDevicePage0_t pcie_device_pg0;
10379         Mpi2RaidVolPage1_t *volume_pg1;
10380         Mpi2RaidVolPage0_t *volume_pg0;
10381         Mpi2RaidPhysDiskPage0_t pd_pg0;
10382         Mpi2EventIrConfigElement_t element;
10383         Mpi2ConfigReply_t mpi_reply;
10384         u8 phys_disk_num, port_id;
10385         u16 ioc_status;
10386         u16 handle, parent_handle;
10387         u64 sas_address;
10388         struct _sas_device *sas_device;
10389         struct _pcie_device *pcie_device;
10390         struct _sas_node *expander_device;
10391         static struct _raid_device *raid_device;
10392         u8 retry_count;
10393         unsigned long flags;
10394
10395         volume_pg0 = kzalloc(sizeof(*volume_pg0), GFP_KERNEL);
10396         if (!volume_pg0)
10397                 return;
10398
10399         volume_pg1 = kzalloc(sizeof(*volume_pg1), GFP_KERNEL);
10400         if (!volume_pg1) {
10401                 kfree(volume_pg0);
10402                 return;
10403         }
10404
10405         ioc_info(ioc, "scan devices: start\n");
10406
10407         _scsih_sas_host_refresh(ioc);
10408
10409         ioc_info(ioc, "\tscan devices: expanders start\n");
10410
10411         /* expanders */
10412         handle = 0xFFFF;
10413         while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
10414             MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL, handle))) {
10415                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10416                     MPI2_IOCSTATUS_MASK;
10417                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10418                         ioc_info(ioc, "\tbreak from expander scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10419                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10420                         break;
10421                 }
10422                 handle = le16_to_cpu(expander_pg0.DevHandle);
10423                 spin_lock_irqsave(&ioc->sas_node_lock, flags);
10424                 port_id = expander_pg0.PhysicalPort;
10425                 expander_device = mpt3sas_scsih_expander_find_by_sas_address(
10426                     ioc, le64_to_cpu(expander_pg0.SASAddress),
10427                     mpt3sas_get_port_by_id(ioc, port_id, 0));
10428                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
10429                 if (expander_device)
10430                         _scsih_refresh_expander_links(ioc, expander_device,
10431                             handle);
10432                 else {
10433                         ioc_info(ioc, "\tBEFORE adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",
10434                                  handle,
10435                                  (u64)le64_to_cpu(expander_pg0.SASAddress));
10436                         _scsih_expander_add(ioc, handle);
10437                         ioc_info(ioc, "\tAFTER adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",
10438                                  handle,
10439                                  (u64)le64_to_cpu(expander_pg0.SASAddress));
10440                 }
10441         }
10442
10443         ioc_info(ioc, "\tscan devices: expanders complete\n");
10444
10445         if (!ioc->ir_firmware)
10446                 goto skip_to_sas;
10447
10448         ioc_info(ioc, "\tscan devices: phys disk start\n");
10449
10450         /* phys disk */
10451         phys_disk_num = 0xFF;
10452         while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
10453             &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_GET_NEXT_PHYSDISKNUM,
10454             phys_disk_num))) {
10455                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10456                     MPI2_IOCSTATUS_MASK;
10457                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10458                         ioc_info(ioc, "\tbreak from phys disk scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10459                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10460                         break;
10461                 }
10462                 phys_disk_num = pd_pg0.PhysDiskNum;
10463                 handle = le16_to_cpu(pd_pg0.DevHandle);
10464                 sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
10465                 if (sas_device) {
10466                         sas_device_put(sas_device);
10467                         continue;
10468                 }
10469                 if (mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
10470                     &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
10471                     handle) != 0)
10472                         continue;
10473                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10474                     MPI2_IOCSTATUS_MASK;
10475                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10476                         ioc_info(ioc, "\tbreak from phys disk scan ioc_status(0x%04x), loginfo(0x%08x)\n",
10477                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10478                         break;
10479                 }
10480                 parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
10481                 if (!_scsih_get_sas_address(ioc, parent_handle,
10482                     &sas_address)) {
10483                         ioc_info(ioc, "\tBEFORE adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",
10484                                  handle,
10485                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10486                         port_id = sas_device_pg0.PhysicalPort;
10487                         mpt3sas_transport_update_links(ioc, sas_address,
10488                             handle, sas_device_pg0.PhyNum,
10489                             MPI2_SAS_NEG_LINK_RATE_1_5,
10490                             mpt3sas_get_port_by_id(ioc, port_id, 0));
10491                         set_bit(handle, ioc->pd_handles);
10492                         retry_count = 0;
10493                         /* This will retry adding the end device.
10494                          * _scsih_add_device() will decide on retries and
10495                          * return "1" when it should be retried
10496                          */
10497                         while (_scsih_add_device(ioc, handle, retry_count++,
10498                             1)) {
10499                                 ssleep(1);
10500                         }
10501                         ioc_info(ioc, "\tAFTER adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",
10502                                  handle,
10503                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10504                 }
10505         }
10506
10507         ioc_info(ioc, "\tscan devices: phys disk complete\n");
10508
10509         ioc_info(ioc, "\tscan devices: volumes start\n");
10510
10511         /* volumes */
10512         handle = 0xFFFF;
10513         while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,
10514             volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
10515                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10516                     MPI2_IOCSTATUS_MASK;
10517                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10518                         ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10519                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10520                         break;
10521                 }
10522                 handle = le16_to_cpu(volume_pg1->DevHandle);
10523                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
10524                 raid_device = _scsih_raid_device_find_by_wwid(ioc,
10525                     le64_to_cpu(volume_pg1->WWID));
10526                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
10527                 if (raid_device)
10528                         continue;
10529                 if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,
10530                     volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
10531                      sizeof(Mpi2RaidVolPage0_t)))
10532                         continue;
10533                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10534                     MPI2_IOCSTATUS_MASK;
10535                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10536                         ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10537                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10538                         break;
10539                 }
10540                 if (volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
10541                     volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
10542                     volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_DEGRADED) {
10543                         memset(&element, 0, sizeof(Mpi2EventIrConfigElement_t));
10544                         element.ReasonCode = MPI2_EVENT_IR_CHANGE_RC_ADDED;
10545                         element.VolDevHandle = volume_pg1->DevHandle;
10546                         ioc_info(ioc, "\tBEFORE adding volume: handle (0x%04x)\n",
10547                                  volume_pg1->DevHandle);
10548                         _scsih_sas_volume_add(ioc, &element);
10549                         ioc_info(ioc, "\tAFTER adding volume: handle (0x%04x)\n",
10550                                  volume_pg1->DevHandle);
10551                 }
10552         }
10553
10554         ioc_info(ioc, "\tscan devices: volumes complete\n");
10555
10556  skip_to_sas:
10557
10558         ioc_info(ioc, "\tscan devices: end devices start\n");
10559
10560         /* sas devices */
10561         handle = 0xFFFF;
10562         while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
10563             &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
10564             handle))) {
10565                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10566                     MPI2_IOCSTATUS_MASK;
10567                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10568                         ioc_info(ioc, "\tbreak from end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10569                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10570                         break;
10571                 }
10572                 handle = le16_to_cpu(sas_device_pg0.DevHandle);
10573                 if (!(_scsih_is_end_device(
10574                     le32_to_cpu(sas_device_pg0.DeviceInfo))))
10575                         continue;
10576                 port_id = sas_device_pg0.PhysicalPort;
10577                 sas_device = mpt3sas_get_sdev_by_addr(ioc,
10578                     le64_to_cpu(sas_device_pg0.SASAddress),
10579                     mpt3sas_get_port_by_id(ioc, port_id, 0));
10580                 if (sas_device) {
10581                         sas_device_put(sas_device);
10582                         continue;
10583                 }
10584                 parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
10585                 if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address)) {
10586                         ioc_info(ioc, "\tBEFORE adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",
10587                                  handle,
10588                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10589                         mpt3sas_transport_update_links(ioc, sas_address, handle,
10590                             sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
10591                             mpt3sas_get_port_by_id(ioc, port_id, 0));
10592                         retry_count = 0;
10593                         /* This will retry adding the end device.
10594                          * _scsih_add_device() will decide on retries and
10595                          * return "1" when it should be retried
10596                          */
10597                         while (_scsih_add_device(ioc, handle, retry_count++,
10598                             0)) {
10599                                 ssleep(1);
10600                         }
10601                         ioc_info(ioc, "\tAFTER adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",
10602                                  handle,
10603                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10604                 }
10605         }
10606         ioc_info(ioc, "\tscan devices: end devices complete\n");
10607         ioc_info(ioc, "\tscan devices: pcie end devices start\n");
10608
10609         /* pcie devices */
10610         handle = 0xFFFF;
10611         while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
10612                 &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
10613                 handle))) {
10614                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus)
10615                                 & MPI2_IOCSTATUS_MASK;
10616                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10617                         ioc_info(ioc, "\tbreak from pcie end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10618                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10619                         break;
10620                 }
10621                 handle = le16_to_cpu(pcie_device_pg0.DevHandle);
10622                 if (!(_scsih_is_nvme_pciescsi_device(
10623                         le32_to_cpu(pcie_device_pg0.DeviceInfo))))
10624                         continue;
10625                 pcie_device = mpt3sas_get_pdev_by_wwid(ioc,
10626                                 le64_to_cpu(pcie_device_pg0.WWID));
10627                 if (pcie_device) {
10628                         pcie_device_put(pcie_device);
10629                         continue;
10630                 }
10631                 retry_count = 0;
10632                 parent_handle = le16_to_cpu(pcie_device_pg0.ParentDevHandle);
10633                 _scsih_pcie_add_device(ioc, handle);
10634
10635                 ioc_info(ioc, "\tAFTER adding pcie end device: handle (0x%04x), wwid(0x%016llx)\n",
10636                          handle, (u64)le64_to_cpu(pcie_device_pg0.WWID));
10637         }
10638
10639         kfree(volume_pg0);
10640         kfree(volume_pg1);
10641
10642         ioc_info(ioc, "\tpcie devices: pcie end devices complete\n");
10643         ioc_info(ioc, "scan devices: complete\n");
10644 }
10645
10646 /**
10647  * mpt3sas_scsih_pre_reset_handler - reset callback handler (for scsih)
10648  * @ioc: per adapter object
10649  *
10650  * The handler for doing any required cleanup or initialization.
10651  */
10652 void mpt3sas_scsih_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)
10653 {
10654         dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));
10655 }
10656
10657 /**
10658  * mpt3sas_scsih_clear_outstanding_scsi_tm_commands - clears outstanding
10659  *                                                      scsi & tm cmds.
10660  * @ioc: per adapter object
10661  *
10662  * The handler for doing any required cleanup or initialization.
10663  */
10664 void
10665 mpt3sas_scsih_clear_outstanding_scsi_tm_commands(struct MPT3SAS_ADAPTER *ioc)
10666 {
10667         dtmprintk(ioc,
10668             ioc_info(ioc, "%s: clear outstanding scsi & tm cmds\n", __func__));
10669         if (ioc->scsih_cmds.status & MPT3_CMD_PENDING) {
10670                 ioc->scsih_cmds.status |= MPT3_CMD_RESET;
10671                 mpt3sas_base_free_smid(ioc, ioc->scsih_cmds.smid);
10672                 complete(&ioc->scsih_cmds.done);
10673         }
10674         if (ioc->tm_cmds.status & MPT3_CMD_PENDING) {
10675                 ioc->tm_cmds.status |= MPT3_CMD_RESET;
10676                 mpt3sas_base_free_smid(ioc, ioc->tm_cmds.smid);
10677                 complete(&ioc->tm_cmds.done);
10678         }
10679
10680         memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz);
10681         memset(ioc->device_remove_in_progress, 0,
10682                ioc->device_remove_in_progress_sz);
10683         _scsih_fw_event_cleanup_queue(ioc);
10684         _scsih_flush_running_cmds(ioc);
10685 }
10686
10687 /**
10688  * mpt3sas_scsih_reset_done_handler - reset callback handler (for scsih)
10689  * @ioc: per adapter object
10690  *
10691  * The handler for doing any required cleanup or initialization.
10692  */
10693 void
10694 mpt3sas_scsih_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)
10695 {
10696         dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));
10697         if (!(disable_discovery > 0 && !ioc->sas_hba.num_phys)) {
10698                 if (ioc->multipath_on_hba) {
10699                         _scsih_sas_port_refresh(ioc);
10700                         _scsih_update_vphys_after_reset(ioc);
10701                 }
10702                 _scsih_prep_device_scan(ioc);
10703                 _scsih_create_enclosure_list_after_reset(ioc);
10704                 _scsih_search_responding_sas_devices(ioc);
10705                 _scsih_search_responding_pcie_devices(ioc);
10706                 _scsih_search_responding_raid_devices(ioc);
10707                 _scsih_search_responding_expanders(ioc);
10708                 _scsih_error_recovery_delete_devices(ioc);
10709         }
10710 }
10711
10712 /**
10713  * _mpt3sas_fw_work - delayed task for processing firmware events
10714  * @ioc: per adapter object
10715  * @fw_event: The fw_event_work object
10716  * Context: user.
10717  */
10718 static void
10719 _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
10720 {
10721         ioc->current_event = fw_event;
10722         _scsih_fw_event_del_from_list(ioc, fw_event);
10723
10724         /* the queue is being flushed so ignore this event */
10725         if (ioc->remove_host || ioc->pci_error_recovery) {
10726                 fw_event_work_put(fw_event);
10727                 ioc->current_event = NULL;
10728                 return;
10729         }
10730
10731         switch (fw_event->event) {
10732         case MPT3SAS_PROCESS_TRIGGER_DIAG:
10733                 mpt3sas_process_trigger_data(ioc,
10734                         (struct SL_WH_TRIGGERS_EVENT_DATA_T *)
10735                         fw_event->event_data);
10736                 break;
10737         case MPT3SAS_REMOVE_UNRESPONDING_DEVICES:
10738                 while (scsi_host_in_recovery(ioc->shost) ||
10739                                          ioc->shost_recovery) {
10740                         /*
10741                          * If we're unloading or cancelling the work, bail.
10742                          * Otherwise, this can become an infinite loop.
10743                          */
10744                         if (ioc->remove_host || ioc->fw_events_cleanup)
10745                                 goto out;
10746                         ssleep(1);
10747                 }
10748                 _scsih_remove_unresponding_devices(ioc);
10749                 _scsih_del_dirty_vphy(ioc);
10750                 _scsih_del_dirty_port_entries(ioc);
10751                 if (ioc->is_gen35_ioc)
10752                         _scsih_update_device_qdepth(ioc);
10753                 _scsih_scan_for_devices_after_reset(ioc);
10754                 /*
10755                  * If diag reset has occurred during the driver load
10756                  * then driver has to complete the driver load operation
10757                  * by executing the following items:
10758                  *- Register the devices from sas_device_init_list to SML
10759                  *- clear is_driver_loading flag,
10760                  *- start the watchdog thread.
10761                  * In happy driver load path, above things are taken care of when
10762                  * driver executes scsih_scan_finished().
10763                  */
10764                 if (ioc->is_driver_loading)
10765                         _scsih_complete_devices_scanning(ioc);
10766                 _scsih_set_nvme_max_shutdown_latency(ioc);
10767                 break;
10768         case MPT3SAS_PORT_ENABLE_COMPLETE:
10769                 ioc->start_scan = 0;
10770                 if (missing_delay[0] != -1 && missing_delay[1] != -1)
10771                         mpt3sas_base_update_missing_delay(ioc, missing_delay[0],
10772                             missing_delay[1]);
10773                 dewtprintk(ioc,
10774                            ioc_info(ioc, "port enable: complete from worker thread\n"));
10775                 break;
10776         case MPT3SAS_TURN_ON_PFA_LED:
10777                 _scsih_turn_on_pfa_led(ioc, fw_event->device_handle);
10778                 break;
10779         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
10780                 _scsih_sas_topology_change_event(ioc, fw_event);
10781                 break;
10782         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
10783                 if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
10784                         _scsih_sas_device_status_change_event_debug(ioc,
10785                             (Mpi2EventDataSasDeviceStatusChange_t *)
10786                             fw_event->event_data);
10787                 break;
10788         case MPI2_EVENT_SAS_DISCOVERY:
10789                 _scsih_sas_discovery_event(ioc, fw_event);
10790                 break;
10791         case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
10792                 _scsih_sas_device_discovery_error_event(ioc, fw_event);
10793                 break;
10794         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
10795                 _scsih_sas_broadcast_primitive_event(ioc, fw_event);
10796                 break;
10797         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
10798                 _scsih_sas_enclosure_dev_status_change_event(ioc,
10799                     fw_event);
10800                 break;
10801         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
10802                 _scsih_sas_ir_config_change_event(ioc, fw_event);
10803                 break;
10804         case MPI2_EVENT_IR_VOLUME:
10805                 _scsih_sas_ir_volume_event(ioc, fw_event);
10806                 break;
10807         case MPI2_EVENT_IR_PHYSICAL_DISK:
10808                 _scsih_sas_ir_physical_disk_event(ioc, fw_event);
10809                 break;
10810         case MPI2_EVENT_IR_OPERATION_STATUS:
10811                 _scsih_sas_ir_operation_status_event(ioc, fw_event);
10812                 break;
10813         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
10814                 _scsih_pcie_device_status_change_event(ioc, fw_event);
10815                 break;
10816         case MPI2_EVENT_PCIE_ENUMERATION:
10817                 _scsih_pcie_enumeration_event(ioc, fw_event);
10818                 break;
10819         case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
10820                 _scsih_pcie_topology_change_event(ioc, fw_event);
10821                 ioc->current_event = NULL;
10822                 return;
10823         }
10824 out:
10825         fw_event_work_put(fw_event);
10826         ioc->current_event = NULL;
10827 }
10828
10829 /**
10830  * _firmware_event_work
10831  * @work: The fw_event_work object
10832  * Context: user.
10833  *
10834  * wrappers for the work thread handling firmware events
10835  */
10836
10837 static void
10838 _firmware_event_work(struct work_struct *work)
10839 {
10840         struct fw_event_work *fw_event = container_of(work,
10841             struct fw_event_work, work);
10842
10843         _mpt3sas_fw_work(fw_event->ioc, fw_event);
10844 }
10845
10846 /**
10847  * mpt3sas_scsih_event_callback - firmware event handler (called at ISR time)
10848  * @ioc: per adapter object
10849  * @msix_index: MSIX table index supplied by the OS
10850  * @reply: reply message frame(lower 32bit addr)
10851  * Context: interrupt.
10852  *
10853  * This function merely adds a new work task into ioc->firmware_event_thread.
10854  * The tasks are worked from _firmware_event_work in user context.
10855  *
10856  * Return: 1 meaning mf should be freed from _base_interrupt
10857  *         0 means the mf is freed from this function.
10858  */
10859 u8
10860 mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
10861         u32 reply)
10862 {
10863         struct fw_event_work *fw_event;
10864         Mpi2EventNotificationReply_t *mpi_reply;
10865         u16 event;
10866         u16 sz;
10867         Mpi26EventDataActiveCableExcept_t *ActiveCableEventData;
10868
10869         /* events turned off due to host reset */
10870         if (ioc->pci_error_recovery)
10871                 return 1;
10872
10873         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
10874
10875         if (unlikely(!mpi_reply)) {
10876                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
10877                         __FILE__, __LINE__, __func__);
10878                 return 1;
10879         }
10880
10881         event = le16_to_cpu(mpi_reply->Event);
10882
10883         if (event != MPI2_EVENT_LOG_ENTRY_ADDED)
10884                 mpt3sas_trigger_event(ioc, event, 0);
10885
10886         switch (event) {
10887         /* handle these */
10888         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
10889         {
10890                 Mpi2EventDataSasBroadcastPrimitive_t *baen_data =
10891                     (Mpi2EventDataSasBroadcastPrimitive_t *)
10892                     mpi_reply->EventData;
10893
10894                 if (baen_data->Primitive !=
10895                     MPI2_EVENT_PRIMITIVE_ASYNCHRONOUS_EVENT)
10896                         return 1;
10897
10898                 if (ioc->broadcast_aen_busy) {
10899                         ioc->broadcast_aen_pending++;
10900                         return 1;
10901                 } else
10902                         ioc->broadcast_aen_busy = 1;
10903                 break;
10904         }
10905
10906         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
10907                 _scsih_check_topo_delete_events(ioc,
10908                     (Mpi2EventDataSasTopologyChangeList_t *)
10909                     mpi_reply->EventData);
10910                 /*
10911                  * No need to add the topology change list
10912                  * event to fw event work queue when
10913                  * diag reset is going on. Since during diag
10914                  * reset driver scan the devices by reading
10915                  * sas device page0's not by processing the
10916                  * events.
10917                  */
10918                 if (ioc->shost_recovery)
10919                         return 1;
10920                 break;
10921         case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
10922         _scsih_check_pcie_topo_remove_events(ioc,
10923                     (Mpi26EventDataPCIeTopologyChangeList_t *)
10924                     mpi_reply->EventData);
10925                 if (ioc->shost_recovery)
10926                         return 1;
10927                 break;
10928         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
10929                 _scsih_check_ir_config_unhide_events(ioc,
10930                     (Mpi2EventDataIrConfigChangeList_t *)
10931                     mpi_reply->EventData);
10932                 break;
10933         case MPI2_EVENT_IR_VOLUME:
10934                 _scsih_check_volume_delete_events(ioc,
10935                     (Mpi2EventDataIrVolume_t *)
10936                     mpi_reply->EventData);
10937                 break;
10938         case MPI2_EVENT_LOG_ENTRY_ADDED:
10939         {
10940                 Mpi2EventDataLogEntryAdded_t *log_entry;
10941                 u32 log_code;
10942
10943                 if (!ioc->is_warpdrive)
10944                         break;
10945
10946                 log_entry = (Mpi2EventDataLogEntryAdded_t *)
10947                     mpi_reply->EventData;
10948                 log_code = le32_to_cpu(*(__le32 *)log_entry->LogData);
10949
10950                 if (le16_to_cpu(log_entry->LogEntryQualifier)
10951                     != MPT2_WARPDRIVE_LOGENTRY)
10952                         break;
10953
10954                 switch (log_code) {
10955                 case MPT2_WARPDRIVE_LC_SSDT:
10956                         ioc_warn(ioc, "WarpDrive Warning: IO Throttling has occurred in the WarpDrive subsystem. Check WarpDrive documentation for additional details.\n");
10957                         break;
10958                 case MPT2_WARPDRIVE_LC_SSDLW:
10959                         ioc_warn(ioc, "WarpDrive Warning: Program/Erase Cycles for the WarpDrive subsystem in degraded range. Check WarpDrive documentation for additional details.\n");
10960                         break;
10961                 case MPT2_WARPDRIVE_LC_SSDLF:
10962                         ioc_err(ioc, "WarpDrive Fatal Error: There are no Program/Erase Cycles for the WarpDrive subsystem. The storage device will be in read-only mode. Check WarpDrive documentation for additional details.\n");
10963                         break;
10964                 case MPT2_WARPDRIVE_LC_BRMF:
10965                         ioc_err(ioc, "WarpDrive Fatal Error: The Backup Rail Monitor has failed on the WarpDrive subsystem. Check WarpDrive documentation for additional details.\n");
10966                         break;
10967                 }
10968
10969                 break;
10970         }
10971         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
10972                 _scsih_sas_device_status_change_event(ioc,
10973                     (Mpi2EventDataSasDeviceStatusChange_t *)
10974                     mpi_reply->EventData);
10975                 break;
10976         case MPI2_EVENT_IR_OPERATION_STATUS:
10977         case MPI2_EVENT_SAS_DISCOVERY:
10978         case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
10979         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
10980         case MPI2_EVENT_IR_PHYSICAL_DISK:
10981         case MPI2_EVENT_PCIE_ENUMERATION:
10982         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
10983                 break;
10984
10985         case MPI2_EVENT_TEMP_THRESHOLD:
10986                 _scsih_temp_threshold_events(ioc,
10987                         (Mpi2EventDataTemperature_t *)
10988                         mpi_reply->EventData);
10989                 break;
10990         case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
10991                 ActiveCableEventData =
10992                     (Mpi26EventDataActiveCableExcept_t *) mpi_reply->EventData;
10993                 switch (ActiveCableEventData->ReasonCode) {
10994                 case MPI26_EVENT_ACTIVE_CABLE_INSUFFICIENT_POWER:
10995                         ioc_notice(ioc, "Currently an active cable with ReceptacleID %d\n",
10996                                    ActiveCableEventData->ReceptacleID);
10997                         pr_notice("cannot be powered and devices connected\n");
10998                         pr_notice("to this active cable will not be seen\n");
10999                         pr_notice("This active cable requires %d mW of power\n",
11000                             le32_to_cpu(
11001                             ActiveCableEventData->ActiveCablePowerRequirement));
11002                         break;
11003
11004                 case MPI26_EVENT_ACTIVE_CABLE_DEGRADED:
11005                         ioc_notice(ioc, "Currently a cable with ReceptacleID %d\n",
11006                                    ActiveCableEventData->ReceptacleID);
11007                         pr_notice(
11008                             "is not running at optimal speed(12 Gb/s rate)\n");
11009                         break;
11010                 }
11011
11012                 break;
11013
11014         default: /* ignore the rest */
11015                 return 1;
11016         }
11017
11018         sz = le16_to_cpu(mpi_reply->EventDataLength) * 4;
11019         fw_event = alloc_fw_event_work(sz);
11020         if (!fw_event) {
11021                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
11022                         __FILE__, __LINE__, __func__);
11023                 return 1;
11024         }
11025
11026         memcpy(fw_event->event_data, mpi_reply->EventData, sz);
11027         fw_event->ioc = ioc;
11028         fw_event->VF_ID = mpi_reply->VF_ID;
11029         fw_event->VP_ID = mpi_reply->VP_ID;
11030         fw_event->event = event;
11031         _scsih_fw_event_add(ioc, fw_event);
11032         fw_event_work_put(fw_event);
11033         return 1;
11034 }
11035
11036 /**
11037  * _scsih_expander_node_remove - removing expander device from list.
11038  * @ioc: per adapter object
11039  * @sas_expander: the sas_device object
11040  *
11041  * Removing object and freeing associated memory from the
11042  * ioc->sas_expander_list.
11043  */
11044 static void
11045 _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
11046         struct _sas_node *sas_expander)
11047 {
11048         struct _sas_port *mpt3sas_port, *next;
11049         unsigned long flags;
11050         int port_id;
11051
11052         /* remove sibling ports attached to this expander */
11053         list_for_each_entry_safe(mpt3sas_port, next,
11054            &sas_expander->sas_port_list, port_list) {
11055                 if (ioc->shost_recovery)
11056                         return;
11057                 if (mpt3sas_port->remote_identify.device_type ==
11058                     SAS_END_DEVICE)
11059                         mpt3sas_device_remove_by_sas_address(ioc,
11060                             mpt3sas_port->remote_identify.sas_address,
11061                             mpt3sas_port->hba_port);
11062                 else if (mpt3sas_port->remote_identify.device_type ==
11063                     SAS_EDGE_EXPANDER_DEVICE ||
11064                     mpt3sas_port->remote_identify.device_type ==
11065                     SAS_FANOUT_EXPANDER_DEVICE)
11066                         mpt3sas_expander_remove(ioc,
11067                             mpt3sas_port->remote_identify.sas_address,
11068                             mpt3sas_port->hba_port);
11069         }
11070
11071         port_id = sas_expander->port->port_id;
11072
11073         mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,
11074             sas_expander->sas_address_parent, sas_expander->port);
11075
11076         ioc_info(ioc,
11077             "expander_remove: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
11078             sas_expander->handle, (unsigned long long)
11079             sas_expander->sas_address,
11080             port_id);
11081
11082         spin_lock_irqsave(&ioc->sas_node_lock, flags);
11083         list_del(&sas_expander->list);
11084         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
11085
11086         kfree(sas_expander->phy);
11087         kfree(sas_expander);
11088 }
11089
11090 /**
11091  * _scsih_nvme_shutdown - NVMe shutdown notification
11092  * @ioc: per adapter object
11093  *
11094  * Sending IoUnitControl request with shutdown operation code to alert IOC that
11095  * the host system is shutting down so that IOC can issue NVMe shutdown to
11096  * NVMe drives attached to it.
11097  */
11098 static void
11099 _scsih_nvme_shutdown(struct MPT3SAS_ADAPTER *ioc)
11100 {
11101         Mpi26IoUnitControlRequest_t *mpi_request;
11102         Mpi26IoUnitControlReply_t *mpi_reply;
11103         u16 smid;
11104
11105         /* are there any NVMe devices ? */
11106         if (list_empty(&ioc->pcie_device_list))
11107                 return;
11108
11109         mutex_lock(&ioc->scsih_cmds.mutex);
11110
11111         if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
11112                 ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
11113                 goto out;
11114         }
11115
11116         ioc->scsih_cmds.status = MPT3_CMD_PENDING;
11117
11118         smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
11119         if (!smid) {
11120                 ioc_err(ioc,
11121                     "%s: failed obtaining a smid\n", __func__);
11122                 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11123                 goto out;
11124         }
11125
11126         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
11127         ioc->scsih_cmds.smid = smid;
11128         memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t));
11129         mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL;
11130         mpi_request->Operation = MPI26_CTRL_OP_SHUTDOWN;
11131
11132         init_completion(&ioc->scsih_cmds.done);
11133         ioc->put_smid_default(ioc, smid);
11134         /* Wait for max_shutdown_latency seconds */
11135         ioc_info(ioc,
11136                 "Io Unit Control shutdown (sending), Shutdown latency %d sec\n",
11137                 ioc->max_shutdown_latency);
11138         wait_for_completion_timeout(&ioc->scsih_cmds.done,
11139                         ioc->max_shutdown_latency*HZ);
11140
11141         if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
11142                 ioc_err(ioc, "%s: timeout\n", __func__);
11143                 goto out;
11144         }
11145
11146         if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
11147                 mpi_reply = ioc->scsih_cmds.reply;
11148                 ioc_info(ioc, "Io Unit Control shutdown (complete):"
11149                         "ioc_status(0x%04x), loginfo(0x%08x)\n",
11150                         le16_to_cpu(mpi_reply->IOCStatus),
11151                         le32_to_cpu(mpi_reply->IOCLogInfo));
11152         }
11153  out:
11154         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11155         mutex_unlock(&ioc->scsih_cmds.mutex);
11156 }
11157
11158
11159 /**
11160  * _scsih_ir_shutdown - IR shutdown notification
11161  * @ioc: per adapter object
11162  *
11163  * Sending RAID Action to alert the Integrated RAID subsystem of the IOC that
11164  * the host system is shutting down.
11165  */
11166 static void
11167 _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)
11168 {
11169         Mpi2RaidActionRequest_t *mpi_request;
11170         Mpi2RaidActionReply_t *mpi_reply;
11171         u16 smid;
11172
11173         /* is IR firmware build loaded ? */
11174         if (!ioc->ir_firmware)
11175                 return;
11176
11177         /* are there any volumes ? */
11178         if (list_empty(&ioc->raid_device_list))
11179                 return;
11180
11181         mutex_lock(&ioc->scsih_cmds.mutex);
11182
11183         if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
11184                 ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
11185                 goto out;
11186         }
11187         ioc->scsih_cmds.status = MPT3_CMD_PENDING;
11188
11189         smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
11190         if (!smid) {
11191                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
11192                 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11193                 goto out;
11194         }
11195
11196         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
11197         ioc->scsih_cmds.smid = smid;
11198         memset(mpi_request, 0, sizeof(Mpi2RaidActionRequest_t));
11199
11200         mpi_request->Function = MPI2_FUNCTION_RAID_ACTION;
11201         mpi_request->Action = MPI2_RAID_ACTION_SYSTEM_SHUTDOWN_INITIATED;
11202
11203         if (!ioc->hide_ir_msg)
11204                 ioc_info(ioc, "IR shutdown (sending)\n");
11205         init_completion(&ioc->scsih_cmds.done);
11206         ioc->put_smid_default(ioc, smid);
11207         wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);
11208
11209         if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
11210                 ioc_err(ioc, "%s: timeout\n", __func__);
11211                 goto out;
11212         }
11213
11214         if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
11215                 mpi_reply = ioc->scsih_cmds.reply;
11216                 if (!ioc->hide_ir_msg)
11217                         ioc_info(ioc, "IR shutdown (complete): ioc_status(0x%04x), loginfo(0x%08x)\n",
11218                                  le16_to_cpu(mpi_reply->IOCStatus),
11219                                  le32_to_cpu(mpi_reply->IOCLogInfo));
11220         }
11221
11222  out:
11223         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11224         mutex_unlock(&ioc->scsih_cmds.mutex);
11225 }
11226
11227 /**
11228  * _scsih_get_shost_and_ioc - get shost and ioc
11229  *                      and verify whether they are NULL or not
11230  * @pdev: PCI device struct
11231  * @shost: address of scsi host pointer
11232  * @ioc: address of HBA adapter pointer
11233  *
11234  * Return zero if *shost and *ioc are not NULL otherwise return error number.
11235  */
11236 static int
11237 _scsih_get_shost_and_ioc(struct pci_dev *pdev,
11238         struct Scsi_Host **shost, struct MPT3SAS_ADAPTER **ioc)
11239 {
11240         *shost = pci_get_drvdata(pdev);
11241         if (*shost == NULL) {
11242                 dev_err(&pdev->dev, "pdev's driver data is null\n");
11243                 return -ENXIO;
11244         }
11245
11246         *ioc = shost_priv(*shost);
11247         if (*ioc == NULL) {
11248                 dev_err(&pdev->dev, "shost's private data is null\n");
11249                 return -ENXIO;
11250         }
11251
11252         return 0;
11253 }
11254
11255 /**
11256  * scsih_remove - detach and remove add host
11257  * @pdev: PCI device struct
11258  *
11259  * Routine called when unloading the driver.
11260  */
11261 static void scsih_remove(struct pci_dev *pdev)
11262 {
11263         struct Scsi_Host *shost;
11264         struct MPT3SAS_ADAPTER *ioc;
11265         struct _sas_port *mpt3sas_port, *next_port;
11266         struct _raid_device *raid_device, *next;
11267         struct MPT3SAS_TARGET *sas_target_priv_data;
11268         struct _pcie_device *pcie_device, *pcienext;
11269         struct workqueue_struct *wq;
11270         unsigned long flags;
11271         Mpi2ConfigReply_t mpi_reply;
11272         struct hba_port *port, *port_next;
11273
11274         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
11275                 return;
11276
11277         ioc->remove_host = 1;
11278
11279         if (!pci_device_is_present(pdev)) {
11280                 mpt3sas_base_pause_mq_polling(ioc);
11281                 _scsih_flush_running_cmds(ioc);
11282         }
11283
11284         _scsih_fw_event_cleanup_queue(ioc);
11285
11286         spin_lock_irqsave(&ioc->fw_event_lock, flags);
11287         wq = ioc->firmware_event_thread;
11288         ioc->firmware_event_thread = NULL;
11289         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
11290         if (wq)
11291                 destroy_workqueue(wq);
11292         /*
11293          * Copy back the unmodified ioc page1. so that on next driver load,
11294          * current modified changes on ioc page1 won't take effect.
11295          */
11296         if (ioc->is_aero_ioc)
11297                 mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,
11298                                 &ioc->ioc_pg1_copy);
11299         /* release all the volumes */
11300         _scsih_ir_shutdown(ioc);
11301         mpt3sas_destroy_debugfs(ioc);
11302         sas_remove_host(shost);
11303         list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,
11304             list) {
11305                 if (raid_device->starget) {
11306                         sas_target_priv_data =
11307                             raid_device->starget->hostdata;
11308                         sas_target_priv_data->deleted = 1;
11309                         scsi_remove_target(&raid_device->starget->dev);
11310                 }
11311                 ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
11312                          raid_device->handle, (u64)raid_device->wwid);
11313                 _scsih_raid_device_remove(ioc, raid_device);
11314         }
11315         list_for_each_entry_safe(pcie_device, pcienext, &ioc->pcie_device_list,
11316                 list) {
11317                 _scsih_pcie_device_remove_from_sml(ioc, pcie_device);
11318                 list_del_init(&pcie_device->list);
11319                 pcie_device_put(pcie_device);
11320         }
11321
11322         /* free ports attached to the sas_host */
11323         list_for_each_entry_safe(mpt3sas_port, next_port,
11324            &ioc->sas_hba.sas_port_list, port_list) {
11325                 if (mpt3sas_port->remote_identify.device_type ==
11326                     SAS_END_DEVICE)
11327                         mpt3sas_device_remove_by_sas_address(ioc,
11328                             mpt3sas_port->remote_identify.sas_address,
11329                             mpt3sas_port->hba_port);
11330                 else if (mpt3sas_port->remote_identify.device_type ==
11331                     SAS_EDGE_EXPANDER_DEVICE ||
11332                     mpt3sas_port->remote_identify.device_type ==
11333                     SAS_FANOUT_EXPANDER_DEVICE)
11334                         mpt3sas_expander_remove(ioc,
11335                             mpt3sas_port->remote_identify.sas_address,
11336                             mpt3sas_port->hba_port);
11337         }
11338
11339         list_for_each_entry_safe(port, port_next,
11340             &ioc->port_table_list, list) {
11341                 list_del(&port->list);
11342                 kfree(port);
11343         }
11344
11345         /* free phys attached to the sas_host */
11346         if (ioc->sas_hba.num_phys) {
11347                 kfree(ioc->sas_hba.phy);
11348                 ioc->sas_hba.phy = NULL;
11349                 ioc->sas_hba.num_phys = 0;
11350         }
11351
11352         mpt3sas_base_detach(ioc);
11353         spin_lock(&gioc_lock);
11354         list_del(&ioc->list);
11355         spin_unlock(&gioc_lock);
11356         scsi_host_put(shost);
11357 }
11358
11359 /**
11360  * scsih_shutdown - routine call during system shutdown
11361  * @pdev: PCI device struct
11362  */
11363 static void
11364 scsih_shutdown(struct pci_dev *pdev)
11365 {
11366         struct Scsi_Host *shost;
11367         struct MPT3SAS_ADAPTER *ioc;
11368         struct workqueue_struct *wq;
11369         unsigned long flags;
11370         Mpi2ConfigReply_t mpi_reply;
11371
11372         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
11373                 return;
11374
11375         ioc->remove_host = 1;
11376
11377         if (!pci_device_is_present(pdev)) {
11378                 mpt3sas_base_pause_mq_polling(ioc);
11379                 _scsih_flush_running_cmds(ioc);
11380         }
11381
11382         _scsih_fw_event_cleanup_queue(ioc);
11383
11384         spin_lock_irqsave(&ioc->fw_event_lock, flags);
11385         wq = ioc->firmware_event_thread;
11386         ioc->firmware_event_thread = NULL;
11387         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
11388         if (wq)
11389                 destroy_workqueue(wq);
11390         /*
11391          * Copy back the unmodified ioc page1 so that on next driver load,
11392          * current modified changes on ioc page1 won't take effect.
11393          */
11394         if (ioc->is_aero_ioc)
11395                 mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,
11396                                 &ioc->ioc_pg1_copy);
11397
11398         _scsih_ir_shutdown(ioc);
11399         _scsih_nvme_shutdown(ioc);
11400         mpt3sas_base_mask_interrupts(ioc);
11401         mpt3sas_base_stop_watchdog(ioc);
11402         ioc->shost_recovery = 1;
11403         mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);
11404         ioc->shost_recovery = 0;
11405         mpt3sas_base_free_irq(ioc);
11406         mpt3sas_base_disable_msix(ioc);
11407 }
11408
11409
11410 /**
11411  * _scsih_probe_boot_devices - reports 1st device
11412  * @ioc: per adapter object
11413  *
11414  * If specified in bios page 2, this routine reports the 1st
11415  * device scsi-ml or sas transport for persistent boot device
11416  * purposes.  Please refer to function _scsih_determine_boot_device()
11417  */
11418 static void
11419 _scsih_probe_boot_devices(struct MPT3SAS_ADAPTER *ioc)
11420 {
11421         u32 channel;
11422         void *device;
11423         struct _sas_device *sas_device;
11424         struct _raid_device *raid_device;
11425         struct _pcie_device *pcie_device;
11426         u16 handle;
11427         u64 sas_address_parent;
11428         u64 sas_address;
11429         unsigned long flags;
11430         int rc;
11431         int tid;
11432         struct hba_port *port;
11433
11434          /* no Bios, return immediately */
11435         if (!ioc->bios_pg3.BiosVersion)
11436                 return;
11437
11438         device = NULL;
11439         if (ioc->req_boot_device.device) {
11440                 device =  ioc->req_boot_device.device;
11441                 channel = ioc->req_boot_device.channel;
11442         } else if (ioc->req_alt_boot_device.device) {
11443                 device =  ioc->req_alt_boot_device.device;
11444                 channel = ioc->req_alt_boot_device.channel;
11445         } else if (ioc->current_boot_device.device) {
11446                 device =  ioc->current_boot_device.device;
11447                 channel = ioc->current_boot_device.channel;
11448         }
11449
11450         if (!device)
11451                 return;
11452
11453         if (channel == RAID_CHANNEL) {
11454                 raid_device = device;
11455                 /*
11456                  * If this boot vd is already registered with SML then
11457                  * no need to register it again as part of device scanning
11458                  * after diag reset during driver load operation.
11459                  */
11460                 if (raid_device->starget)
11461                         return;
11462                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
11463                     raid_device->id, 0);
11464                 if (rc)
11465                         _scsih_raid_device_remove(ioc, raid_device);
11466         } else if (channel == PCIE_CHANNEL) {
11467                 pcie_device = device;
11468                 /*
11469                  * If this boot NVMe device is already registered with SML then
11470                  * no need to register it again as part of device scanning
11471                  * after diag reset during driver load operation.
11472                  */
11473                 if (pcie_device->starget)
11474                         return;
11475                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11476                 tid = pcie_device->id;
11477                 list_move_tail(&pcie_device->list, &ioc->pcie_device_list);
11478                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
11479                 rc = scsi_add_device(ioc->shost, PCIE_CHANNEL, tid, 0);
11480                 if (rc)
11481                         _scsih_pcie_device_remove(ioc, pcie_device);
11482         } else {
11483                 sas_device = device;
11484                 /*
11485                  * If this boot sas/sata device is already registered with SML
11486                  * then no need to register it again as part of device scanning
11487                  * after diag reset during driver load operation.
11488                  */
11489                 if (sas_device->starget)
11490                         return;
11491                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
11492                 handle = sas_device->handle;
11493                 sas_address_parent = sas_device->sas_address_parent;
11494                 sas_address = sas_device->sas_address;
11495                 port = sas_device->port;
11496                 list_move_tail(&sas_device->list, &ioc->sas_device_list);
11497                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
11498
11499                 if (ioc->hide_drives)
11500                         return;
11501
11502                 if (!port)
11503                         return;
11504
11505                 if (!mpt3sas_transport_port_add(ioc, handle,
11506                     sas_address_parent, port)) {
11507                         _scsih_sas_device_remove(ioc, sas_device);
11508                 } else if (!sas_device->starget) {
11509                         if (!ioc->is_driver_loading) {
11510                                 mpt3sas_transport_port_remove(ioc,
11511                                     sas_address,
11512                                     sas_address_parent, port);
11513                                 _scsih_sas_device_remove(ioc, sas_device);
11514                         }
11515                 }
11516         }
11517 }
11518
11519 /**
11520  * _scsih_probe_raid - reporting raid volumes to scsi-ml
11521  * @ioc: per adapter object
11522  *
11523  * Called during initial loading of the driver.
11524  */
11525 static void
11526 _scsih_probe_raid(struct MPT3SAS_ADAPTER *ioc)
11527 {
11528         struct _raid_device *raid_device, *raid_next;
11529         int rc;
11530
11531         list_for_each_entry_safe(raid_device, raid_next,
11532             &ioc->raid_device_list, list) {
11533                 if (raid_device->starget)
11534                         continue;
11535                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
11536                     raid_device->id, 0);
11537                 if (rc)
11538                         _scsih_raid_device_remove(ioc, raid_device);
11539         }
11540 }
11541
11542 static struct _sas_device *get_next_sas_device(struct MPT3SAS_ADAPTER *ioc)
11543 {
11544         struct _sas_device *sas_device = NULL;
11545         unsigned long flags;
11546
11547         spin_lock_irqsave(&ioc->sas_device_lock, flags);
11548         if (!list_empty(&ioc->sas_device_init_list)) {
11549                 sas_device = list_first_entry(&ioc->sas_device_init_list,
11550                                 struct _sas_device, list);
11551                 sas_device_get(sas_device);
11552         }
11553         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
11554
11555         return sas_device;
11556 }
11557
11558 static void sas_device_make_active(struct MPT3SAS_ADAPTER *ioc,
11559                 struct _sas_device *sas_device)
11560 {
11561         unsigned long flags;
11562
11563         spin_lock_irqsave(&ioc->sas_device_lock, flags);
11564
11565         /*
11566          * Since we dropped the lock during the call to port_add(), we need to
11567          * be careful here that somebody else didn't move or delete this item
11568          * while we were busy with other things.
11569          *
11570          * If it was on the list, we need a put() for the reference the list
11571          * had. Either way, we need a get() for the destination list.
11572          */
11573         if (!list_empty(&sas_device->list)) {
11574                 list_del_init(&sas_device->list);
11575                 sas_device_put(sas_device);
11576         }
11577
11578         sas_device_get(sas_device);
11579         list_add_tail(&sas_device->list, &ioc->sas_device_list);
11580
11581         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
11582 }
11583
11584 /**
11585  * _scsih_probe_sas - reporting sas devices to sas transport
11586  * @ioc: per adapter object
11587  *
11588  * Called during initial loading of the driver.
11589  */
11590 static void
11591 _scsih_probe_sas(struct MPT3SAS_ADAPTER *ioc)
11592 {
11593         struct _sas_device *sas_device;
11594
11595         if (ioc->hide_drives)
11596                 return;
11597
11598         while ((sas_device = get_next_sas_device(ioc))) {
11599                 if (!mpt3sas_transport_port_add(ioc, sas_device->handle,
11600                     sas_device->sas_address_parent, sas_device->port)) {
11601                         _scsih_sas_device_remove(ioc, sas_device);
11602                         sas_device_put(sas_device);
11603                         continue;
11604                 } else if (!sas_device->starget) {
11605                         /*
11606                          * When asyn scanning is enabled, its not possible to
11607                          * remove devices while scanning is turned on due to an
11608                          * oops in scsi_sysfs_add_sdev()->add_device()->
11609                          * sysfs_addrm_start()
11610                          */
11611                         if (!ioc->is_driver_loading) {
11612                                 mpt3sas_transport_port_remove(ioc,
11613                                     sas_device->sas_address,
11614                                     sas_device->sas_address_parent,
11615                                     sas_device->port);
11616                                 _scsih_sas_device_remove(ioc, sas_device);
11617                                 sas_device_put(sas_device);
11618                                 continue;
11619                         }
11620                 }
11621                 sas_device_make_active(ioc, sas_device);
11622                 sas_device_put(sas_device);
11623         }
11624 }
11625
11626 /**
11627  * get_next_pcie_device - Get the next pcie device
11628  * @ioc: per adapter object
11629  *
11630  * Get the next pcie device from pcie_device_init_list list.
11631  *
11632  * Return: pcie device structure if pcie_device_init_list list is not empty
11633  * otherwise returns NULL
11634  */
11635 static struct _pcie_device *get_next_pcie_device(struct MPT3SAS_ADAPTER *ioc)
11636 {
11637         struct _pcie_device *pcie_device = NULL;
11638         unsigned long flags;
11639
11640         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11641         if (!list_empty(&ioc->pcie_device_init_list)) {
11642                 pcie_device = list_first_entry(&ioc->pcie_device_init_list,
11643                                 struct _pcie_device, list);
11644                 pcie_device_get(pcie_device);
11645         }
11646         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
11647
11648         return pcie_device;
11649 }
11650
11651 /**
11652  * pcie_device_make_active - Add pcie device to pcie_device_list list
11653  * @ioc: per adapter object
11654  * @pcie_device: pcie device object
11655  *
11656  * Add the pcie device which has registered with SCSI Transport Later to
11657  * pcie_device_list list
11658  */
11659 static void pcie_device_make_active(struct MPT3SAS_ADAPTER *ioc,
11660                 struct _pcie_device *pcie_device)
11661 {
11662         unsigned long flags;
11663
11664         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11665
11666         if (!list_empty(&pcie_device->list)) {
11667                 list_del_init(&pcie_device->list);
11668                 pcie_device_put(pcie_device);
11669         }
11670         pcie_device_get(pcie_device);
11671         list_add_tail(&pcie_device->list, &ioc->pcie_device_list);
11672
11673         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
11674 }
11675
11676 /**
11677  * _scsih_probe_pcie - reporting PCIe devices to scsi-ml
11678  * @ioc: per adapter object
11679  *
11680  * Called during initial loading of the driver.
11681  */
11682 static void
11683 _scsih_probe_pcie(struct MPT3SAS_ADAPTER *ioc)
11684 {
11685         struct _pcie_device *pcie_device;
11686         int rc;
11687
11688         /* PCIe Device List */
11689         while ((pcie_device = get_next_pcie_device(ioc))) {
11690                 if (pcie_device->starget) {
11691                         pcie_device_put(pcie_device);
11692                         continue;
11693                 }
11694                 if (pcie_device->access_status ==
11695                     MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED) {
11696                         pcie_device_make_active(ioc, pcie_device);
11697                         pcie_device_put(pcie_device);
11698                         continue;
11699                 }
11700                 rc = scsi_add_device(ioc->shost, PCIE_CHANNEL,
11701                         pcie_device->id, 0);
11702                 if (rc) {
11703                         _scsih_pcie_device_remove(ioc, pcie_device);
11704                         pcie_device_put(pcie_device);
11705                         continue;
11706                 } else if (!pcie_device->starget) {
11707                         /*
11708                          * When async scanning is enabled, its not possible to
11709                          * remove devices while scanning is turned on due to an
11710                          * oops in scsi_sysfs_add_sdev()->add_device()->
11711                          * sysfs_addrm_start()
11712                          */
11713                         if (!ioc->is_driver_loading) {
11714                         /* TODO-- Need to find out whether this condition will
11715                          * occur or not
11716                          */
11717                                 _scsih_pcie_device_remove(ioc, pcie_device);
11718                                 pcie_device_put(pcie_device);
11719                                 continue;
11720                         }
11721                 }
11722                 pcie_device_make_active(ioc, pcie_device);
11723                 pcie_device_put(pcie_device);
11724         }
11725 }
11726
11727 /**
11728  * _scsih_probe_devices - probing for devices
11729  * @ioc: per adapter object
11730  *
11731  * Called during initial loading of the driver.
11732  */
11733 static void
11734 _scsih_probe_devices(struct MPT3SAS_ADAPTER *ioc)
11735 {
11736         u16 volume_mapping_flags;
11737
11738         if (!(ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR))
11739                 return;  /* return when IOC doesn't support initiator mode */
11740
11741         _scsih_probe_boot_devices(ioc);
11742
11743         if (ioc->ir_firmware) {
11744                 volume_mapping_flags =
11745                     le16_to_cpu(ioc->ioc_pg8.IRVolumeMappingFlags) &
11746                     MPI2_IOCPAGE8_IRFLAGS_MASK_VOLUME_MAPPING_MODE;
11747                 if (volume_mapping_flags ==
11748                     MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING) {
11749                         _scsih_probe_raid(ioc);
11750                         _scsih_probe_sas(ioc);
11751                 } else {
11752                         _scsih_probe_sas(ioc);
11753                         _scsih_probe_raid(ioc);
11754                 }
11755         } else {
11756                 _scsih_probe_sas(ioc);
11757                 _scsih_probe_pcie(ioc);
11758         }
11759 }
11760
11761 /**
11762  * scsih_scan_start - scsi lld callback for .scan_start
11763  * @shost: SCSI host pointer
11764  *
11765  * The shost has the ability to discover targets on its own instead
11766  * of scanning the entire bus.  In our implemention, we will kick off
11767  * firmware discovery.
11768  */
11769 static void
11770 scsih_scan_start(struct Scsi_Host *shost)
11771 {
11772         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
11773         int rc;
11774         if (diag_buffer_enable != -1 && diag_buffer_enable != 0)
11775                 mpt3sas_enable_diag_buffer(ioc, diag_buffer_enable);
11776         else if (ioc->manu_pg11.HostTraceBufferMaxSizeKB != 0)
11777                 mpt3sas_enable_diag_buffer(ioc, 1);
11778
11779         if (disable_discovery > 0)
11780                 return;
11781
11782         ioc->start_scan = 1;
11783         rc = mpt3sas_port_enable(ioc);
11784
11785         if (rc != 0)
11786                 ioc_info(ioc, "port enable: FAILED\n");
11787 }
11788
11789 /**
11790  * _scsih_complete_devices_scanning - add the devices to sml and
11791  * complete ioc initialization.
11792  * @ioc: per adapter object
11793  *
11794  * Return nothing.
11795  */
11796 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc)
11797 {
11798
11799         if (ioc->wait_for_discovery_to_complete) {
11800                 ioc->wait_for_discovery_to_complete = 0;
11801                 _scsih_probe_devices(ioc);
11802         }
11803
11804         mpt3sas_base_start_watchdog(ioc);
11805         ioc->is_driver_loading = 0;
11806 }
11807
11808 /**
11809  * scsih_scan_finished - scsi lld callback for .scan_finished
11810  * @shost: SCSI host pointer
11811  * @time: elapsed time of the scan in jiffies
11812  *
11813  * This function will be called periodicallyn until it returns 1 with the
11814  * scsi_host and the elapsed time of the scan in jiffies. In our implemention,
11815  * we wait for firmware discovery to complete, then return 1.
11816  */
11817 static int
11818 scsih_scan_finished(struct Scsi_Host *shost, unsigned long time)
11819 {
11820         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
11821         u32 ioc_state;
11822         int issue_hard_reset = 0;
11823
11824         if (disable_discovery > 0) {
11825                 ioc->is_driver_loading = 0;
11826                 ioc->wait_for_discovery_to_complete = 0;
11827                 return 1;
11828         }
11829
11830         if (time >= (300 * HZ)) {
11831                 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11832                 ioc_info(ioc, "port enable: FAILED with timeout (timeout=300s)\n");
11833                 ioc->is_driver_loading = 0;
11834                 return 1;
11835         }
11836
11837         if (ioc->start_scan) {
11838                 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
11839                 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
11840                         mpt3sas_print_fault_code(ioc, ioc_state &
11841                             MPI2_DOORBELL_DATA_MASK);
11842                         issue_hard_reset = 1;
11843                         goto out;
11844                 } else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
11845                                 MPI2_IOC_STATE_COREDUMP) {
11846                         mpt3sas_base_coredump_info(ioc, ioc_state &
11847                             MPI2_DOORBELL_DATA_MASK);
11848                         mpt3sas_base_wait_for_coredump_completion(ioc, __func__);
11849                         issue_hard_reset = 1;
11850                         goto out;
11851                 }
11852                 return 0;
11853         }
11854
11855         if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) {
11856                 ioc_info(ioc,
11857                     "port enable: aborted due to diag reset\n");
11858                 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11859                 goto out;
11860         }
11861         if (ioc->start_scan_failed) {
11862                 ioc_info(ioc, "port enable: FAILED with (ioc_status=0x%08x)\n",
11863                          ioc->start_scan_failed);
11864                 ioc->is_driver_loading = 0;
11865                 ioc->wait_for_discovery_to_complete = 0;
11866                 ioc->remove_host = 1;
11867                 return 1;
11868         }
11869
11870         ioc_info(ioc, "port enable: SUCCESS\n");
11871         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11872         _scsih_complete_devices_scanning(ioc);
11873
11874 out:
11875         if (issue_hard_reset) {
11876                 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11877                 if (mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET))
11878                         ioc->is_driver_loading = 0;
11879         }
11880         return 1;
11881 }
11882
11883 /**
11884  * scsih_map_queues - map reply queues with request queues
11885  * @shost: SCSI host pointer
11886  */
11887 static void scsih_map_queues(struct Scsi_Host *shost)
11888 {
11889         struct MPT3SAS_ADAPTER *ioc =
11890             (struct MPT3SAS_ADAPTER *)shost->hostdata;
11891         struct blk_mq_queue_map *map;
11892         int i, qoff, offset;
11893         int nr_msix_vectors = ioc->iopoll_q_start_index;
11894         int iopoll_q_count = ioc->reply_queue_count - nr_msix_vectors;
11895
11896         if (shost->nr_hw_queues == 1)
11897                 return;
11898
11899         for (i = 0, qoff = 0; i < shost->nr_maps; i++) {
11900                 map = &shost->tag_set.map[i];
11901                 map->nr_queues = 0;
11902                 offset = 0;
11903                 if (i == HCTX_TYPE_DEFAULT) {
11904                         map->nr_queues =
11905                             nr_msix_vectors - ioc->high_iops_queues;
11906                         offset = ioc->high_iops_queues;
11907                 } else if (i == HCTX_TYPE_POLL)
11908                         map->nr_queues = iopoll_q_count;
11909
11910                 if (!map->nr_queues)
11911                         BUG_ON(i == HCTX_TYPE_DEFAULT);
11912
11913                 /*
11914                  * The poll queue(s) doesn't have an IRQ (and hence IRQ
11915                  * affinity), so use the regular blk-mq cpu mapping
11916                  */
11917                 map->queue_offset = qoff;
11918                 if (i != HCTX_TYPE_POLL)
11919                         blk_mq_pci_map_queues(map, ioc->pdev, offset);
11920                 else
11921                         blk_mq_map_queues(map);
11922
11923                 qoff += map->nr_queues;
11924         }
11925 }
11926
11927 /* shost template for SAS 2.0 HBA devices */
11928 static const struct scsi_host_template mpt2sas_driver_template = {
11929         .module                         = THIS_MODULE,
11930         .name                           = "Fusion MPT SAS Host",
11931         .proc_name                      = MPT2SAS_DRIVER_NAME,
11932         .queuecommand                   = scsih_qcmd,
11933         .target_alloc                   = scsih_target_alloc,
11934         .slave_alloc                    = scsih_slave_alloc,
11935         .slave_configure                = scsih_slave_configure,
11936         .target_destroy                 = scsih_target_destroy,
11937         .slave_destroy                  = scsih_slave_destroy,
11938         .scan_finished                  = scsih_scan_finished,
11939         .scan_start                     = scsih_scan_start,
11940         .change_queue_depth             = scsih_change_queue_depth,
11941         .eh_abort_handler               = scsih_abort,
11942         .eh_device_reset_handler        = scsih_dev_reset,
11943         .eh_target_reset_handler        = scsih_target_reset,
11944         .eh_host_reset_handler          = scsih_host_reset,
11945         .bios_param                     = scsih_bios_param,
11946         .can_queue                      = 1,
11947         .this_id                        = -1,
11948         .sg_tablesize                   = MPT2SAS_SG_DEPTH,
11949         .max_sectors                    = 32767,
11950         .cmd_per_lun                    = 7,
11951         .shost_groups                   = mpt3sas_host_groups,
11952         .sdev_groups                    = mpt3sas_dev_groups,
11953         .track_queue_depth              = 1,
11954         .cmd_size                       = sizeof(struct scsiio_tracker),
11955 };
11956
11957 /* raid transport support for SAS 2.0 HBA devices */
11958 static struct raid_function_template mpt2sas_raid_functions = {
11959         .cookie         = &mpt2sas_driver_template,
11960         .is_raid        = scsih_is_raid,
11961         .get_resync     = scsih_get_resync,
11962         .get_state      = scsih_get_state,
11963 };
11964
11965 /* shost template for SAS 3.0 HBA devices */
11966 static const struct scsi_host_template mpt3sas_driver_template = {
11967         .module                         = THIS_MODULE,
11968         .name                           = "Fusion MPT SAS Host",
11969         .proc_name                      = MPT3SAS_DRIVER_NAME,
11970         .queuecommand                   = scsih_qcmd,
11971         .target_alloc                   = scsih_target_alloc,
11972         .slave_alloc                    = scsih_slave_alloc,
11973         .slave_configure                = scsih_slave_configure,
11974         .target_destroy                 = scsih_target_destroy,
11975         .slave_destroy                  = scsih_slave_destroy,
11976         .scan_finished                  = scsih_scan_finished,
11977         .scan_start                     = scsih_scan_start,
11978         .change_queue_depth             = scsih_change_queue_depth,
11979         .eh_abort_handler               = scsih_abort,
11980         .eh_device_reset_handler        = scsih_dev_reset,
11981         .eh_target_reset_handler        = scsih_target_reset,
11982         .eh_host_reset_handler          = scsih_host_reset,
11983         .bios_param                     = scsih_bios_param,
11984         .can_queue                      = 1,
11985         .this_id                        = -1,
11986         .sg_tablesize                   = MPT3SAS_SG_DEPTH,
11987         .max_sectors                    = 32767,
11988         .max_segment_size               = 0xffffffff,
11989         .cmd_per_lun                    = 128,
11990         .shost_groups                   = mpt3sas_host_groups,
11991         .sdev_groups                    = mpt3sas_dev_groups,
11992         .track_queue_depth              = 1,
11993         .cmd_size                       = sizeof(struct scsiio_tracker),
11994         .map_queues                     = scsih_map_queues,
11995         .mq_poll                        = mpt3sas_blk_mq_poll,
11996 };
11997
11998 /* raid transport support for SAS 3.0 HBA devices */
11999 static struct raid_function_template mpt3sas_raid_functions = {
12000         .cookie         = &mpt3sas_driver_template,
12001         .is_raid        = scsih_is_raid,
12002         .get_resync     = scsih_get_resync,
12003         .get_state      = scsih_get_state,
12004 };
12005
12006 /**
12007  * _scsih_determine_hba_mpi_version - determine in which MPI version class
12008  *                                      this device belongs to.
12009  * @pdev: PCI device struct
12010  *
12011  * return MPI2_VERSION for SAS 2.0 HBA devices,
12012  *      MPI25_VERSION for SAS 3.0 HBA devices, and
12013  *      MPI26 VERSION for Cutlass & Invader SAS 3.0 HBA devices
12014  */
12015 static u16
12016 _scsih_determine_hba_mpi_version(struct pci_dev *pdev)
12017 {
12018
12019         switch (pdev->device) {
12020         case MPI2_MFGPAGE_DEVID_SSS6200:
12021         case MPI2_MFGPAGE_DEVID_SAS2004:
12022         case MPI2_MFGPAGE_DEVID_SAS2008:
12023         case MPI2_MFGPAGE_DEVID_SAS2108_1:
12024         case MPI2_MFGPAGE_DEVID_SAS2108_2:
12025         case MPI2_MFGPAGE_DEVID_SAS2108_3:
12026         case MPI2_MFGPAGE_DEVID_SAS2116_1:
12027         case MPI2_MFGPAGE_DEVID_SAS2116_2:
12028         case MPI2_MFGPAGE_DEVID_SAS2208_1:
12029         case MPI2_MFGPAGE_DEVID_SAS2208_2:
12030         case MPI2_MFGPAGE_DEVID_SAS2208_3:
12031         case MPI2_MFGPAGE_DEVID_SAS2208_4:
12032         case MPI2_MFGPAGE_DEVID_SAS2208_5:
12033         case MPI2_MFGPAGE_DEVID_SAS2208_6:
12034         case MPI2_MFGPAGE_DEVID_SAS2308_1:
12035         case MPI2_MFGPAGE_DEVID_SAS2308_2:
12036         case MPI2_MFGPAGE_DEVID_SAS2308_3:
12037         case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP:
12038         case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1:
12039                 return MPI2_VERSION;
12040         case MPI25_MFGPAGE_DEVID_SAS3004:
12041         case MPI25_MFGPAGE_DEVID_SAS3008:
12042         case MPI25_MFGPAGE_DEVID_SAS3108_1:
12043         case MPI25_MFGPAGE_DEVID_SAS3108_2:
12044         case MPI25_MFGPAGE_DEVID_SAS3108_5:
12045         case MPI25_MFGPAGE_DEVID_SAS3108_6:
12046                 return MPI25_VERSION;
12047         case MPI26_MFGPAGE_DEVID_SAS3216:
12048         case MPI26_MFGPAGE_DEVID_SAS3224:
12049         case MPI26_MFGPAGE_DEVID_SAS3316_1:
12050         case MPI26_MFGPAGE_DEVID_SAS3316_2:
12051         case MPI26_MFGPAGE_DEVID_SAS3316_3:
12052         case MPI26_MFGPAGE_DEVID_SAS3316_4:
12053         case MPI26_MFGPAGE_DEVID_SAS3324_1:
12054         case MPI26_MFGPAGE_DEVID_SAS3324_2:
12055         case MPI26_MFGPAGE_DEVID_SAS3324_3:
12056         case MPI26_MFGPAGE_DEVID_SAS3324_4:
12057         case MPI26_MFGPAGE_DEVID_SAS3508:
12058         case MPI26_MFGPAGE_DEVID_SAS3508_1:
12059         case MPI26_MFGPAGE_DEVID_SAS3408:
12060         case MPI26_MFGPAGE_DEVID_SAS3516:
12061         case MPI26_MFGPAGE_DEVID_SAS3516_1:
12062         case MPI26_MFGPAGE_DEVID_SAS3416:
12063         case MPI26_MFGPAGE_DEVID_SAS3616:
12064         case MPI26_ATLAS_PCIe_SWITCH_DEVID:
12065         case MPI26_MFGPAGE_DEVID_CFG_SEC_3916:
12066         case MPI26_MFGPAGE_DEVID_HARD_SEC_3916:
12067         case MPI26_MFGPAGE_DEVID_CFG_SEC_3816:
12068         case MPI26_MFGPAGE_DEVID_HARD_SEC_3816:
12069         case MPI26_MFGPAGE_DEVID_INVALID0_3916:
12070         case MPI26_MFGPAGE_DEVID_INVALID1_3916:
12071         case MPI26_MFGPAGE_DEVID_INVALID0_3816:
12072         case MPI26_MFGPAGE_DEVID_INVALID1_3816:
12073                 return MPI26_VERSION;
12074         }
12075         return 0;
12076 }
12077
12078 /**
12079  * _scsih_probe - attach and add scsi host
12080  * @pdev: PCI device struct
12081  * @id: pci device id
12082  *
12083  * Return: 0 success, anything else error.
12084  */
12085 static int
12086 _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
12087 {
12088         struct MPT3SAS_ADAPTER *ioc;
12089         struct Scsi_Host *shost = NULL;
12090         int rv;
12091         u16 hba_mpi_version;
12092         int iopoll_q_count = 0;
12093
12094         /* Determine in which MPI version class this pci device belongs */
12095         hba_mpi_version = _scsih_determine_hba_mpi_version(pdev);
12096         if (hba_mpi_version == 0)
12097                 return -ENODEV;
12098
12099         /* Enumerate only SAS 2.0 HBA's if hbas_to_enumerate is one,
12100          * for other generation HBA's return with -ENODEV
12101          */
12102         if ((hbas_to_enumerate == 1) && (hba_mpi_version !=  MPI2_VERSION))
12103                 return -ENODEV;
12104
12105         /* Enumerate only SAS 3.0 HBA's if hbas_to_enumerate is two,
12106          * for other generation HBA's return with -ENODEV
12107          */
12108         if ((hbas_to_enumerate == 2) && (!(hba_mpi_version ==  MPI25_VERSION
12109                 || hba_mpi_version ==  MPI26_VERSION)))
12110                 return -ENODEV;
12111
12112         switch (hba_mpi_version) {
12113         case MPI2_VERSION:
12114                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
12115                         PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM);
12116                 /* Use mpt2sas driver host template for SAS 2.0 HBA's */
12117                 shost = scsi_host_alloc(&mpt2sas_driver_template,
12118                   sizeof(struct MPT3SAS_ADAPTER));
12119                 if (!shost)
12120                         return -ENODEV;
12121                 ioc = shost_priv(shost);
12122                 memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));
12123                 ioc->hba_mpi_version_belonged = hba_mpi_version;
12124                 ioc->id = mpt2_ids++;
12125                 sprintf(ioc->driver_name, "%s", MPT2SAS_DRIVER_NAME);
12126                 switch (pdev->device) {
12127                 case MPI2_MFGPAGE_DEVID_SSS6200:
12128                         ioc->is_warpdrive = 1;
12129                         ioc->hide_ir_msg = 1;
12130                         break;
12131                 case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP:
12132                 case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1:
12133                         ioc->is_mcpu_endpoint = 1;
12134                         break;
12135                 default:
12136                         ioc->mfg_pg10_hide_flag = MFG_PAGE10_EXPOSE_ALL_DISKS;
12137                         break;
12138                 }
12139
12140                 if (multipath_on_hba == -1 || multipath_on_hba == 0)
12141                         ioc->multipath_on_hba = 0;
12142                 else
12143                         ioc->multipath_on_hba = 1;
12144
12145                 break;
12146         case MPI25_VERSION:
12147         case MPI26_VERSION:
12148                 /* Use mpt3sas driver host template for SAS 3.0 HBA's */
12149                 shost = scsi_host_alloc(&mpt3sas_driver_template,
12150                   sizeof(struct MPT3SAS_ADAPTER));
12151                 if (!shost)
12152                         return -ENODEV;
12153                 ioc = shost_priv(shost);
12154                 memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));
12155                 ioc->hba_mpi_version_belonged = hba_mpi_version;
12156                 ioc->id = mpt3_ids++;
12157                 sprintf(ioc->driver_name, "%s", MPT3SAS_DRIVER_NAME);
12158                 switch (pdev->device) {
12159                 case MPI26_MFGPAGE_DEVID_SAS3508:
12160                 case MPI26_MFGPAGE_DEVID_SAS3508_1:
12161                 case MPI26_MFGPAGE_DEVID_SAS3408:
12162                 case MPI26_MFGPAGE_DEVID_SAS3516:
12163                 case MPI26_MFGPAGE_DEVID_SAS3516_1:
12164                 case MPI26_MFGPAGE_DEVID_SAS3416:
12165                 case MPI26_MFGPAGE_DEVID_SAS3616:
12166                 case MPI26_ATLAS_PCIe_SWITCH_DEVID:
12167                         ioc->is_gen35_ioc = 1;
12168                         break;
12169                 case MPI26_MFGPAGE_DEVID_INVALID0_3816:
12170                 case MPI26_MFGPAGE_DEVID_INVALID0_3916:
12171                         dev_err(&pdev->dev,
12172                             "HBA with DeviceId 0x%04x, sub VendorId 0x%04x, sub DeviceId 0x%04x is Invalid",
12173                             pdev->device, pdev->subsystem_vendor,
12174                             pdev->subsystem_device);
12175                         return 1;
12176                 case MPI26_MFGPAGE_DEVID_INVALID1_3816:
12177                 case MPI26_MFGPAGE_DEVID_INVALID1_3916:
12178                         dev_err(&pdev->dev,
12179                             "HBA with DeviceId 0x%04x, sub VendorId 0x%04x, sub DeviceId 0x%04x is Tampered",
12180                             pdev->device, pdev->subsystem_vendor,
12181                             pdev->subsystem_device);
12182                         return 1;
12183                 case MPI26_MFGPAGE_DEVID_CFG_SEC_3816:
12184                 case MPI26_MFGPAGE_DEVID_CFG_SEC_3916:
12185                         dev_info(&pdev->dev,
12186                             "HBA is in Configurable Secure mode\n");
12187                         fallthrough;
12188                 case MPI26_MFGPAGE_DEVID_HARD_SEC_3816:
12189                 case MPI26_MFGPAGE_DEVID_HARD_SEC_3916:
12190                         ioc->is_aero_ioc = ioc->is_gen35_ioc = 1;
12191                         break;
12192                 default:
12193                         ioc->is_gen35_ioc = ioc->is_aero_ioc = 0;
12194                 }
12195                 if ((ioc->hba_mpi_version_belonged == MPI25_VERSION &&
12196                         pdev->revision >= SAS3_PCI_DEVICE_C0_REVISION) ||
12197                         (ioc->hba_mpi_version_belonged == MPI26_VERSION)) {
12198                         ioc->combined_reply_queue = 1;
12199                         if (ioc->is_gen35_ioc)
12200                                 ioc->combined_reply_index_count =
12201                                  MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35;
12202                         else
12203                                 ioc->combined_reply_index_count =
12204                                  MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3;
12205                 }
12206
12207                 switch (ioc->is_gen35_ioc) {
12208                 case 0:
12209                         if (multipath_on_hba == -1 || multipath_on_hba == 0)
12210                                 ioc->multipath_on_hba = 0;
12211                         else
12212                                 ioc->multipath_on_hba = 1;
12213                         break;
12214                 case 1:
12215                         if (multipath_on_hba == -1 || multipath_on_hba > 0)
12216                                 ioc->multipath_on_hba = 1;
12217                         else
12218                                 ioc->multipath_on_hba = 0;
12219                         break;
12220                 default:
12221                         break;
12222                 }
12223
12224                 break;
12225         default:
12226                 return -ENODEV;
12227         }
12228
12229         INIT_LIST_HEAD(&ioc->list);
12230         spin_lock(&gioc_lock);
12231         list_add_tail(&ioc->list, &mpt3sas_ioc_list);
12232         spin_unlock(&gioc_lock);
12233         ioc->shost = shost;
12234         ioc->pdev = pdev;
12235         ioc->scsi_io_cb_idx = scsi_io_cb_idx;
12236         ioc->tm_cb_idx = tm_cb_idx;
12237         ioc->ctl_cb_idx = ctl_cb_idx;
12238         ioc->base_cb_idx = base_cb_idx;
12239         ioc->port_enable_cb_idx = port_enable_cb_idx;
12240         ioc->transport_cb_idx = transport_cb_idx;
12241         ioc->scsih_cb_idx = scsih_cb_idx;
12242         ioc->config_cb_idx = config_cb_idx;
12243         ioc->tm_tr_cb_idx = tm_tr_cb_idx;
12244         ioc->tm_tr_volume_cb_idx = tm_tr_volume_cb_idx;
12245         ioc->tm_sas_control_cb_idx = tm_sas_control_cb_idx;
12246         ioc->logging_level = logging_level;
12247         ioc->schedule_dead_ioc_flush_running_cmds = &_scsih_flush_running_cmds;
12248         /* Host waits for minimum of six seconds */
12249         ioc->max_shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;
12250         /*
12251          * Enable MEMORY MOVE support flag.
12252          */
12253         ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_MEMMOVE;
12254         /* Enable ADDITIONAL QUERY support flag. */
12255         ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY;
12256
12257         ioc->enable_sdev_max_qd = enable_sdev_max_qd;
12258
12259         /* misc semaphores and spin locks */
12260         mutex_init(&ioc->reset_in_progress_mutex);
12261         /* initializing pci_access_mutex lock */
12262         mutex_init(&ioc->pci_access_mutex);
12263         spin_lock_init(&ioc->ioc_reset_in_progress_lock);
12264         spin_lock_init(&ioc->scsi_lookup_lock);
12265         spin_lock_init(&ioc->sas_device_lock);
12266         spin_lock_init(&ioc->sas_node_lock);
12267         spin_lock_init(&ioc->fw_event_lock);
12268         spin_lock_init(&ioc->raid_device_lock);
12269         spin_lock_init(&ioc->pcie_device_lock);
12270         spin_lock_init(&ioc->diag_trigger_lock);
12271
12272         INIT_LIST_HEAD(&ioc->sas_device_list);
12273         INIT_LIST_HEAD(&ioc->sas_device_init_list);
12274         INIT_LIST_HEAD(&ioc->sas_expander_list);
12275         INIT_LIST_HEAD(&ioc->enclosure_list);
12276         INIT_LIST_HEAD(&ioc->pcie_device_list);
12277         INIT_LIST_HEAD(&ioc->pcie_device_init_list);
12278         INIT_LIST_HEAD(&ioc->fw_event_list);
12279         INIT_LIST_HEAD(&ioc->raid_device_list);
12280         INIT_LIST_HEAD(&ioc->sas_hba.sas_port_list);
12281         INIT_LIST_HEAD(&ioc->delayed_tr_list);
12282         INIT_LIST_HEAD(&ioc->delayed_sc_list);
12283         INIT_LIST_HEAD(&ioc->delayed_event_ack_list);
12284         INIT_LIST_HEAD(&ioc->delayed_tr_volume_list);
12285         INIT_LIST_HEAD(&ioc->reply_queue_list);
12286         INIT_LIST_HEAD(&ioc->port_table_list);
12287
12288         sprintf(ioc->name, "%s_cm%d", ioc->driver_name, ioc->id);
12289
12290         /* init shost parameters */
12291         shost->max_cmd_len = 32;
12292         shost->max_lun = max_lun;
12293         shost->transportt = mpt3sas_transport_template;
12294         shost->unique_id = ioc->id;
12295
12296         if (ioc->is_mcpu_endpoint) {
12297                 /* mCPU MPI support 64K max IO */
12298                 shost->max_sectors = 128;
12299                 ioc_info(ioc, "The max_sectors value is set to %d\n",
12300                          shost->max_sectors);
12301         } else {
12302                 if (max_sectors != 0xFFFF) {
12303                         if (max_sectors < 64) {
12304                                 shost->max_sectors = 64;
12305                                 ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767. Assigning value of 64.\n",
12306                                          max_sectors);
12307                         } else if (max_sectors > 32767) {
12308                                 shost->max_sectors = 32767;
12309                                 ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767.Assigning default value of 32767.\n",
12310                                          max_sectors);
12311                         } else {
12312                                 shost->max_sectors = max_sectors & 0xFFFE;
12313                                 ioc_info(ioc, "The max_sectors value is set to %d\n",
12314                                          shost->max_sectors);
12315                         }
12316                 }
12317         }
12318         /* register EEDP capabilities with SCSI layer */
12319         if (prot_mask >= 0)
12320                 scsi_host_set_prot(shost, (prot_mask & 0x07));
12321         else
12322                 scsi_host_set_prot(shost, SHOST_DIF_TYPE1_PROTECTION
12323                                    | SHOST_DIF_TYPE2_PROTECTION
12324                                    | SHOST_DIF_TYPE3_PROTECTION);
12325
12326         scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
12327
12328         /* event thread */
12329         snprintf(ioc->firmware_event_name, sizeof(ioc->firmware_event_name),
12330             "fw_event_%s%d", ioc->driver_name, ioc->id);
12331         ioc->firmware_event_thread = alloc_ordered_workqueue(
12332             ioc->firmware_event_name, 0);
12333         if (!ioc->firmware_event_thread) {
12334                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
12335                         __FILE__, __LINE__, __func__);
12336                 rv = -ENODEV;
12337                 goto out_thread_fail;
12338         }
12339
12340         shost->host_tagset = 0;
12341
12342         if (ioc->is_gen35_ioc && host_tagset_enable)
12343                 shost->host_tagset = 1;
12344
12345         ioc->is_driver_loading = 1;
12346         if ((mpt3sas_base_attach(ioc))) {
12347                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
12348                         __FILE__, __LINE__, __func__);
12349                 rv = -ENODEV;
12350                 goto out_attach_fail;
12351         }
12352
12353         if (ioc->is_warpdrive) {
12354                 if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_EXPOSE_ALL_DISKS)
12355                         ioc->hide_drives = 0;
12356                 else if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_HIDE_ALL_DISKS)
12357                         ioc->hide_drives = 1;
12358                 else {
12359                         if (mpt3sas_get_num_volumes(ioc))
12360                                 ioc->hide_drives = 1;
12361                         else
12362                                 ioc->hide_drives = 0;
12363                 }
12364         } else
12365                 ioc->hide_drives = 0;
12366
12367         shost->nr_hw_queues = 1;
12368
12369         if (shost->host_tagset) {
12370                 shost->nr_hw_queues =
12371                     ioc->reply_queue_count - ioc->high_iops_queues;
12372
12373                 iopoll_q_count =
12374                     ioc->reply_queue_count - ioc->iopoll_q_start_index;
12375
12376                 shost->nr_maps = iopoll_q_count ? 3 : 1;
12377
12378                 dev_info(&ioc->pdev->dev,
12379                     "Max SCSIIO MPT commands: %d shared with nr_hw_queues = %d\n",
12380                     shost->can_queue, shost->nr_hw_queues);
12381         }
12382
12383         rv = scsi_add_host(shost, &pdev->dev);
12384         if (rv) {
12385                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
12386                         __FILE__, __LINE__, __func__);
12387                 goto out_add_shost_fail;
12388         }
12389
12390         scsi_scan_host(shost);
12391         mpt3sas_setup_debugfs(ioc);
12392         return 0;
12393 out_add_shost_fail:
12394         mpt3sas_base_detach(ioc);
12395  out_attach_fail:
12396         destroy_workqueue(ioc->firmware_event_thread);
12397  out_thread_fail:
12398         spin_lock(&gioc_lock);
12399         list_del(&ioc->list);
12400         spin_unlock(&gioc_lock);
12401         scsi_host_put(shost);
12402         return rv;
12403 }
12404
12405 /**
12406  * scsih_suspend - power management suspend main entry point
12407  * @dev: Device struct
12408  *
12409  * Return: 0 success, anything else error.
12410  */
12411 static int __maybe_unused
12412 scsih_suspend(struct device *dev)
12413 {
12414         struct pci_dev *pdev = to_pci_dev(dev);
12415         struct Scsi_Host *shost;
12416         struct MPT3SAS_ADAPTER *ioc;
12417         int rc;
12418
12419         rc = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);
12420         if (rc)
12421                 return rc;
12422
12423         mpt3sas_base_stop_watchdog(ioc);
12424         scsi_block_requests(shost);
12425         _scsih_nvme_shutdown(ioc);
12426         ioc_info(ioc, "pdev=0x%p, slot=%s, entering operating state\n",
12427                  pdev, pci_name(pdev));
12428
12429         mpt3sas_base_free_resources(ioc);
12430         return 0;
12431 }
12432
12433 /**
12434  * scsih_resume - power management resume main entry point
12435  * @dev: Device struct
12436  *
12437  * Return: 0 success, anything else error.
12438  */
12439 static int __maybe_unused
12440 scsih_resume(struct device *dev)
12441 {
12442         struct pci_dev *pdev = to_pci_dev(dev);
12443         struct Scsi_Host *shost;
12444         struct MPT3SAS_ADAPTER *ioc;
12445         pci_power_t device_state = pdev->current_state;
12446         int r;
12447
12448         r = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);
12449         if (r)
12450                 return r;
12451
12452         ioc_info(ioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",
12453                  pdev, pci_name(pdev), device_state);
12454
12455         ioc->pdev = pdev;
12456         r = mpt3sas_base_map_resources(ioc);
12457         if (r)
12458                 return r;
12459         ioc_info(ioc, "Issuing Hard Reset as part of OS Resume\n");
12460         mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET);
12461         scsi_unblock_requests(shost);
12462         mpt3sas_base_start_watchdog(ioc);
12463         return 0;
12464 }
12465
12466 /**
12467  * scsih_pci_error_detected - Called when a PCI error is detected.
12468  * @pdev: PCI device struct
12469  * @state: PCI channel state
12470  *
12471  * Description: Called when a PCI error is detected.
12472  *
12473  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
12474  */
12475 static pci_ers_result_t
12476 scsih_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
12477 {
12478         struct Scsi_Host *shost;
12479         struct MPT3SAS_ADAPTER *ioc;
12480
12481         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12482                 return PCI_ERS_RESULT_DISCONNECT;
12483
12484         ioc_info(ioc, "PCI error: detected callback, state(%d)!!\n", state);
12485
12486         switch (state) {
12487         case pci_channel_io_normal:
12488                 return PCI_ERS_RESULT_CAN_RECOVER;
12489         case pci_channel_io_frozen:
12490                 /* Fatal error, prepare for slot reset */
12491                 ioc->pci_error_recovery = 1;
12492                 scsi_block_requests(ioc->shost);
12493                 mpt3sas_base_stop_watchdog(ioc);
12494                 mpt3sas_base_free_resources(ioc);
12495                 return PCI_ERS_RESULT_NEED_RESET;
12496         case pci_channel_io_perm_failure:
12497                 /* Permanent error, prepare for device removal */
12498                 ioc->pci_error_recovery = 1;
12499                 mpt3sas_base_stop_watchdog(ioc);
12500                 mpt3sas_base_pause_mq_polling(ioc);
12501                 _scsih_flush_running_cmds(ioc);
12502                 return PCI_ERS_RESULT_DISCONNECT;
12503         }
12504         return PCI_ERS_RESULT_NEED_RESET;
12505 }
12506
12507 /**
12508  * scsih_pci_slot_reset - Called when PCI slot has been reset.
12509  * @pdev: PCI device struct
12510  *
12511  * Description: This routine is called by the pci error recovery
12512  * code after the PCI slot has been reset, just before we
12513  * should resume normal operations.
12514  */
12515 static pci_ers_result_t
12516 scsih_pci_slot_reset(struct pci_dev *pdev)
12517 {
12518         struct Scsi_Host *shost;
12519         struct MPT3SAS_ADAPTER *ioc;
12520         int rc;
12521
12522         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12523                 return PCI_ERS_RESULT_DISCONNECT;
12524
12525         ioc_info(ioc, "PCI error: slot reset callback!!\n");
12526
12527         ioc->pci_error_recovery = 0;
12528         ioc->pdev = pdev;
12529         pci_restore_state(pdev);
12530         rc = mpt3sas_base_map_resources(ioc);
12531         if (rc)
12532                 return PCI_ERS_RESULT_DISCONNECT;
12533
12534         ioc_info(ioc, "Issuing Hard Reset as part of PCI Slot Reset\n");
12535         rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
12536
12537         ioc_warn(ioc, "hard reset: %s\n",
12538                  (rc == 0) ? "success" : "failed");
12539
12540         if (!rc)
12541                 return PCI_ERS_RESULT_RECOVERED;
12542         else
12543                 return PCI_ERS_RESULT_DISCONNECT;
12544 }
12545
12546 /**
12547  * scsih_pci_resume() - resume normal ops after PCI reset
12548  * @pdev: pointer to PCI device
12549  *
12550  * Called when the error recovery driver tells us that its
12551  * OK to resume normal operation. Use completion to allow
12552  * halted scsi ops to resume.
12553  */
12554 static void
12555 scsih_pci_resume(struct pci_dev *pdev)
12556 {
12557         struct Scsi_Host *shost;
12558         struct MPT3SAS_ADAPTER *ioc;
12559
12560         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12561                 return;
12562
12563         ioc_info(ioc, "PCI error: resume callback!!\n");
12564
12565         mpt3sas_base_start_watchdog(ioc);
12566         scsi_unblock_requests(ioc->shost);
12567 }
12568
12569 /**
12570  * scsih_pci_mmio_enabled - Enable MMIO and dump debug registers
12571  * @pdev: pointer to PCI device
12572  */
12573 static pci_ers_result_t
12574 scsih_pci_mmio_enabled(struct pci_dev *pdev)
12575 {
12576         struct Scsi_Host *shost;
12577         struct MPT3SAS_ADAPTER *ioc;
12578
12579         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12580                 return PCI_ERS_RESULT_DISCONNECT;
12581
12582         ioc_info(ioc, "PCI error: mmio enabled callback!!\n");
12583
12584         /* TODO - dump whatever for debugging purposes */
12585
12586         /* This called only if scsih_pci_error_detected returns
12587          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
12588          * works, no need to reset slot.
12589          */
12590         return PCI_ERS_RESULT_RECOVERED;
12591 }
12592
12593 /**
12594  * scsih_ncq_prio_supp - Check for NCQ command priority support
12595  * @sdev: scsi device struct
12596  *
12597  * This is called when a user indicates they would like to enable
12598  * ncq command priorities. This works only on SATA devices.
12599  */
12600 bool scsih_ncq_prio_supp(struct scsi_device *sdev)
12601 {
12602         struct scsi_vpd *vpd;
12603         bool ncq_prio_supp = false;
12604
12605         rcu_read_lock();
12606         vpd = rcu_dereference(sdev->vpd_pg89);
12607         if (!vpd || vpd->len < 214)
12608                 goto out;
12609
12610         ncq_prio_supp = (vpd->data[213] >> 4) & 1;
12611 out:
12612         rcu_read_unlock();
12613
12614         return ncq_prio_supp;
12615 }
12616 /*
12617  * The pci device ids are defined in mpi/mpi2_cnfg.h.
12618  */
12619 static const struct pci_device_id mpt3sas_pci_table[] = {
12620         /* Spitfire ~ 2004 */
12621         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2004,
12622                 PCI_ANY_ID, PCI_ANY_ID },
12623         /* Falcon ~ 2008 */
12624         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2008,
12625                 PCI_ANY_ID, PCI_ANY_ID },
12626         /* Liberator ~ 2108 */
12627         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_1,
12628                 PCI_ANY_ID, PCI_ANY_ID },
12629         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_2,
12630                 PCI_ANY_ID, PCI_ANY_ID },
12631         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_3,
12632                 PCI_ANY_ID, PCI_ANY_ID },
12633         /* Meteor ~ 2116 */
12634         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2116_1,
12635                 PCI_ANY_ID, PCI_ANY_ID },
12636         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2116_2,
12637                 PCI_ANY_ID, PCI_ANY_ID },
12638         /* Thunderbolt ~ 2208 */
12639         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_1,
12640                 PCI_ANY_ID, PCI_ANY_ID },
12641         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_2,
12642                 PCI_ANY_ID, PCI_ANY_ID },
12643         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_3,
12644                 PCI_ANY_ID, PCI_ANY_ID },
12645         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_4,
12646                 PCI_ANY_ID, PCI_ANY_ID },
12647         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_5,
12648                 PCI_ANY_ID, PCI_ANY_ID },
12649         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_6,
12650                 PCI_ANY_ID, PCI_ANY_ID },
12651         /* Mustang ~ 2308 */
12652         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_1,
12653                 PCI_ANY_ID, PCI_ANY_ID },
12654         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_2,
12655                 PCI_ANY_ID, PCI_ANY_ID },
12656         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_3,
12657                 PCI_ANY_ID, PCI_ANY_ID },
12658         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP,
12659                 PCI_ANY_ID, PCI_ANY_ID },
12660         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1,
12661                 PCI_ANY_ID, PCI_ANY_ID },
12662         /* SSS6200 */
12663         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SSS6200,
12664                 PCI_ANY_ID, PCI_ANY_ID },
12665         /* Fury ~ 3004 and 3008 */
12666         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3004,
12667                 PCI_ANY_ID, PCI_ANY_ID },
12668         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3008,
12669                 PCI_ANY_ID, PCI_ANY_ID },
12670         /* Invader ~ 3108 */
12671         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_1,
12672                 PCI_ANY_ID, PCI_ANY_ID },
12673         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_2,
12674                 PCI_ANY_ID, PCI_ANY_ID },
12675         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_5,
12676                 PCI_ANY_ID, PCI_ANY_ID },
12677         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_6,
12678                 PCI_ANY_ID, PCI_ANY_ID },
12679         /* Cutlass ~ 3216 and 3224 */
12680         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3216,
12681                 PCI_ANY_ID, PCI_ANY_ID },
12682         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3224,
12683                 PCI_ANY_ID, PCI_ANY_ID },
12684         /* Intruder ~ 3316 and 3324 */
12685         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_1,
12686                 PCI_ANY_ID, PCI_ANY_ID },
12687         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_2,
12688                 PCI_ANY_ID, PCI_ANY_ID },
12689         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_3,
12690                 PCI_ANY_ID, PCI_ANY_ID },
12691         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_4,
12692                 PCI_ANY_ID, PCI_ANY_ID },
12693         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_1,
12694                 PCI_ANY_ID, PCI_ANY_ID },
12695         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_2,
12696                 PCI_ANY_ID, PCI_ANY_ID },
12697         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_3,
12698                 PCI_ANY_ID, PCI_ANY_ID },
12699         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_4,
12700                 PCI_ANY_ID, PCI_ANY_ID },
12701         /* Ventura, Crusader, Harpoon & Tomcat ~ 3516, 3416, 3508 & 3408*/
12702         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508,
12703                 PCI_ANY_ID, PCI_ANY_ID },
12704         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508_1,
12705                 PCI_ANY_ID, PCI_ANY_ID },
12706         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3408,
12707                 PCI_ANY_ID, PCI_ANY_ID },
12708         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516,
12709                 PCI_ANY_ID, PCI_ANY_ID },
12710         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516_1,
12711                 PCI_ANY_ID, PCI_ANY_ID },
12712         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3416,
12713                 PCI_ANY_ID, PCI_ANY_ID },
12714         /* Mercator ~ 3616*/
12715         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3616,
12716                 PCI_ANY_ID, PCI_ANY_ID },
12717
12718         /* Aero SI 0x00E1 Configurable Secure
12719          * 0x00E2 Hard Secure
12720          */
12721         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_CFG_SEC_3916,
12722                 PCI_ANY_ID, PCI_ANY_ID },
12723         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3916,
12724                 PCI_ANY_ID, PCI_ANY_ID },
12725
12726         /*
12727          *  Aero SI –> 0x00E0 Invalid, 0x00E3 Tampered
12728          */
12729         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID0_3916,
12730                 PCI_ANY_ID, PCI_ANY_ID },
12731         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID1_3916,
12732                 PCI_ANY_ID, PCI_ANY_ID },
12733
12734         /* Atlas PCIe Switch Management Port */
12735         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_ATLAS_PCIe_SWITCH_DEVID,
12736                 PCI_ANY_ID, PCI_ANY_ID },
12737
12738         /* Sea SI 0x00E5 Configurable Secure
12739          * 0x00E6 Hard Secure
12740          */
12741         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_CFG_SEC_3816,
12742                 PCI_ANY_ID, PCI_ANY_ID },
12743         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3816,
12744                 PCI_ANY_ID, PCI_ANY_ID },
12745
12746         /*
12747          * ATTO Branded ExpressSAS H12xx GT
12748          */
12749         { MPI2_MFGPAGE_VENDORID_ATTO, MPI26_MFGPAGE_DEVID_HARD_SEC_3816,
12750                 PCI_ANY_ID, PCI_ANY_ID },
12751
12752         /*
12753          *  Sea SI –> 0x00E4 Invalid, 0x00E7 Tampered
12754          */
12755         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID0_3816,
12756                 PCI_ANY_ID, PCI_ANY_ID },
12757         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID1_3816,
12758                 PCI_ANY_ID, PCI_ANY_ID },
12759
12760         {0}     /* Terminating entry */
12761 };
12762 MODULE_DEVICE_TABLE(pci, mpt3sas_pci_table);
12763
12764 static struct pci_error_handlers _mpt3sas_err_handler = {
12765         .error_detected = scsih_pci_error_detected,
12766         .mmio_enabled   = scsih_pci_mmio_enabled,
12767         .slot_reset     = scsih_pci_slot_reset,
12768         .resume         = scsih_pci_resume,
12769 };
12770
12771 static SIMPLE_DEV_PM_OPS(scsih_pm_ops, scsih_suspend, scsih_resume);
12772
12773 static struct pci_driver mpt3sas_driver = {
12774         .name           = MPT3SAS_DRIVER_NAME,
12775         .id_table       = mpt3sas_pci_table,
12776         .probe          = _scsih_probe,
12777         .remove         = scsih_remove,
12778         .shutdown       = scsih_shutdown,
12779         .err_handler    = &_mpt3sas_err_handler,
12780         .driver.pm      = &scsih_pm_ops,
12781 };
12782
12783 /**
12784  * scsih_init - main entry point for this driver.
12785  *
12786  * Return: 0 success, anything else error.
12787  */
12788 static int
12789 scsih_init(void)
12790 {
12791         mpt2_ids = 0;
12792         mpt3_ids = 0;
12793
12794         mpt3sas_base_initialize_callback_handler();
12795
12796          /* queuecommand callback hander */
12797         scsi_io_cb_idx = mpt3sas_base_register_callback_handler(_scsih_io_done);
12798
12799         /* task management callback handler */
12800         tm_cb_idx = mpt3sas_base_register_callback_handler(_scsih_tm_done);
12801
12802         /* base internal commands callback handler */
12803         base_cb_idx = mpt3sas_base_register_callback_handler(mpt3sas_base_done);
12804         port_enable_cb_idx = mpt3sas_base_register_callback_handler(
12805             mpt3sas_port_enable_done);
12806
12807         /* transport internal commands callback handler */
12808         transport_cb_idx = mpt3sas_base_register_callback_handler(
12809             mpt3sas_transport_done);
12810
12811         /* scsih internal commands callback handler */
12812         scsih_cb_idx = mpt3sas_base_register_callback_handler(_scsih_done);
12813
12814         /* configuration page API internal commands callback handler */
12815         config_cb_idx = mpt3sas_base_register_callback_handler(
12816             mpt3sas_config_done);
12817
12818         /* ctl module callback handler */
12819         ctl_cb_idx = mpt3sas_base_register_callback_handler(mpt3sas_ctl_done);
12820
12821         tm_tr_cb_idx = mpt3sas_base_register_callback_handler(
12822             _scsih_tm_tr_complete);
12823
12824         tm_tr_volume_cb_idx = mpt3sas_base_register_callback_handler(
12825             _scsih_tm_volume_tr_complete);
12826
12827         tm_sas_control_cb_idx = mpt3sas_base_register_callback_handler(
12828             _scsih_sas_control_complete);
12829
12830         mpt3sas_init_debugfs();
12831         return 0;
12832 }
12833
12834 /**
12835  * scsih_exit - exit point for this driver (when it is a module).
12836  *
12837  * Return: 0 success, anything else error.
12838  */
12839 static void
12840 scsih_exit(void)
12841 {
12842
12843         mpt3sas_base_release_callback_handler(scsi_io_cb_idx);
12844         mpt3sas_base_release_callback_handler(tm_cb_idx);
12845         mpt3sas_base_release_callback_handler(base_cb_idx);
12846         mpt3sas_base_release_callback_handler(port_enable_cb_idx);
12847         mpt3sas_base_release_callback_handler(transport_cb_idx);
12848         mpt3sas_base_release_callback_handler(scsih_cb_idx);
12849         mpt3sas_base_release_callback_handler(config_cb_idx);
12850         mpt3sas_base_release_callback_handler(ctl_cb_idx);
12851
12852         mpt3sas_base_release_callback_handler(tm_tr_cb_idx);
12853         mpt3sas_base_release_callback_handler(tm_tr_volume_cb_idx);
12854         mpt3sas_base_release_callback_handler(tm_sas_control_cb_idx);
12855
12856 /* raid transport support */
12857         if (hbas_to_enumerate != 1)
12858                 raid_class_release(mpt3sas_raid_template);
12859         if (hbas_to_enumerate != 2)
12860                 raid_class_release(mpt2sas_raid_template);
12861         sas_release_transport(mpt3sas_transport_template);
12862         mpt3sas_exit_debugfs();
12863 }
12864
12865 /**
12866  * _mpt3sas_init - main entry point for this driver.
12867  *
12868  * Return: 0 success, anything else error.
12869  */
12870 static int __init
12871 _mpt3sas_init(void)
12872 {
12873         int error;
12874
12875         pr_info("%s version %s loaded\n", MPT3SAS_DRIVER_NAME,
12876                                         MPT3SAS_DRIVER_VERSION);
12877
12878         mpt3sas_transport_template =
12879             sas_attach_transport(&mpt3sas_transport_functions);
12880         if (!mpt3sas_transport_template)
12881                 return -ENODEV;
12882
12883         /* No need attach mpt3sas raid functions template
12884          * if hbas_to_enumarate value is one.
12885          */
12886         if (hbas_to_enumerate != 1) {
12887                 mpt3sas_raid_template =
12888                                 raid_class_attach(&mpt3sas_raid_functions);
12889                 if (!mpt3sas_raid_template) {
12890                         sas_release_transport(mpt3sas_transport_template);
12891                         return -ENODEV;
12892                 }
12893         }
12894
12895         /* No need to attach mpt2sas raid functions template
12896          * if hbas_to_enumarate value is two
12897          */
12898         if (hbas_to_enumerate != 2) {
12899                 mpt2sas_raid_template =
12900                                 raid_class_attach(&mpt2sas_raid_functions);
12901                 if (!mpt2sas_raid_template) {
12902                         sas_release_transport(mpt3sas_transport_template);
12903                         return -ENODEV;
12904                 }
12905         }
12906
12907         error = scsih_init();
12908         if (error) {
12909                 scsih_exit();
12910                 return error;
12911         }
12912
12913         mpt3sas_ctl_init(hbas_to_enumerate);
12914
12915         error = pci_register_driver(&mpt3sas_driver);
12916         if (error)
12917                 scsih_exit();
12918
12919         return error;
12920 }
12921
12922 /**
12923  * _mpt3sas_exit - exit point for this driver (when it is a module).
12924  *
12925  */
12926 static void __exit
12927 _mpt3sas_exit(void)
12928 {
12929         pr_info("mpt3sas version %s unloading\n",
12930                                 MPT3SAS_DRIVER_VERSION);
12931
12932         pci_unregister_driver(&mpt3sas_driver);
12933
12934         mpt3sas_ctl_exit(hbas_to_enumerate);
12935
12936         scsih_exit();
12937 }
12938
12939 module_init(_mpt3sas_init);
12940 module_exit(_mpt3sas_exit);