drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2021 Broadcom Inc.
   6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 #if defined(writeq) && defined(CONFIG_64BIT)
  14 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  15 {
  16         writeq(b, addr);
  17 }
  18 #else
  19 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  20 {
  21         __u64 data_out = b;
  22
  23         writel((u32)(data_out), addr);
  24         writel((u32)(data_out >> 32), (addr + 4));
  25 }
  26 #endif
  27
  28 static inline bool
  29 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  30 {
  31         u16 pi, ci, max_entries;
  32         bool is_qfull = false;
  33
  34         pi = op_req_q->pi;
  35         ci = READ_ONCE(op_req_q->ci);
  36         max_entries = op_req_q->num_requests;
  37
  38         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  39                 is_qfull = true;
  40
  41         return is_qfull;
  42 }
  43
  44 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  45 {
  46         u16 i, max_vectors;
  47
  48         max_vectors = mrioc->intr_info_count;
  49
  50         for (i = 0; i < max_vectors; i++)
  51                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  52 }
  53
  54 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  55 {
  56         mrioc->intr_enabled = 0;
  57         mpi3mr_sync_irqs(mrioc);
  58 }
  59
  60 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  61 {
  62         mrioc->intr_enabled = 1;
  63 }
  64
  65 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  66 {
  67         u16 i;
  68
  69         mpi3mr_ioc_disable_intr(mrioc);
  70
  71         if (!mrioc->intr_info)
  72                 return;
  73
  74         for (i = 0; i < mrioc->intr_info_count; i++)
  75                 free_irq(pci_irq_vector(mrioc->pdev, i),
  76                     (mrioc->intr_info + i));
  77
  78         kfree(mrioc->intr_info);
  79         mrioc->intr_info = NULL;
  80         mrioc->intr_info_count = 0;
  81         pci_free_irq_vectors(mrioc->pdev);
  82 }
  83
  84 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  85         dma_addr_t dma_addr)
  86 {
  87         struct mpi3_sge_common *sgel = paddr;
  88
  89         sgel->flags = flags;
  90         sgel->length = cpu_to_le32(length);
  91         sgel->address = cpu_to_le64(dma_addr);
  92 }
  93
  94 void mpi3mr_build_zero_len_sge(void *paddr)
  95 {
  96         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
  97
  98         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
  99 }
 100
 101 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 102         dma_addr_t phys_addr)
 103 {
 104         if (!phys_addr)
 105                 return NULL;
 106
 107         if ((phys_addr < mrioc->reply_buf_dma) ||
 108             (phys_addr > mrioc->reply_buf_dma_max_address))
 109                 return NULL;
 110
 111         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 112 }
 113
 114 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 121 }
 122
 123 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 124         u64 reply_dma)
 125 {
 126         u32 old_idx = 0;
 127         unsigned long flags;
 128
 129         spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
 130         old_idx  =  mrioc->reply_free_queue_host_index;
 131         mrioc->reply_free_queue_host_index = (
 132             (mrioc->reply_free_queue_host_index ==
 133             (mrioc->reply_free_qsz - 1)) ? 0 :
 134             (mrioc->reply_free_queue_host_index + 1));
 135         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 136         writel(mrioc->reply_free_queue_host_index,
 137             &mrioc->sysif_regs->reply_free_host_index);
 138         spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
 139 }
 140
 141 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 142         u64 sense_buf_dma)
 143 {
 144         u32 old_idx = 0;
 145         unsigned long flags;
 146
 147         spin_lock_irqsave(&mrioc->sbq_lock, flags);
 148         old_idx  =  mrioc->sbq_host_index;
 149         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 150             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 151             (mrioc->sbq_host_index + 1));
 152         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 153         writel(mrioc->sbq_host_index,
 154             &mrioc->sysif_regs->sense_buffer_free_host_index);
 155         spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
 156 }
 157
 158 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 159         struct mpi3_event_notification_reply *event_reply)
 160 {
 161         char *desc = NULL;
 162         u16 event;
 163
 164         event = event_reply->event;
 165
 166         switch (event) {
 167         case MPI3_EVENT_LOG_DATA:
 168                 desc = "Log Data";
 169                 break;
 170         case MPI3_EVENT_CHANGE:
 171                 desc = "Event Change";
 172                 break;
 173         case MPI3_EVENT_GPIO_INTERRUPT:
 174                 desc = "GPIO Interrupt";
 175                 break;
 176         case MPI3_EVENT_TEMP_THRESHOLD:
 177                 desc = "Temperature Threshold";
 178                 break;
 179         case MPI3_EVENT_CABLE_MGMT:
 180                 desc = "Cable Management";
 181                 break;
 182         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 183                 desc = "Energy Pack Change";
 184                 break;
 185         case MPI3_EVENT_DEVICE_ADDED:
 186         {
 187                 struct mpi3_device_page0 *event_data =
 188                     (struct mpi3_device_page0 *)event_reply->event_data;
 189                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 190                     event_data->dev_handle, event_data->device_form);
 191                 return;
 192         }
 193         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 194         {
 195                 struct mpi3_device_page0 *event_data =
 196                     (struct mpi3_device_page0 *)event_reply->event_data;
 197                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 198                     event_data->dev_handle, event_data->device_form);
 199                 return;
 200         }
 201         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 202         {
 203                 struct mpi3_event_data_device_status_change *event_data =
 204                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 205                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 206                     event_data->dev_handle, event_data->reason_code);
 207                 return;
 208         }
 209         case MPI3_EVENT_SAS_DISCOVERY:
 210         {
 211                 struct mpi3_event_data_sas_discovery *event_data =
 212                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 213                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 214                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 215                     "start" : "stop",
 216                     le32_to_cpu(event_data->discovery_status));
 217                 return;
 218         }
 219         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 220                 desc = "SAS Broadcast Primitive";
 221                 break;
 222         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 223                 desc = "SAS Notify Primitive";
 224                 break;
 225         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 226                 desc = "SAS Init Device Status Change";
 227                 break;
 228         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 229                 desc = "SAS Init Table Overflow";
 230                 break;
 231         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 232                 desc = "SAS Topology Change List";
 233                 break;
 234         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 235                 desc = "Enclosure Device Status Change";
 236                 break;
 237         case MPI3_EVENT_HARD_RESET_RECEIVED:
 238                 desc = "Hard Reset Received";
 239                 break;
 240         case MPI3_EVENT_SAS_PHY_COUNTER:
 241                 desc = "SAS PHY Counter";
 242                 break;
 243         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 244                 desc = "SAS Device Discovery Error";
 245                 break;
 246         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 247                 desc = "PCIE Topology Change List";
 248                 break;
 249         case MPI3_EVENT_PCIE_ENUMERATION:
 250         {
 251                 struct mpi3_event_data_pcie_enumeration *event_data =
 252                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 253                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 254                     (event_data->reason_code ==
 255                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 256                 if (event_data->enumeration_status)
 257                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 258                             le32_to_cpu(event_data->enumeration_status));
 259                 return;
 260         }
 261         case MPI3_EVENT_PREPARE_FOR_RESET:
 262                 desc = "Prepare For Reset";
 263                 break;
 264         }
 265
 266         if (!desc)
 267                 return;
 268
 269         ioc_info(mrioc, "%s\n", desc);
 270 }
 271
 272 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 273         struct mpi3_default_reply *def_reply)
 274 {
 275         struct mpi3_event_notification_reply *event_reply =
 276             (struct mpi3_event_notification_reply *)def_reply;
 277
 278         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 279         mpi3mr_print_event_data(mrioc, event_reply);
 280         mpi3mr_os_handle_events(mrioc, event_reply);
 281 }
 282
 283 static struct mpi3mr_drv_cmd *
 284 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 285         struct mpi3_default_reply *def_reply)
 286 {
 287         u16 idx;
 288
 289         switch (host_tag) {
 290         case MPI3MR_HOSTTAG_INITCMDS:
 291                 return &mrioc->init_cmds;
 292         case MPI3MR_HOSTTAG_BLK_TMS:
 293                 return &mrioc->host_tm_cmds;
 294         case MPI3MR_HOSTTAG_INVALID:
 295                 if (def_reply && def_reply->function ==
 296                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 297                         mpi3mr_handle_events(mrioc, def_reply);
 298                 return NULL;
 299         default:
 300                 break;
 301         }
 302         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 303             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 304                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 305                 return &mrioc->dev_rmhs_cmds[idx];
 306         }
 307
 308         return NULL;
 309 }
 310
 311 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 312         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 313 {
 314         u16 reply_desc_type, host_tag = 0;
 315         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 316         u32 ioc_loginfo = 0;
 317         struct mpi3_status_reply_descriptor *status_desc;
 318         struct mpi3_address_reply_descriptor *addr_desc;
 319         struct mpi3_success_reply_descriptor *success_desc;
 320         struct mpi3_default_reply *def_reply = NULL;
 321         struct mpi3mr_drv_cmd *cmdptr = NULL;
 322         struct mpi3_scsi_io_reply *scsi_reply;
 323         u8 *sense_buf = NULL;
 324
 325         *reply_dma = 0;
 326         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 327             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 328         switch (reply_desc_type) {
 329         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 330                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 331                 host_tag = le16_to_cpu(status_desc->host_tag);
 332                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 333                 if (ioc_status &
 334                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 335                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 336                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 337                 break;
 338         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 339                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 340                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 341                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 342                 if (!def_reply)
 343                         goto out;
 344                 host_tag = le16_to_cpu(def_reply->host_tag);
 345                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 346                 if (ioc_status &
 347                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 348                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 349                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 350                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 351                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 352                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 353                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 354                 }
 355                 break;
 356         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 357                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 358                 host_tag = le16_to_cpu(success_desc->host_tag);
 359                 break;
 360         default:
 361                 break;
 362         }
 363
 364         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 365         if (cmdptr) {
 366                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 367                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 368                         cmdptr->ioc_loginfo = ioc_loginfo;
 369                         cmdptr->ioc_status = ioc_status;
 370                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 371                         if (def_reply) {
 372                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 373                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 374                                     mrioc->facts.reply_sz);
 375                         }
 376                         if (cmdptr->is_waiting) {
 377                                 complete(&cmdptr->done);
 378                                 cmdptr->is_waiting = 0;
 379                         } else if (cmdptr->callback)
 380                                 cmdptr->callback(mrioc, cmdptr);
 381                 }
 382         }
 383 out:
 384         if (sense_buf)
 385                 mpi3mr_repost_sense_buf(mrioc,
 386                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 387 }
 388
 389 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 390 {
 391         u32 exp_phase = mrioc->admin_reply_ephase;
 392         u32 admin_reply_ci = mrioc->admin_reply_ci;
 393         u32 num_admin_replies = 0;
 394         u64 reply_dma = 0;
 395         struct mpi3_default_reply_descriptor *reply_desc;
 396
 397         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 398             admin_reply_ci;
 399
 400         if ((le16_to_cpu(reply_desc->reply_flags) &
 401             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 402                 return 0;
 403
 404         do {
 405                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 406                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 407                 if (reply_dma)
 408                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 409                 num_admin_replies++;
 410                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 411                         admin_reply_ci = 0;
 412                         exp_phase ^= 1;
 413                 }
 414                 reply_desc =
 415                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 416                     admin_reply_ci;
 417                 if ((le16_to_cpu(reply_desc->reply_flags) &
 418                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 419                         break;
 420         } while (1);
 421
 422         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 423         mrioc->admin_reply_ci = admin_reply_ci;
 424         mrioc->admin_reply_ephase = exp_phase;
 425
 426         return num_admin_replies;
 427 }
 428
 429 /**
 430  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 431  *      queue's consumer index from operational reply descriptor queue.
 432  * @op_reply_q: op_reply_qinfo object
 433  * @reply_ci: operational reply descriptor's queue consumer index
 434  *
 435  * Returns reply descriptor frame address
 436  */
 437 static inline struct mpi3_default_reply_descriptor *
 438 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 439 {
 440         void *segment_base_addr;
 441         struct segments *segments = op_reply_q->q_segments;
 442         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 443
 444         segment_base_addr =
 445             segments[reply_ci / op_reply_q->segment_qd].segment;
 446         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 447             (reply_ci % op_reply_q->segment_qd);
 448         return reply_desc;
 449 }
 450
 451 static int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 452         struct mpi3mr_intr_info *intr_info)
 453 {
 454         struct op_reply_qinfo *op_reply_q = intr_info->op_reply_q;
 455         struct op_req_qinfo *op_req_q;
 456         u32 exp_phase;
 457         u32 reply_ci;
 458         u32 num_op_reply = 0;
 459         u64 reply_dma = 0;
 460         struct mpi3_default_reply_descriptor *reply_desc;
 461         u16 req_q_idx = 0, reply_qidx;
 462
 463         reply_qidx = op_reply_q->qid - 1;
 464
 465         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 466                 return 0;
 467
 468         exp_phase = op_reply_q->ephase;
 469         reply_ci = op_reply_q->ci;
 470
 471         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 472         if ((le16_to_cpu(reply_desc->reply_flags) &
 473             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 474                 atomic_dec(&op_reply_q->in_use);
 475                 return 0;
 476         }
 477
 478         do {
 479                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 480                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 481
 482                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 483                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 484                     reply_qidx);
 485                 atomic_dec(&op_reply_q->pend_ios);
 486                 if (reply_dma)
 487                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 488                 num_op_reply++;
 489
 490                 if (++reply_ci == op_reply_q->num_replies) {
 491                         reply_ci = 0;
 492                         exp_phase ^= 1;
 493                 }
 494
 495                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 496
 497                 if ((le16_to_cpu(reply_desc->reply_flags) &
 498                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 499                         break;
 500                 /*
 501                  * Exit completion loop to avoid CPU lockup
 502                  * Ensure remaining completion happens from threaded ISR.
 503                  */
 504                 if (num_op_reply > mrioc->max_host_ios) {
 505                         intr_info->op_reply_q->enable_irq_poll = true;
 506                         break;
 507                 }
 508
 509         } while (1);
 510
 511         writel(reply_ci,
 512             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 513         op_reply_q->ci = reply_ci;
 514         op_reply_q->ephase = exp_phase;
 515
 516         atomic_dec(&op_reply_q->in_use);
 517         return num_op_reply;
 518 }
 519
 520 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 521 {
 522         struct mpi3mr_intr_info *intr_info = privdata;
 523         struct mpi3mr_ioc *mrioc;
 524         u16 midx;
 525         u32 num_admin_replies = 0, num_op_reply = 0;
 526
 527         if (!intr_info)
 528                 return IRQ_NONE;
 529
 530         mrioc = intr_info->mrioc;
 531
 532         if (!mrioc->intr_enabled)
 533                 return IRQ_NONE;
 534
 535         midx = intr_info->msix_index;
 536
 537         if (!midx)
 538                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 539         if (intr_info->op_reply_q)
 540                 num_op_reply = mpi3mr_process_op_reply_q(mrioc, intr_info);
 541
 542         if (num_admin_replies || num_op_reply)
 543                 return IRQ_HANDLED;
 544         else
 545                 return IRQ_NONE;
 546 }
 547
 548 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 549 {
 550         struct mpi3mr_intr_info *intr_info = privdata;
 551         struct mpi3mr_ioc *mrioc;
 552         u16 midx;
 553         int ret;
 554
 555         if (!intr_info)
 556                 return IRQ_NONE;
 557
 558         mrioc = intr_info->mrioc;
 559         midx = intr_info->msix_index;
 560         /* Call primary ISR routine */
 561         ret = mpi3mr_isr_primary(irq, privdata);
 562
 563         /*
 564          * If more IOs are expected, schedule IRQ polling thread.
 565          * Otherwise exit from ISR.
 566          */
 567         if (!intr_info->op_reply_q)
 568                 return ret;
 569
 570         if (!intr_info->op_reply_q->enable_irq_poll ||
 571             !atomic_read(&intr_info->op_reply_q->pend_ios))
 572                 return ret;
 573
 574         disable_irq_nosync(pci_irq_vector(mrioc->pdev, midx));
 575
 576         return IRQ_WAKE_THREAD;
 577 }
 578
 579 /**
 580  * mpi3mr_isr_poll - Reply queue polling routine
 581  * @irq: IRQ
 582  * @privdata: Interrupt info
 583  *
 584  * poll for pending I/O completions in a loop until pending I/Os
 585  * present or controller queue depth I/Os are processed.
 586  *
 587  * Return: IRQ_NONE or IRQ_HANDLED
 588  */
 589 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 590 {
 591         struct mpi3mr_intr_info *intr_info = privdata;
 592         struct mpi3mr_ioc *mrioc;
 593         u16 midx;
 594         u32 num_op_reply = 0;
 595
 596         if (!intr_info || !intr_info->op_reply_q)
 597                 return IRQ_NONE;
 598
 599         mrioc = intr_info->mrioc;
 600         midx = intr_info->msix_index;
 601
 602         /* Poll for pending IOs completions */
 603         do {
 604                 if (!mrioc->intr_enabled)
 605                         break;
 606
 607                 if (!midx)
 608                         mpi3mr_process_admin_reply_q(mrioc);
 609                 if (intr_info->op_reply_q)
 610                         num_op_reply +=
 611                             mpi3mr_process_op_reply_q(mrioc, intr_info);
 612
 613                 usleep_range(mrioc->irqpoll_sleep, 10 * mrioc->irqpoll_sleep);
 614
 615         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 616             (num_op_reply < mrioc->max_host_ios));
 617
 618         intr_info->op_reply_q->enable_irq_poll = false;
 619         enable_irq(pci_irq_vector(mrioc->pdev, midx));
 620
 621         return IRQ_HANDLED;
 622 }
 623
 624 /**
 625  * mpi3mr_request_irq - Request IRQ and register ISR
 626  * @mrioc: Adapter instance reference
 627  * @index: IRQ vector index
 628  *
 629  * Request threaded ISR with primary ISR and secondary
 630  *
 631  * Return: 0 on success and non zero on failures.
 632  */
 633 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 634 {
 635         struct pci_dev *pdev = mrioc->pdev;
 636         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 637         int retval = 0;
 638
 639         intr_info->mrioc = mrioc;
 640         intr_info->msix_index = index;
 641         intr_info->op_reply_q = NULL;
 642
 643         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 644             mrioc->driver_name, mrioc->id, index);
 645
 646         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 647             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 648         if (retval) {
 649                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 650                     intr_info->name, pci_irq_vector(pdev, index));
 651                 return retval;
 652         }
 653
 654         return retval;
 655 }
 656
 657 /**
 658  * mpi3mr_setup_isr - Setup ISR for the controller
 659  * @mrioc: Adapter instance reference
 660  * @setup_one: Request one IRQ or more
 661  *
 662  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 663  *
 664  * Return: 0 on success and non zero on failures.
 665  */
 666 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 667 {
 668         unsigned int irq_flags = PCI_IRQ_MSIX;
 669         int max_vectors;
 670         int retval;
 671         int i;
 672         struct irq_affinity desc = { .pre_vectors =  1};
 673
 674         mpi3mr_cleanup_isr(mrioc);
 675
 676         if (setup_one || reset_devices)
 677                 max_vectors = 1;
 678         else {
 679                 max_vectors =
 680                     min_t(int, mrioc->cpu_count + 1, mrioc->msix_count);
 681
 682                 ioc_info(mrioc,
 683                     "MSI-X vectors supported: %d, no of cores: %d,",
 684                     mrioc->msix_count, mrioc->cpu_count);
 685                 ioc_info(mrioc,
 686                     "MSI-x vectors requested: %d\n", max_vectors);
 687         }
 688
 689         irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 690
 691         mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 692         retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 693                                 1, max_vectors, irq_flags, &desc);
 694         if (retval < 0) {
 695                 ioc_err(mrioc, "Cannot alloc irq vectors\n");
 696                 goto out_failed;
 697         }
 698         if (retval != max_vectors) {
 699                 ioc_info(mrioc,
 700                     "allocated vectors (%d) are less than configured (%d)\n",
 701                     retval, max_vectors);
 702                 /*
 703                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 704                  * between Admin queue and operational queue
 705                  */
 706                 if (retval == 1)
 707                         mrioc->op_reply_q_offset = 0;
 708
 709                 max_vectors = retval;
 710         }
 711         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 712             GFP_KERNEL);
 713         if (!mrioc->intr_info) {
 714                 retval = -ENOMEM;
 715                 pci_free_irq_vectors(mrioc->pdev);
 716                 goto out_failed;
 717         }
 718         for (i = 0; i < max_vectors; i++) {
 719                 retval = mpi3mr_request_irq(mrioc, i);
 720                 if (retval) {
 721                         mrioc->intr_info_count = i;
 722                         goto out_failed;
 723                 }
 724         }
 725         mrioc->intr_info_count = max_vectors;
 726         mpi3mr_ioc_enable_intr(mrioc);
 727         return 0;
 728
 729 out_failed:
 730         mpi3mr_cleanup_isr(mrioc);
 731
 732         return retval;
 733 }
 734
 735 static const struct {
 736         enum mpi3mr_iocstate value;
 737         char *name;
 738 } mrioc_states[] = {
 739         { MRIOC_STATE_READY, "ready" },
 740         { MRIOC_STATE_FAULT, "fault" },
 741         { MRIOC_STATE_RESET, "reset" },
 742         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 743         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 744         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 745 };
 746
 747 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 748 {
 749         int i;
 750         char *name = NULL;
 751
 752         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 753                 if (mrioc_states[i].value == mrioc_state) {
 754                         name = mrioc_states[i].name;
 755                         break;
 756                 }
 757         }
 758         return name;
 759 }
 760
 761 /* Reset reason to name mapper structure*/
 762 static const struct {
 763         enum mpi3mr_reset_reason value;
 764         char *name;
 765 } mpi3mr_reset_reason_codes[] = {
 766         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 767         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 768         { MPI3MR_RESET_FROM_IOCTL, "application invocation" },
 769         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 770         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 771         { MPI3MR_RESET_FROM_IOCTL_TIMEOUT, "IOCTL timeout" },
 772         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 773         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 774         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 775         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 776         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 777         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 778         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 779         {
 780                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 781                 "create request queue timeout"
 782         },
 783         {
 784                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 785                 "create reply queue timeout"
 786         },
 787         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 788         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 789         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 790         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 791         {
 792                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 793                 "component image activation timeout"
 794         },
 795         {
 796                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 797                 "get package version timeout"
 798         },
 799         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 800         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 801 };
 802
 803 /**
 804  * mpi3mr_reset_rc_name - get reset reason code name
 805  * @reason_code: reset reason code value
 806  *
 807  * Map reset reason to an NULL terminated ASCII string
 808  *
 809  * Return: name corresponding to reset reason value or NULL.
 810  */
 811 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 812 {
 813         int i;
 814         char *name = NULL;
 815
 816         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 817                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 818                         name = mpi3mr_reset_reason_codes[i].name;
 819                         break;
 820                 }
 821         }
 822         return name;
 823 }
 824
 825 /* Reset type to name mapper structure*/
 826 static const struct {
 827         u16 reset_type;
 828         char *name;
 829 } mpi3mr_reset_types[] = {
 830         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 831         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 832 };
 833
 834 /**
 835  * mpi3mr_reset_type_name - get reset type name
 836  * @reset_type: reset type value
 837  *
 838  * Map reset type to an NULL terminated ASCII string
 839  *
 840  * Return: name corresponding to reset type value or NULL.
 841  */
 842 static const char *mpi3mr_reset_type_name(u16 reset_type)
 843 {
 844         int i;
 845         char *name = NULL;
 846
 847         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 848                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 849                         name = mpi3mr_reset_types[i].name;
 850                         break;
 851                 }
 852         }
 853         return name;
 854 }
 855
 856 /**
 857  * mpi3mr_print_fault_info - Display fault information
 858  * @mrioc: Adapter instance reference
 859  *
 860  * Display the controller fault information if there is a
 861  * controller fault.
 862  *
 863  * Return: Nothing.
 864  */
 865 static void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 866 {
 867         u32 ioc_status, code, code1, code2, code3;
 868
 869         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 870
 871         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
 872                 code = readl(&mrioc->sysif_regs->fault);
 873                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
 874                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
 875                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
 876
 877                 ioc_info(mrioc,
 878                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
 879                     code, code1, code2, code3);
 880         }
 881 }
 882
 883 /**
 884  * mpi3mr_get_iocstate - Get IOC State
 885  * @mrioc: Adapter instance reference
 886  *
 887  * Return a proper IOC state enum based on the IOC status and
 888  * IOC configuration and unrcoverable state of the controller.
 889  *
 890  * Return: Current IOC state.
 891  */
 892 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
 893 {
 894         u32 ioc_status, ioc_config;
 895         u8 ready, enabled;
 896
 897         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 898         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 899
 900         if (mrioc->unrecoverable)
 901                 return MRIOC_STATE_UNRECOVERABLE;
 902         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
 903                 return MRIOC_STATE_FAULT;
 904
 905         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
 906         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
 907
 908         if (ready && enabled)
 909                 return MRIOC_STATE_READY;
 910         if ((!ready) && (!enabled))
 911                 return MRIOC_STATE_RESET;
 912         if ((!ready) && (enabled))
 913                 return MRIOC_STATE_BECOMING_READY;
 914
 915         return MRIOC_STATE_RESET_REQUESTED;
 916 }
 917
 918 /**
 919  * mpi3mr_clear_reset_history - clear reset history
 920  * @mrioc: Adapter instance reference
 921  *
 922  * Write the reset history bit in IOC status to clear the bit,
 923  * if it is already set.
 924  *
 925  * Return: Nothing.
 926  */
 927 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
 928 {
 929         u32 ioc_status;
 930
 931         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 932         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
 933                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
 934 }
 935
 936 /**
 937  * mpi3mr_issue_and_process_mur - Message unit Reset handler
 938  * @mrioc: Adapter instance reference
 939  * @reset_reason: Reset reason code
 940  *
 941  * Issue Message unit Reset to the controller and wait for it to
 942  * be complete.
 943  *
 944  * Return: 0 on success, -1 on failure.
 945  */
 946 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
 947         u32 reset_reason)
 948 {
 949         u32 ioc_config, timeout, ioc_status;
 950         int retval = -1;
 951
 952         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
 953         if (mrioc->unrecoverable) {
 954                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
 955                 return retval;
 956         }
 957         mpi3mr_clear_reset_history(mrioc);
 958         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
 959         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 960         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
 961         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
 962
 963         timeout = mrioc->ready_timeout * 10;
 964         do {
 965                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 966                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
 967                         mpi3mr_clear_reset_history(mrioc);
 968                         ioc_config =
 969                             readl(&mrioc->sysif_regs->ioc_configuration);
 970                         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
 971                               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
 972                             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC))) {
 973                                 retval = 0;
 974                                 break;
 975                         }
 976                 }
 977                 msleep(100);
 978         } while (--timeout);
 979
 980         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 981         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 982
 983         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
 984             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
 985         return retval;
 986 }
 987
 988 /**
 989  * mpi3mr_bring_ioc_ready - Bring controller to ready state
 990  * @mrioc: Adapter instance reference
 991  *
 992  * Set Enable IOC bit in IOC configuration register and wait for
 993  * the controller to become ready.
 994  *
 995  * Return: 0 on success, -1 on failure.
 996  */
 997 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
 998 {
 999         u32 ioc_config, timeout;
1000         enum mpi3mr_iocstate current_state;
1001
1002         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1003         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1004         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1005
1006         timeout = mrioc->ready_timeout * 10;
1007         do {
1008                 current_state = mpi3mr_get_iocstate(mrioc);
1009                 if (current_state == MRIOC_STATE_READY)
1010                         return 0;
1011                 msleep(100);
1012         } while (--timeout);
1013
1014         return -1;
1015 }
1016
1017 /**
1018  * mpi3mr_soft_reset_success - Check softreset is success or not
1019  * @ioc_status: IOC status register value
1020  * @ioc_config: IOC config register value
1021  *
1022  * Check whether the soft reset is successful or not based on
1023  * IOC status and IOC config register values.
1024  *
1025  * Return: True when the soft reset is success, false otherwise.
1026  */
1027 static inline bool
1028 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1029 {
1030         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1031             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1032             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1033                 return true;
1034         return false;
1035 }
1036
1037 /**
1038  * mpi3mr_diagfault_success - Check diag fault is success or not
1039  * @mrioc: Adapter reference
1040  * @ioc_status: IOC status register value
1041  *
1042  * Check whether the controller hit diag reset fault code.
1043  *
1044  * Return: True when there is diag fault, false otherwise.
1045  */
1046 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1047         u32 ioc_status)
1048 {
1049         u32 fault;
1050
1051         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1052                 return false;
1053         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1054         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET)
1055                 return true;
1056         return false;
1057 }
1058
1059 /**
1060  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1061  * @mrioc: Adapter reference
1062  *
1063  * Set diag save bit in IOC configuration register to enable
1064  * snapdump.
1065  *
1066  * Return: Nothing.
1067  */
1068 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1069 {
1070         u32 ioc_config;
1071
1072         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1073         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1074         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1075 }
1076
1077 /**
1078  * mpi3mr_issue_reset - Issue reset to the controller
1079  * @mrioc: Adapter reference
1080  * @reset_type: Reset type
1081  * @reset_reason: Reset reason code
1082  *
1083  * Unlock the host diagnostic registers and write the specific
1084  * reset type to that, wait for reset acknowledgment from the
1085  * controller, if the reset is not successful retry for the
1086  * predefined number of times.
1087  *
1088  * Return: 0 on success, non-zero on failure.
1089  */
1090 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1091         u32 reset_reason)
1092 {
1093         int retval = -1;
1094         u8 unlock_retry_count, reset_retry_count = 0;
1095         u32 host_diagnostic, timeout, ioc_status, ioc_config;
1096
1097         pci_cfg_access_lock(mrioc->pdev);
1098         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1099             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1100                 goto out;
1101         if (mrioc->unrecoverable)
1102                 goto out;
1103 retry_reset:
1104         unlock_retry_count = 0;
1105         mpi3mr_clear_reset_history(mrioc);
1106         do {
1107                 ioc_info(mrioc,
1108                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1109                     ++unlock_retry_count);
1110                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1111                         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1112                         mrioc->unrecoverable = 1;
1113                         goto out;
1114                 }
1115
1116                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1117                     &mrioc->sysif_regs->write_sequence);
1118                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1119                     &mrioc->sysif_regs->write_sequence);
1120                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1121                     &mrioc->sysif_regs->write_sequence);
1122                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1123                     &mrioc->sysif_regs->write_sequence);
1124                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1125                     &mrioc->sysif_regs->write_sequence);
1126                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1127                     &mrioc->sysif_regs->write_sequence);
1128                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1129                     &mrioc->sysif_regs->write_sequence);
1130                 usleep_range(1000, 1100);
1131                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1132                 ioc_info(mrioc,
1133                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1134                     unlock_retry_count, host_diagnostic);
1135         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1136
1137         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1138         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1139             mpi3mr_reset_type_name(reset_type),
1140             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1141         writel(host_diagnostic | reset_type,
1142             &mrioc->sysif_regs->host_diagnostic);
1143         timeout = mrioc->ready_timeout * 10;
1144         if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) {
1145                 do {
1146                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1147                         if (ioc_status &
1148                             MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
1149                                 mpi3mr_clear_reset_history(mrioc);
1150                                 ioc_config =
1151                                     readl(&mrioc->sysif_regs->ioc_configuration);
1152                                 if (mpi3mr_soft_reset_success(ioc_status,
1153                                     ioc_config)) {
1154                                         retval = 0;
1155                                         break;
1156                                 }
1157                         }
1158                         msleep(100);
1159                 } while (--timeout);
1160                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1161                     &mrioc->sysif_regs->write_sequence);
1162         } else if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT) {
1163                 do {
1164                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1165                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1166                                 retval = 0;
1167                                 break;
1168                         }
1169                         msleep(100);
1170                 } while (--timeout);
1171                 mpi3mr_clear_reset_history(mrioc);
1172                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1173                     &mrioc->sysif_regs->write_sequence);
1174         }
1175         if (retval && ((++reset_retry_count) < MPI3MR_MAX_RESET_RETRY_COUNT)) {
1176                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1177                 ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1178                 ioc_info(mrioc,
1179                     "Base IOC Sts/Config after reset try %d is (0x%x)/(0x%x)\n",
1180                     reset_retry_count, ioc_status, ioc_config);
1181                 goto retry_reset;
1182         }
1183
1184 out:
1185         pci_cfg_access_unlock(mrioc->pdev);
1186         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1187         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1188
1189         ioc_info(mrioc,
1190             "Base IOC Sts/Config after %s reset is (0x%x)/(0x%x)\n",
1191             (!retval) ? "successful" : "failed", ioc_status,
1192             ioc_config);
1193         return retval;
1194 }
1195
1196 /**
1197  * mpi3mr_admin_request_post - Post request to admin queue
1198  * @mrioc: Adapter reference
1199  * @admin_req: MPI3 request
1200  * @admin_req_sz: Request size
1201  * @ignore_reset: Ignore reset in process
1202  *
1203  * Post the MPI3 request into admin request queue and
1204  * inform the controller, if the queue is full return
1205  * appropriate error.
1206  *
1207  * Return: 0 on success, non-zero on failure.
1208  */
1209 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1210         u16 admin_req_sz, u8 ignore_reset)
1211 {
1212         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1213         int retval = 0;
1214         unsigned long flags;
1215         u8 *areq_entry;
1216
1217         if (mrioc->unrecoverable) {
1218                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1219                 return -EFAULT;
1220         }
1221
1222         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1223         areq_pi = mrioc->admin_req_pi;
1224         areq_ci = mrioc->admin_req_ci;
1225         max_entries = mrioc->num_admin_req;
1226         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1227             (areq_pi == (max_entries - 1)))) {
1228                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1229                 retval = -EAGAIN;
1230                 goto out;
1231         }
1232         if (!ignore_reset && mrioc->reset_in_progress) {
1233                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1234                 retval = -EAGAIN;
1235                 goto out;
1236         }
1237         areq_entry = (u8 *)mrioc->admin_req_base +
1238             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1239         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1240         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1241
1242         if (++areq_pi == max_entries)
1243                 areq_pi = 0;
1244         mrioc->admin_req_pi = areq_pi;
1245
1246         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1247
1248 out:
1249         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1250
1251         return retval;
1252 }
1253
1254 /**
1255  * mpi3mr_free_op_req_q_segments - free request memory segments
1256  * @mrioc: Adapter instance reference
1257  * @q_idx: operational request queue index
1258  *
1259  * Free memory segments allocated for operational request queue
1260  *
1261  * Return: Nothing.
1262  */
1263 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1264 {
1265         u16 j;
1266         int size;
1267         struct segments *segments;
1268
1269         segments = mrioc->req_qinfo[q_idx].q_segments;
1270         if (!segments)
1271                 return;
1272
1273         if (mrioc->enable_segqueue) {
1274                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1275                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1276                         dma_free_coherent(&mrioc->pdev->dev,
1277                             MPI3MR_MAX_SEG_LIST_SIZE,
1278                             mrioc->req_qinfo[q_idx].q_segment_list,
1279                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1280                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1281                 }
1282         } else
1283                 size = mrioc->req_qinfo[q_idx].num_requests *
1284                     mrioc->facts.op_req_sz;
1285
1286         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1287                 if (!segments[j].segment)
1288                         continue;
1289                 dma_free_coherent(&mrioc->pdev->dev,
1290                     size, segments[j].segment, segments[j].segment_dma);
1291                 segments[j].segment = NULL;
1292         }
1293         kfree(mrioc->req_qinfo[q_idx].q_segments);
1294         mrioc->req_qinfo[q_idx].q_segments = NULL;
1295         mrioc->req_qinfo[q_idx].qid = 0;
1296 }
1297
1298 /**
1299  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1300  * @mrioc: Adapter instance reference
1301  * @q_idx: operational reply queue index
1302  *
1303  * Free memory segments allocated for operational reply queue
1304  *
1305  * Return: Nothing.
1306  */
1307 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1308 {
1309         u16 j;
1310         int size;
1311         struct segments *segments;
1312
1313         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1314         if (!segments)
1315                 return;
1316
1317         if (mrioc->enable_segqueue) {
1318                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1319                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1320                         dma_free_coherent(&mrioc->pdev->dev,
1321                             MPI3MR_MAX_SEG_LIST_SIZE,
1322                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1323                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1324                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1325                 }
1326         } else
1327                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1328                     mrioc->op_reply_desc_sz;
1329
1330         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1331                 if (!segments[j].segment)
1332                         continue;
1333                 dma_free_coherent(&mrioc->pdev->dev,
1334                     size, segments[j].segment, segments[j].segment_dma);
1335                 segments[j].segment = NULL;
1336         }
1337
1338         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1339         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1340         mrioc->op_reply_qinfo[q_idx].qid = 0;
1341 }
1342
1343 /**
1344  * mpi3mr_delete_op_reply_q - delete operational reply queue
1345  * @mrioc: Adapter instance reference
1346  * @qidx: operational reply queue index
1347  *
1348  * Delete operatinal reply queue by issuing MPI request
1349  * through admin queue.
1350  *
1351  * Return:  0 on success, non-zero on failure.
1352  */
1353 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1354 {
1355         struct mpi3_delete_reply_queue_request delq_req;
1356         int retval = 0;
1357         u16 reply_qid = 0, midx;
1358
1359         reply_qid = mrioc->op_reply_qinfo[qidx].qid;
1360
1361         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1362
1363         if (!reply_qid) {
1364                 retval = -1;
1365                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1366                 goto out;
1367         }
1368
1369         memset(&delq_req, 0, sizeof(delq_req));
1370         mutex_lock(&mrioc->init_cmds.mutex);
1371         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1372                 retval = -1;
1373                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1374                 mutex_unlock(&mrioc->init_cmds.mutex);
1375                 goto out;
1376         }
1377         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1378         mrioc->init_cmds.is_waiting = 1;
1379         mrioc->init_cmds.callback = NULL;
1380         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1381         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1382         delq_req.queue_id = cpu_to_le16(reply_qid);
1383
1384         init_completion(&mrioc->init_cmds.done);
1385         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1386             1);
1387         if (retval) {
1388                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1389                 goto out_unlock;
1390         }
1391         wait_for_completion_timeout(&mrioc->init_cmds.done,
1392             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1393         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1394                 ioc_err(mrioc, "Issue DelRepQ: command timed out\n");
1395                 mpi3mr_set_diagsave(mrioc);
1396                 mpi3mr_issue_reset(mrioc,
1397                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1398                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1399                 mrioc->unrecoverable = 1;
1400
1401                 retval = -1;
1402                 goto out_unlock;
1403         }
1404         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1405             != MPI3_IOCSTATUS_SUCCESS) {
1406                 ioc_err(mrioc,
1407                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1408                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1409                     mrioc->init_cmds.ioc_loginfo);
1410                 retval = -1;
1411                 goto out_unlock;
1412         }
1413         mrioc->intr_info[midx].op_reply_q = NULL;
1414
1415         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1416 out_unlock:
1417         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1418         mutex_unlock(&mrioc->init_cmds.mutex);
1419 out:
1420
1421         return retval;
1422 }
1423
1424 /**
1425  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1426  * @mrioc: Adapter instance reference
1427  * @qidx: request queue index
1428  *
1429  * Allocate segmented memory pools for operational reply
1430  * queue.
1431  *
1432  * Return: 0 on success, non-zero on failure.
1433  */
1434 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1435 {
1436         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1437         int i, size;
1438         u64 *q_segment_list_entry = NULL;
1439         struct segments *segments;
1440
1441         if (mrioc->enable_segqueue) {
1442                 op_reply_q->segment_qd =
1443                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1444
1445                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1446
1447                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1448                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1449                     GFP_KERNEL);
1450                 if (!op_reply_q->q_segment_list)
1451                         return -ENOMEM;
1452                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1453         } else {
1454                 op_reply_q->segment_qd = op_reply_q->num_replies;
1455                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1456         }
1457
1458         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1459             op_reply_q->segment_qd);
1460
1461         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1462             sizeof(struct segments), GFP_KERNEL);
1463         if (!op_reply_q->q_segments)
1464                 return -ENOMEM;
1465
1466         segments = op_reply_q->q_segments;
1467         for (i = 0; i < op_reply_q->num_segments; i++) {
1468                 segments[i].segment =
1469                     dma_alloc_coherent(&mrioc->pdev->dev,
1470                     size, &segments[i].segment_dma, GFP_KERNEL);
1471                 if (!segments[i].segment)
1472                         return -ENOMEM;
1473                 if (mrioc->enable_segqueue)
1474                         q_segment_list_entry[i] =
1475                             (unsigned long)segments[i].segment_dma;
1476         }
1477
1478         return 0;
1479 }
1480
1481 /**
1482  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1483  * @mrioc: Adapter instance reference
1484  * @qidx: request queue index
1485  *
1486  * Allocate segmented memory pools for operational request
1487  * queue.
1488  *
1489  * Return: 0 on success, non-zero on failure.
1490  */
1491 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1492 {
1493         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1494         int i, size;
1495         u64 *q_segment_list_entry = NULL;
1496         struct segments *segments;
1497
1498         if (mrioc->enable_segqueue) {
1499                 op_req_q->segment_qd =
1500                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1501
1502                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1503
1504                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1505                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1506                     GFP_KERNEL);
1507                 if (!op_req_q->q_segment_list)
1508                         return -ENOMEM;
1509                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1510
1511         } else {
1512                 op_req_q->segment_qd = op_req_q->num_requests;
1513                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1514         }
1515
1516         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1517             op_req_q->segment_qd);
1518
1519         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1520             sizeof(struct segments), GFP_KERNEL);
1521         if (!op_req_q->q_segments)
1522                 return -ENOMEM;
1523
1524         segments = op_req_q->q_segments;
1525         for (i = 0; i < op_req_q->num_segments; i++) {
1526                 segments[i].segment =
1527                     dma_alloc_coherent(&mrioc->pdev->dev,
1528                     size, &segments[i].segment_dma, GFP_KERNEL);
1529                 if (!segments[i].segment)
1530                         return -ENOMEM;
1531                 if (mrioc->enable_segqueue)
1532                         q_segment_list_entry[i] =
1533                             (unsigned long)segments[i].segment_dma;
1534         }
1535
1536         return 0;
1537 }
1538
1539 /**
1540  * mpi3mr_create_op_reply_q - create operational reply queue
1541  * @mrioc: Adapter instance reference
1542  * @qidx: operational reply queue index
1543  *
1544  * Create operatinal reply queue by issuing MPI request
1545  * through admin queue.
1546  *
1547  * Return:  0 on success, non-zero on failure.
1548  */
1549 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1550 {
1551         struct mpi3_create_reply_queue_request create_req;
1552         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1553         int retval = 0;
1554         u16 reply_qid = 0, midx;
1555
1556         reply_qid = op_reply_q->qid;
1557
1558         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1559
1560         if (reply_qid) {
1561                 retval = -1;
1562                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1563                     reply_qid);
1564
1565                 return retval;
1566         }
1567
1568         reply_qid = qidx + 1;
1569         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1570         op_reply_q->ci = 0;
1571         op_reply_q->ephase = 1;
1572         atomic_set(&op_reply_q->pend_ios, 0);
1573         atomic_set(&op_reply_q->in_use, 0);
1574         op_reply_q->enable_irq_poll = false;
1575
1576         if (!op_reply_q->q_segments) {
1577                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1578                 if (retval) {
1579                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1580                         goto out;
1581                 }
1582         }
1583
1584         memset(&create_req, 0, sizeof(create_req));
1585         mutex_lock(&mrioc->init_cmds.mutex);
1586         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1587                 retval = -1;
1588                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1589                 goto out_unlock;
1590         }
1591         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1592         mrioc->init_cmds.is_waiting = 1;
1593         mrioc->init_cmds.callback = NULL;
1594         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1595         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1596         create_req.queue_id = cpu_to_le16(reply_qid);
1597         create_req.flags = MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1598         create_req.msix_index = cpu_to_le16(mrioc->intr_info[midx].msix_index);
1599         if (mrioc->enable_segqueue) {
1600                 create_req.flags |=
1601                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1602                 create_req.base_address = cpu_to_le64(
1603                     op_reply_q->q_segment_list_dma);
1604         } else
1605                 create_req.base_address = cpu_to_le64(
1606                     op_reply_q->q_segments[0].segment_dma);
1607
1608         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1609
1610         init_completion(&mrioc->init_cmds.done);
1611         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1612             sizeof(create_req), 1);
1613         if (retval) {
1614                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1615                 goto out_unlock;
1616         }
1617         wait_for_completion_timeout(&mrioc->init_cmds.done,
1618             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1619         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1620                 ioc_err(mrioc, "CreateRepQ: command timed out\n");
1621                 mpi3mr_set_diagsave(mrioc);
1622                 mpi3mr_issue_reset(mrioc,
1623                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1624                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1625                 mrioc->unrecoverable = 1;
1626                 retval = -1;
1627                 goto out_unlock;
1628         }
1629         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1630             != MPI3_IOCSTATUS_SUCCESS) {
1631                 ioc_err(mrioc,
1632                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1633                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1634                     mrioc->init_cmds.ioc_loginfo);
1635                 retval = -1;
1636                 goto out_unlock;
1637         }
1638         op_reply_q->qid = reply_qid;
1639         mrioc->intr_info[midx].op_reply_q = op_reply_q;
1640
1641 out_unlock:
1642         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1643         mutex_unlock(&mrioc->init_cmds.mutex);
1644 out:
1645
1646         return retval;
1647 }
1648
1649 /**
1650  * mpi3mr_create_op_req_q - create operational request queue
1651  * @mrioc: Adapter instance reference
1652  * @idx: operational request queue index
1653  * @reply_qid: Reply queue ID
1654  *
1655  * Create operatinal request queue by issuing MPI request
1656  * through admin queue.
1657  *
1658  * Return:  0 on success, non-zero on failure.
1659  */
1660 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1661         u16 reply_qid)
1662 {
1663         struct mpi3_create_request_queue_request create_req;
1664         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1665         int retval = 0;
1666         u16 req_qid = 0;
1667
1668         req_qid = op_req_q->qid;
1669
1670         if (req_qid) {
1671                 retval = -1;
1672                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1673                     req_qid);
1674
1675                 return retval;
1676         }
1677         req_qid = idx + 1;
1678
1679         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1680         op_req_q->ci = 0;
1681         op_req_q->pi = 0;
1682         op_req_q->reply_qid = reply_qid;
1683         spin_lock_init(&op_req_q->q_lock);
1684
1685         if (!op_req_q->q_segments) {
1686                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1687                 if (retval) {
1688                         mpi3mr_free_op_req_q_segments(mrioc, idx);
1689                         goto out;
1690                 }
1691         }
1692
1693         memset(&create_req, 0, sizeof(create_req));
1694         mutex_lock(&mrioc->init_cmds.mutex);
1695         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1696                 retval = -1;
1697                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
1698                 goto out_unlock;
1699         }
1700         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1701         mrioc->init_cmds.is_waiting = 1;
1702         mrioc->init_cmds.callback = NULL;
1703         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1704         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
1705         create_req.queue_id = cpu_to_le16(req_qid);
1706         if (mrioc->enable_segqueue) {
1707                 create_req.flags =
1708                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1709                 create_req.base_address = cpu_to_le64(
1710                     op_req_q->q_segment_list_dma);
1711         } else
1712                 create_req.base_address = cpu_to_le64(
1713                     op_req_q->q_segments[0].segment_dma);
1714         create_req.reply_queue_id = cpu_to_le16(reply_qid);
1715         create_req.size = cpu_to_le16(op_req_q->num_requests);
1716
1717         init_completion(&mrioc->init_cmds.done);
1718         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1719             sizeof(create_req), 1);
1720         if (retval) {
1721                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
1722                 goto out_unlock;
1723         }
1724         wait_for_completion_timeout(&mrioc->init_cmds.done,
1725             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1726         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1727                 ioc_err(mrioc, "CreateReqQ: command timed out\n");
1728                 mpi3mr_set_diagsave(mrioc);
1729                 if (mpi3mr_issue_reset(mrioc,
1730                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1731                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT))
1732                         mrioc->unrecoverable = 1;
1733                 retval = -1;
1734                 goto out_unlock;
1735         }
1736         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1737             != MPI3_IOCSTATUS_SUCCESS) {
1738                 ioc_err(mrioc,
1739                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1740                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1741                     mrioc->init_cmds.ioc_loginfo);
1742                 retval = -1;
1743                 goto out_unlock;
1744         }
1745         op_req_q->qid = req_qid;
1746
1747 out_unlock:
1748         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1749         mutex_unlock(&mrioc->init_cmds.mutex);
1750 out:
1751
1752         return retval;
1753 }
1754
1755 /**
1756  * mpi3mr_create_op_queues - create operational queue pairs
1757  * @mrioc: Adapter instance reference
1758  *
1759  * Allocate memory for operational queue meta data and call
1760  * create request and reply queue functions.
1761  *
1762  * Return: 0 on success, non-zero on failures.
1763  */
1764 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
1765 {
1766         int retval = 0;
1767         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
1768
1769         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
1770             mrioc->facts.max_op_req_q);
1771
1772         msix_count_op_q =
1773             mrioc->intr_info_count - mrioc->op_reply_q_offset;
1774         if (!mrioc->num_queues)
1775                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
1776         num_queues = mrioc->num_queues;
1777         ioc_info(mrioc, "Trying to create %d Operational Q pairs\n",
1778             num_queues);
1779
1780         if (!mrioc->req_qinfo) {
1781                 mrioc->req_qinfo = kcalloc(num_queues,
1782                     sizeof(struct op_req_qinfo), GFP_KERNEL);
1783                 if (!mrioc->req_qinfo) {
1784                         retval = -1;
1785                         goto out_failed;
1786                 }
1787
1788                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
1789                     num_queues, GFP_KERNEL);
1790                 if (!mrioc->op_reply_qinfo) {
1791                         retval = -1;
1792                         goto out_failed;
1793                 }
1794         }
1795
1796         if (mrioc->enable_segqueue)
1797                 ioc_info(mrioc,
1798                     "allocating operational queues through segmented queues\n");
1799
1800         for (i = 0; i < num_queues; i++) {
1801                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
1802                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
1803                         break;
1804                 }
1805                 if (mpi3mr_create_op_req_q(mrioc, i,
1806                     mrioc->op_reply_qinfo[i].qid)) {
1807                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
1808                         mpi3mr_delete_op_reply_q(mrioc, i);
1809                         break;
1810                 }
1811         }
1812
1813         if (i == 0) {
1814                 /* Not even one queue is created successfully*/
1815                 retval = -1;
1816                 goto out_failed;
1817         }
1818         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
1819         ioc_info(mrioc, "Successfully created %d Operational Q pairs\n",
1820             mrioc->num_op_reply_q);
1821
1822         return retval;
1823 out_failed:
1824         kfree(mrioc->req_qinfo);
1825         mrioc->req_qinfo = NULL;
1826
1827         kfree(mrioc->op_reply_qinfo);
1828         mrioc->op_reply_qinfo = NULL;
1829
1830         return retval;
1831 }
1832
1833 /**
1834  * mpi3mr_op_request_post - Post request to operational queue
1835  * @mrioc: Adapter reference
1836  * @op_req_q: Operational request queue info
1837  * @req: MPI3 request
1838  *
1839  * Post the MPI3 request into operational request queue and
1840  * inform the controller, if the queue is full return
1841  * appropriate error.
1842  *
1843  * Return: 0 on success, non-zero on failure.
1844  */
1845 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
1846         struct op_req_qinfo *op_req_q, u8 *req)
1847 {
1848         u16 pi = 0, max_entries, reply_qidx = 0, midx;
1849         int retval = 0;
1850         unsigned long flags;
1851         u8 *req_entry;
1852         void *segment_base_addr;
1853         u16 req_sz = mrioc->facts.op_req_sz;
1854         struct segments *segments = op_req_q->q_segments;
1855
1856         reply_qidx = op_req_q->reply_qid - 1;
1857
1858         if (mrioc->unrecoverable)
1859                 return -EFAULT;
1860
1861         spin_lock_irqsave(&op_req_q->q_lock, flags);
1862         pi = op_req_q->pi;
1863         max_entries = op_req_q->num_requests;
1864
1865         if (mpi3mr_check_req_qfull(op_req_q)) {
1866                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
1867                     reply_qidx, mrioc->op_reply_q_offset);
1868                 mpi3mr_process_op_reply_q(mrioc, &mrioc->intr_info[midx]);
1869
1870                 if (mpi3mr_check_req_qfull(op_req_q)) {
1871                         retval = -EAGAIN;
1872                         goto out;
1873                 }
1874         }
1875
1876         if (mrioc->reset_in_progress) {
1877                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
1878                 retval = -EAGAIN;
1879                 goto out;
1880         }
1881
1882         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
1883         req_entry = (u8 *)segment_base_addr +
1884             ((pi % op_req_q->segment_qd) * req_sz);
1885
1886         memset(req_entry, 0, req_sz);
1887         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
1888
1889         if (++pi == max_entries)
1890                 pi = 0;
1891         op_req_q->pi = pi;
1892
1893         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
1894             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
1895                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
1896
1897         writel(op_req_q->pi,
1898             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
1899
1900 out:
1901         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
1902         return retval;
1903 }
1904
1905 /**
1906  * mpi3mr_sync_timestamp - Issue time stamp sync request
1907  * @mrioc: Adapter reference
1908  *
1909  * Issue IO unit control MPI request to synchornize firmware
1910  * timestamp with host time.
1911  *
1912  * Return: 0 on success, non-zero on failure.
1913  */
1914 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
1915 {
1916         ktime_t current_time;
1917         struct mpi3_iounit_control_request iou_ctrl;
1918         int retval = 0;
1919
1920         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
1921         mutex_lock(&mrioc->init_cmds.mutex);
1922         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1923                 retval = -1;
1924                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
1925                 mutex_unlock(&mrioc->init_cmds.mutex);
1926                 goto out;
1927         }
1928         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1929         mrioc->init_cmds.is_waiting = 1;
1930         mrioc->init_cmds.callback = NULL;
1931         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1932         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
1933         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
1934         current_time = ktime_get_real();
1935         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
1936
1937         init_completion(&mrioc->init_cmds.done);
1938         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
1939             sizeof(iou_ctrl), 0);
1940         if (retval) {
1941                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
1942                 goto out_unlock;
1943         }
1944
1945         wait_for_completion_timeout(&mrioc->init_cmds.done,
1946             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1947         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1948                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
1949                 mrioc->init_cmds.is_waiting = 0;
1950                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
1951                         mpi3mr_soft_reset_handler(mrioc,
1952                             MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
1953                 retval = -1;
1954                 goto out_unlock;
1955         }
1956         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1957             != MPI3_IOCSTATUS_SUCCESS) {
1958                 ioc_err(mrioc,
1959                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1960                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1961                     mrioc->init_cmds.ioc_loginfo);
1962                 retval = -1;
1963                 goto out_unlock;
1964         }
1965
1966 out_unlock:
1967         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1968         mutex_unlock(&mrioc->init_cmds.mutex);
1969
1970 out:
1971         return retval;
1972 }
1973
1974 /**
1975  * mpi3mr_watchdog_work - watchdog thread to monitor faults
1976  * @work: work struct
1977  *
1978  * Watch dog work periodically executed (1 second interval) to
1979  * monitor firmware fault and to issue periodic timer sync to
1980  * the firmware.
1981  *
1982  * Return: Nothing.
1983  */
1984 static void mpi3mr_watchdog_work(struct work_struct *work)
1985 {
1986         struct mpi3mr_ioc *mrioc =
1987             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
1988         unsigned long flags;
1989         enum mpi3mr_iocstate ioc_state;
1990         u32 fault, host_diagnostic;
1991
1992         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
1993                 mrioc->ts_update_counter = 0;
1994                 mpi3mr_sync_timestamp(mrioc);
1995         }
1996
1997         /*Check for fault state every one second and issue Soft reset*/
1998         ioc_state = mpi3mr_get_iocstate(mrioc);
1999         if (ioc_state == MRIOC_STATE_FAULT) {
2000                 fault = readl(&mrioc->sysif_regs->fault) &
2001                     MPI3_SYSIF_FAULT_CODE_MASK;
2002                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2003                 if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2004                         if (!mrioc->diagsave_timeout) {
2005                                 mpi3mr_print_fault_info(mrioc);
2006                                 ioc_warn(mrioc, "Diag save in progress\n");
2007                         }
2008                         if ((mrioc->diagsave_timeout++) <=
2009                             MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2010                                 goto schedule_work;
2011                 } else
2012                         mpi3mr_print_fault_info(mrioc);
2013                 mrioc->diagsave_timeout = 0;
2014
2015                 if (fault == MPI3_SYSIF_FAULT_CODE_FACTORY_RESET) {
2016                         ioc_info(mrioc,
2017                             "Factory Reset fault occurred marking controller as unrecoverable"
2018                             );
2019                         mrioc->unrecoverable = 1;
2020                         goto out;
2021                 }
2022
2023                 if ((fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) ||
2024                     (fault == MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS) ||
2025                     (mrioc->reset_in_progress))
2026                         goto out;
2027                 if (fault == MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET)
2028                         mpi3mr_soft_reset_handler(mrioc,
2029                             MPI3MR_RESET_FROM_CIACTIV_FAULT, 0);
2030                 else
2031                         mpi3mr_soft_reset_handler(mrioc,
2032                             MPI3MR_RESET_FROM_FAULT_WATCH, 0);
2033         }
2034
2035 schedule_work:
2036         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2037         if (mrioc->watchdog_work_q)
2038                 queue_delayed_work(mrioc->watchdog_work_q,
2039                     &mrioc->watchdog_work,
2040                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2041         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2042 out:
2043         return;
2044 }
2045
2046 /**
2047  * mpi3mr_start_watchdog - Start watchdog
2048  * @mrioc: Adapter instance reference
2049  *
2050  * Create and start the watchdog thread to monitor controller
2051  * faults.
2052  *
2053  * Return: Nothing.
2054  */
2055 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2056 {
2057         if (mrioc->watchdog_work_q)
2058                 return;
2059
2060         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2061         snprintf(mrioc->watchdog_work_q_name,
2062             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2063             mrioc->id);
2064         mrioc->watchdog_work_q =
2065             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2066         if (!mrioc->watchdog_work_q) {
2067                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2068                 return;
2069         }
2070
2071         if (mrioc->watchdog_work_q)
2072                 queue_delayed_work(mrioc->watchdog_work_q,
2073                     &mrioc->watchdog_work,
2074                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2075 }
2076
2077 /**
2078  * mpi3mr_stop_watchdog - Stop watchdog
2079  * @mrioc: Adapter instance reference
2080  *
2081  * Stop the watchdog thread created to monitor controller
2082  * faults.
2083  *
2084  * Return: Nothing.
2085  */
2086 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2087 {
2088         unsigned long flags;
2089         struct workqueue_struct *wq;
2090
2091         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2092         wq = mrioc->watchdog_work_q;
2093         mrioc->watchdog_work_q = NULL;
2094         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2095         if (wq) {
2096                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2097                         flush_workqueue(wq);
2098                 destroy_workqueue(wq);
2099         }
2100 }
2101
2102 /**
2103  * mpi3mr_kill_ioc - Kill the controller
2104  * @mrioc: Adapter instance reference
2105  * @reason: reason for the failure.
2106  *
2107  * If fault debug is enabled, display the fault info else issue
2108  * diag fault and freeze the system for controller debug
2109  * purpose.
2110  *
2111  * Return: Nothing.
2112  */
2113 static void mpi3mr_kill_ioc(struct mpi3mr_ioc *mrioc, u32 reason)
2114 {
2115         enum mpi3mr_iocstate ioc_state;
2116
2117         if (!mrioc->fault_dbg)
2118                 return;
2119
2120         dump_stack();
2121
2122         ioc_state = mpi3mr_get_iocstate(mrioc);
2123         if (ioc_state == MRIOC_STATE_FAULT)
2124                 mpi3mr_print_fault_info(mrioc);
2125         else {
2126                 ioc_err(mrioc, "Firmware is halted due to the reason %d\n",
2127                     reason);
2128                 mpi3mr_diagfault_reset_handler(mrioc, reason);
2129         }
2130         if (mrioc->fault_dbg == 2)
2131                 for (;;)
2132                         ;
2133         else
2134                 panic("panic in %s\n", __func__);
2135 }
2136
2137 /**
2138  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2139  * @mrioc: Adapter instance reference
2140  *
2141  * Allocate memory for admin queue pair if required and register
2142  * the admin queue with the controller.
2143  *
2144  * Return: 0 on success, non-zero on failures.
2145  */
2146 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2147 {
2148         int retval = 0;
2149         u32 num_admin_entries = 0;
2150
2151         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2152         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2153             MPI3MR_ADMIN_REQ_FRAME_SZ;
2154         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2155         mrioc->admin_req_base = NULL;
2156
2157         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2158         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2159             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2160         mrioc->admin_reply_ci = 0;
2161         mrioc->admin_reply_ephase = 1;
2162         mrioc->admin_reply_base = NULL;
2163
2164         if (!mrioc->admin_req_base) {
2165                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2166                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2167
2168                 if (!mrioc->admin_req_base) {
2169                         retval = -1;
2170                         goto out_failed;
2171                 }
2172
2173                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2174                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2175                     GFP_KERNEL);
2176
2177                 if (!mrioc->admin_reply_base) {
2178                         retval = -1;
2179                         goto out_failed;
2180                 }
2181         }
2182
2183         num_admin_entries = (mrioc->num_admin_replies << 16) |
2184             (mrioc->num_admin_req);
2185         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2186         mpi3mr_writeq(mrioc->admin_req_dma,
2187             &mrioc->sysif_regs->admin_request_queue_address);
2188         mpi3mr_writeq(mrioc->admin_reply_dma,
2189             &mrioc->sysif_regs->admin_reply_queue_address);
2190         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2191         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2192         return retval;
2193
2194 out_failed:
2195
2196         if (mrioc->admin_reply_base) {
2197                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2198                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2199                 mrioc->admin_reply_base = NULL;
2200         }
2201         if (mrioc->admin_req_base) {
2202                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2203                     mrioc->admin_req_base, mrioc->admin_req_dma);
2204                 mrioc->admin_req_base = NULL;
2205         }
2206         return retval;
2207 }
2208
2209 /**
2210  * mpi3mr_issue_iocfacts - Send IOC Facts
2211  * @mrioc: Adapter instance reference
2212  * @facts_data: Cached IOC facts data
2213  *
2214  * Issue IOC Facts MPI request through admin queue and wait for
2215  * the completion of it or time out.
2216  *
2217  * Return: 0 on success, non-zero on failures.
2218  */
2219 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2220         struct mpi3_ioc_facts_data *facts_data)
2221 {
2222         struct mpi3_ioc_facts_request iocfacts_req;
2223         void *data = NULL;
2224         dma_addr_t data_dma;
2225         u32 data_len = sizeof(*facts_data);
2226         int retval = 0;
2227         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2228
2229         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2230             GFP_KERNEL);
2231
2232         if (!data) {
2233                 retval = -1;
2234                 goto out;
2235         }
2236
2237         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2238         mutex_lock(&mrioc->init_cmds.mutex);
2239         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2240                 retval = -1;
2241                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2242                 mutex_unlock(&mrioc->init_cmds.mutex);
2243                 goto out;
2244         }
2245         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2246         mrioc->init_cmds.is_waiting = 1;
2247         mrioc->init_cmds.callback = NULL;
2248         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2249         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2250
2251         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2252             data_dma);
2253
2254         init_completion(&mrioc->init_cmds.done);
2255         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2256             sizeof(iocfacts_req), 1);
2257         if (retval) {
2258                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2259                 goto out_unlock;
2260         }
2261         wait_for_completion_timeout(&mrioc->init_cmds.done,
2262             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2263         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2264                 ioc_err(mrioc, "Issue IOCFacts: command timed out\n");
2265                 mpi3mr_set_diagsave(mrioc);
2266                 mpi3mr_issue_reset(mrioc,
2267                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2268                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2269                 mrioc->unrecoverable = 1;
2270                 retval = -1;
2271                 goto out_unlock;
2272         }
2273         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2274             != MPI3_IOCSTATUS_SUCCESS) {
2275                 ioc_err(mrioc,
2276                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2277                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2278                     mrioc->init_cmds.ioc_loginfo);
2279                 retval = -1;
2280                 goto out_unlock;
2281         }
2282         memcpy(facts_data, (u8 *)data, data_len);
2283 out_unlock:
2284         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2285         mutex_unlock(&mrioc->init_cmds.mutex);
2286
2287 out:
2288         if (data)
2289                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2290
2291         return retval;
2292 }
2293
2294 /**
2295  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2296  * @mrioc: Adapter instance reference
2297  *
2298  * Check whether the new DMA mask requested through IOCFacts by
2299  * firmware needs to be set, if so set it .
2300  *
2301  * Return: 0 on success, non-zero on failure.
2302  */
2303 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2304 {
2305         struct pci_dev *pdev = mrioc->pdev;
2306         int r;
2307         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2308
2309         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2310                 return 0;
2311
2312         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2313             mrioc->dma_mask, facts_dma_mask);
2314
2315         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2316         if (r) {
2317                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2318                     facts_dma_mask, r);
2319                 return r;
2320         }
2321         mrioc->dma_mask = facts_dma_mask;
2322         return r;
2323 }
2324
2325 /**
2326  * mpi3mr_process_factsdata - Process IOC facts data
2327  * @mrioc: Adapter instance reference
2328  * @facts_data: Cached IOC facts data
2329  *
2330  * Convert IOC facts data into cpu endianness and cache it in
2331  * the driver .
2332  *
2333  * Return: Nothing.
2334  */
2335 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2336         struct mpi3_ioc_facts_data *facts_data)
2337 {
2338         u32 ioc_config, req_sz, facts_flags;
2339
2340         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2341             (sizeof(*facts_data) / 4)) {
2342                 ioc_warn(mrioc,
2343                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2344                     sizeof(*facts_data),
2345                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2346         }
2347
2348         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2349         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2350             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2351         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2352                 ioc_err(mrioc,
2353                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2354                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2355         }
2356
2357         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2358
2359         facts_flags = le32_to_cpu(facts_data->flags);
2360         mrioc->facts.op_req_sz = req_sz;
2361         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2362             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2363             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2364
2365         mrioc->facts.ioc_num = facts_data->ioc_number;
2366         mrioc->facts.who_init = facts_data->who_init;
2367         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2368         mrioc->facts.personality = (facts_flags &
2369             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2370         mrioc->facts.dma_mask = (facts_flags &
2371             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2372             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2373         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2374         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2375         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_request);
2376         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2377         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2378         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2379         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2380         mrioc->facts.max_pds = le16_to_cpu(facts_data->max_pds);
2381         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2382         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2383         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_advanced_host_pds);
2384         mrioc->facts.max_raidpds = le16_to_cpu(facts_data->max_raid_pds);
2385         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2386         mrioc->facts.max_pcie_switches =
2387             le16_to_cpu(facts_data->max_pc_ie_switches);
2388         mrioc->facts.max_sasexpanders =
2389             le16_to_cpu(facts_data->max_sas_expanders);
2390         mrioc->facts.max_sasinitiators =
2391             le16_to_cpu(facts_data->max_sas_initiators);
2392         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2393         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2394         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2395         mrioc->facts.max_op_req_q =
2396             le16_to_cpu(facts_data->max_operational_request_queues);
2397         mrioc->facts.max_op_reply_q =
2398             le16_to_cpu(facts_data->max_operational_reply_queues);
2399         mrioc->facts.ioc_capabilities =
2400             le32_to_cpu(facts_data->ioc_capabilities);
2401         mrioc->facts.fw_ver.build_num =
2402             le16_to_cpu(facts_data->fw_version.build_num);
2403         mrioc->facts.fw_ver.cust_id =
2404             le16_to_cpu(facts_data->fw_version.customer_id);
2405         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2406         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2407         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2408         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2409         mrioc->msix_count = min_t(int, mrioc->msix_count,
2410             mrioc->facts.max_msix_vectors);
2411         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2412         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2413         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2414         mrioc->facts.shutdown_timeout =
2415             le16_to_cpu(facts_data->shutdown_timeout);
2416
2417         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2418             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2419             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2420         ioc_info(mrioc,
2421             "maxreqs(%d), mindh(%d) maxPDs(%d) maxvectors(%d) maxperids(%d)\n",
2422             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2423             mrioc->facts.max_pds, mrioc->facts.max_msix_vectors,
2424             mrioc->facts.max_perids);
2425         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2426             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2427             mrioc->facts.sge_mod_shift);
2428         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2429             mrioc->facts.dma_mask, (facts_flags &
2430             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2431
2432         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
2433
2434         if (reset_devices)
2435                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
2436                     MPI3MR_HOST_IOS_KDUMP);
2437 }
2438
2439 /**
2440  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2441  * @mrioc: Adapter instance reference
2442  *
2443  * Allocate and initialize the reply free buffers, sense
2444  * buffers, reply free queue and sense buffer queue.
2445  *
2446  * Return: 0 on success, non-zero on failures.
2447  */
2448 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2449 {
2450         int retval = 0;
2451         u32 sz, i;
2452         dma_addr_t phy_addr;
2453
2454         if (mrioc->init_cmds.reply)
2455                 goto post_reply_sbuf;
2456
2457         mrioc->init_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2458         if (!mrioc->init_cmds.reply)
2459                 goto out_failed;
2460
2461         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2462                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->facts.reply_sz,
2463                     GFP_KERNEL);
2464                 if (!mrioc->dev_rmhs_cmds[i].reply)
2465                         goto out_failed;
2466         }
2467
2468         mrioc->host_tm_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2469         if (!mrioc->host_tm_cmds.reply)
2470                 goto out_failed;
2471
2472         mrioc->dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8;
2473         if (mrioc->facts.max_devhandle % 8)
2474                 mrioc->dev_handle_bitmap_sz++;
2475         mrioc->removepend_bitmap = kzalloc(mrioc->dev_handle_bitmap_sz,
2476             GFP_KERNEL);
2477         if (!mrioc->removepend_bitmap)
2478                 goto out_failed;
2479
2480         mrioc->devrem_bitmap_sz = MPI3MR_NUM_DEVRMCMD / 8;
2481         if (MPI3MR_NUM_DEVRMCMD % 8)
2482                 mrioc->devrem_bitmap_sz++;
2483         mrioc->devrem_bitmap = kzalloc(mrioc->devrem_bitmap_sz,
2484             GFP_KERNEL);
2485         if (!mrioc->devrem_bitmap)
2486                 goto out_failed;
2487
2488         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2489         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2490         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2491         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2492
2493         /* reply buffer pool, 16 byte align */
2494         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2495         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2496             &mrioc->pdev->dev, sz, 16, 0);
2497         if (!mrioc->reply_buf_pool) {
2498                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2499                 goto out_failed;
2500         }
2501
2502         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2503             &mrioc->reply_buf_dma);
2504         if (!mrioc->reply_buf)
2505                 goto out_failed;
2506
2507         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2508
2509         /* reply free queue, 8 byte align */
2510         sz = mrioc->reply_free_qsz * 8;
2511         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2512             &mrioc->pdev->dev, sz, 8, 0);
2513         if (!mrioc->reply_free_q_pool) {
2514                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2515                 goto out_failed;
2516         }
2517         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
2518             GFP_KERNEL, &mrioc->reply_free_q_dma);
2519         if (!mrioc->reply_free_q)
2520                 goto out_failed;
2521
2522         /* sense buffer pool,  4 byte align */
2523         sz = mrioc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;
2524         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
2525             &mrioc->pdev->dev, sz, 4, 0);
2526         if (!mrioc->sense_buf_pool) {
2527                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
2528                 goto out_failed;
2529         }
2530         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
2531             &mrioc->sense_buf_dma);
2532         if (!mrioc->sense_buf)
2533                 goto out_failed;
2534
2535         /* sense buffer queue, 8 byte align */
2536         sz = mrioc->sense_buf_q_sz * 8;
2537         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
2538             &mrioc->pdev->dev, sz, 8, 0);
2539         if (!mrioc->sense_buf_q_pool) {
2540                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
2541                 goto out_failed;
2542         }
2543         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
2544             GFP_KERNEL, &mrioc->sense_buf_q_dma);
2545         if (!mrioc->sense_buf_q)
2546                 goto out_failed;
2547
2548 post_reply_sbuf:
2549         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2550         ioc_info(mrioc,
2551             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2552             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->facts.reply_sz,
2553             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
2554         sz = mrioc->reply_free_qsz * 8;
2555         ioc_info(mrioc,
2556             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2557             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
2558             (unsigned long long)mrioc->reply_free_q_dma);
2559         sz = mrioc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;
2560         ioc_info(mrioc,
2561             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2562             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSEBUF_SZ,
2563             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
2564         sz = mrioc->sense_buf_q_sz * 8;
2565         ioc_info(mrioc,
2566             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2567             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
2568             (unsigned long long)mrioc->sense_buf_q_dma);
2569
2570         /* initialize Reply buffer Queue */
2571         for (i = 0, phy_addr = mrioc->reply_buf_dma;
2572             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->facts.reply_sz)
2573                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
2574         mrioc->reply_free_q[i] = cpu_to_le64(0);
2575
2576         /* initialize Sense Buffer Queue */
2577         for (i = 0, phy_addr = mrioc->sense_buf_dma;
2578             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSEBUF_SZ)
2579                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
2580         mrioc->sense_buf_q[i] = cpu_to_le64(0);
2581         return retval;
2582
2583 out_failed:
2584         retval = -1;
2585         return retval;
2586 }
2587
2588 /**
2589  * mpi3mr_issue_iocinit - Send IOC Init
2590  * @mrioc: Adapter instance reference
2591  *
2592  * Issue IOC Init MPI request through admin queue and wait for
2593  * the completion of it or time out.
2594  *
2595  * Return: 0 on success, non-zero on failures.
2596  */
2597 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
2598 {
2599         struct mpi3_ioc_init_request iocinit_req;
2600         struct mpi3_driver_info_layout *drv_info;
2601         dma_addr_t data_dma;
2602         u32 data_len = sizeof(*drv_info);
2603         int retval = 0;
2604         ktime_t current_time;
2605
2606         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2607             GFP_KERNEL);
2608         if (!drv_info) {
2609                 retval = -1;
2610                 goto out;
2611         }
2612         drv_info->information_length = cpu_to_le32(data_len);
2613         strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
2614         strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
2615         strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
2616         strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
2617         strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
2618         strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
2619             sizeof(drv_info->driver_release_date));
2620         drv_info->driver_capabilities = 0;
2621         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
2622             sizeof(mrioc->driver_info));
2623
2624         memset(&iocinit_req, 0, sizeof(iocinit_req));
2625         mutex_lock(&mrioc->init_cmds.mutex);
2626         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2627                 retval = -1;
2628                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
2629                 mutex_unlock(&mrioc->init_cmds.mutex);
2630                 goto out;
2631         }
2632         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2633         mrioc->init_cmds.is_waiting = 1;
2634         mrioc->init_cmds.callback = NULL;
2635         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2636         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
2637         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
2638         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
2639         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
2640         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
2641         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
2642         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
2643         iocinit_req.reply_free_queue_address =
2644             cpu_to_le64(mrioc->reply_free_q_dma);
2645         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSEBUF_SZ);
2646         iocinit_req.sense_buffer_free_queue_depth =
2647             cpu_to_le16(mrioc->sense_buf_q_sz);
2648         iocinit_req.sense_buffer_free_queue_address =
2649             cpu_to_le64(mrioc->sense_buf_q_dma);
2650         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
2651
2652         current_time = ktime_get_real();
2653         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
2654
2655         init_completion(&mrioc->init_cmds.done);
2656         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
2657             sizeof(iocinit_req), 1);
2658         if (retval) {
2659                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
2660                 goto out_unlock;
2661         }
2662         wait_for_completion_timeout(&mrioc->init_cmds.done,
2663             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2664         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2665                 mpi3mr_set_diagsave(mrioc);
2666                 mpi3mr_issue_reset(mrioc,
2667                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2668                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
2669                 mrioc->unrecoverable = 1;
2670                 ioc_err(mrioc, "Issue IOCInit: command timed out\n");
2671                 retval = -1;
2672                 goto out_unlock;
2673         }
2674         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2675             != MPI3_IOCSTATUS_SUCCESS) {
2676                 ioc_err(mrioc,
2677                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2678                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2679                     mrioc->init_cmds.ioc_loginfo);
2680                 retval = -1;
2681                 goto out_unlock;
2682         }
2683
2684 out_unlock:
2685         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2686         mutex_unlock(&mrioc->init_cmds.mutex);
2687
2688 out:
2689         if (drv_info)
2690                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
2691                     data_dma);
2692
2693         return retval;
2694 }
2695
2696 /**
2697  * mpi3mr_unmask_events - Unmask events in event mask bitmap
2698  * @mrioc: Adapter instance reference
2699  * @event: MPI event ID
2700  *
2701  * Un mask the specific event by resetting the event_mask
2702  * bitmap.
2703  *
2704  * Return: 0 on success, non-zero on failures.
2705  */
2706 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
2707 {
2708         u32 desired_event;
2709         u8 word;
2710
2711         if (event >= 128)
2712                 return;
2713
2714         desired_event = (1 << (event % 32));
2715         word = event / 32;
2716
2717         mrioc->event_masks[word] &= ~desired_event;
2718 }
2719
2720 /**
2721  * mpi3mr_issue_event_notification - Send event notification
2722  * @mrioc: Adapter instance reference
2723  *
2724  * Issue event notification MPI request through admin queue and
2725  * wait for the completion of it or time out.
2726  *
2727  * Return: 0 on success, non-zero on failures.
2728  */
2729 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
2730 {
2731         struct mpi3_event_notification_request evtnotify_req;
2732         int retval = 0;
2733         u8 i;
2734
2735         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
2736         mutex_lock(&mrioc->init_cmds.mutex);
2737         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2738                 retval = -1;
2739                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
2740                 mutex_unlock(&mrioc->init_cmds.mutex);
2741                 goto out;
2742         }
2743         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2744         mrioc->init_cmds.is_waiting = 1;
2745         mrioc->init_cmds.callback = NULL;
2746         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2747         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
2748         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2749                 evtnotify_req.event_masks[i] =
2750                     cpu_to_le32(mrioc->event_masks[i]);
2751         init_completion(&mrioc->init_cmds.done);
2752         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
2753             sizeof(evtnotify_req), 1);
2754         if (retval) {
2755                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
2756                 goto out_unlock;
2757         }
2758         wait_for_completion_timeout(&mrioc->init_cmds.done,
2759             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2760         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2761                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2762                 mpi3mr_set_diagsave(mrioc);
2763                 mpi3mr_issue_reset(mrioc,
2764                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2765                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
2766                 mrioc->unrecoverable = 1;
2767                 retval = -1;
2768                 goto out_unlock;
2769         }
2770         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2771             != MPI3_IOCSTATUS_SUCCESS) {
2772                 ioc_err(mrioc,
2773                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2774                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2775                     mrioc->init_cmds.ioc_loginfo);
2776                 retval = -1;
2777                 goto out_unlock;
2778         }
2779
2780 out_unlock:
2781         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2782         mutex_unlock(&mrioc->init_cmds.mutex);
2783 out:
2784         return retval;
2785 }
2786
2787 /**
2788  * mpi3mr_send_event_ack - Send event acknowledgment
2789  * @mrioc: Adapter instance reference
2790  * @event: MPI3 event ID
2791  * @event_ctx: Event context
2792  *
2793  * Send event acknowledgment through admin queue and wait for
2794  * it to complete.
2795  *
2796  * Return: 0 on success, non-zero on failures.
2797  */
2798 int mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
2799         u32 event_ctx)
2800 {
2801         struct mpi3_event_ack_request evtack_req;
2802         int retval = 0;
2803
2804         memset(&evtack_req, 0, sizeof(evtack_req));
2805         mutex_lock(&mrioc->init_cmds.mutex);
2806         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2807                 retval = -1;
2808                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
2809                 mutex_unlock(&mrioc->init_cmds.mutex);
2810                 goto out;
2811         }
2812         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2813         mrioc->init_cmds.is_waiting = 1;
2814         mrioc->init_cmds.callback = NULL;
2815         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2816         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
2817         evtack_req.event = event;
2818         evtack_req.event_context = cpu_to_le32(event_ctx);
2819
2820         init_completion(&mrioc->init_cmds.done);
2821         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
2822             sizeof(evtack_req), 1);
2823         if (retval) {
2824                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
2825                 goto out_unlock;
2826         }
2827         wait_for_completion_timeout(&mrioc->init_cmds.done,
2828             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2829         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2830                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2831                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2832                         mpi3mr_soft_reset_handler(mrioc,
2833                             MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
2834                 retval = -1;
2835                 goto out_unlock;
2836         }
2837         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2838             != MPI3_IOCSTATUS_SUCCESS) {
2839                 ioc_err(mrioc,
2840                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2841                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2842                     mrioc->init_cmds.ioc_loginfo);
2843                 retval = -1;
2844                 goto out_unlock;
2845         }
2846
2847 out_unlock:
2848         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2849         mutex_unlock(&mrioc->init_cmds.mutex);
2850 out:
2851         return retval;
2852 }
2853
2854 /**
2855  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
2856  * @mrioc: Adapter instance reference
2857  *
2858  * Allocate chain buffers and set a bitmap to indicate free
2859  * chain buffers. Chain buffers are used to pass the SGE
2860  * information along with MPI3 SCSI IO requests for host I/O.
2861  *
2862  * Return: 0 on success, non-zero on failure
2863  */
2864 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
2865 {
2866         int retval = 0;
2867         u32 sz, i;
2868         u16 num_chains;
2869
2870         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
2871
2872         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
2873             | SHOST_DIX_TYPE1_PROTECTION
2874             | SHOST_DIX_TYPE2_PROTECTION
2875             | SHOST_DIX_TYPE3_PROTECTION))
2876                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
2877
2878         mrioc->chain_buf_count = num_chains;
2879         sz = sizeof(struct chain_element) * num_chains;
2880         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
2881         if (!mrioc->chain_sgl_list)
2882                 goto out_failed;
2883
2884         sz = MPI3MR_PAGE_SIZE_4K;
2885         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
2886             &mrioc->pdev->dev, sz, 16, 0);
2887         if (!mrioc->chain_buf_pool) {
2888                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
2889                 goto out_failed;
2890         }
2891
2892         for (i = 0; i < num_chains; i++) {
2893                 mrioc->chain_sgl_list[i].addr =
2894                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
2895                     &mrioc->chain_sgl_list[i].dma_addr);
2896
2897                 if (!mrioc->chain_sgl_list[i].addr)
2898                         goto out_failed;
2899         }
2900         mrioc->chain_bitmap_sz = num_chains / 8;
2901         if (num_chains % 8)
2902                 mrioc->chain_bitmap_sz++;
2903         mrioc->chain_bitmap = kzalloc(mrioc->chain_bitmap_sz, GFP_KERNEL);
2904         if (!mrioc->chain_bitmap)
2905                 goto out_failed;
2906         return retval;
2907 out_failed:
2908         retval = -1;
2909         return retval;
2910 }
2911
2912 /**
2913  * mpi3mr_port_enable_complete - Mark port enable complete
2914  * @mrioc: Adapter instance reference
2915  * @drv_cmd: Internal command tracker
2916  *
2917  * Call back for asynchronous port enable request sets the
2918  * driver command to indicate port enable request is complete.
2919  *
2920  * Return: Nothing
2921  */
2922 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
2923         struct mpi3mr_drv_cmd *drv_cmd)
2924 {
2925         drv_cmd->state = MPI3MR_CMD_NOTUSED;
2926         drv_cmd->callback = NULL;
2927         mrioc->scan_failed = drv_cmd->ioc_status;
2928         mrioc->scan_started = 0;
2929 }
2930
2931 /**
2932  * mpi3mr_issue_port_enable - Issue Port Enable
2933  * @mrioc: Adapter instance reference
2934  * @async: Flag to wait for completion or not
2935  *
2936  * Issue Port Enable MPI request through admin queue and if the
2937  * async flag is not set wait for the completion of the port
2938  * enable or time out.
2939  *
2940  * Return: 0 on success, non-zero on failures.
2941  */
2942 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
2943 {
2944         struct mpi3_port_enable_request pe_req;
2945         int retval = 0;
2946         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
2947
2948         memset(&pe_req, 0, sizeof(pe_req));
2949         mutex_lock(&mrioc->init_cmds.mutex);
2950         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2951                 retval = -1;
2952                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
2953                 mutex_unlock(&mrioc->init_cmds.mutex);
2954                 goto out;
2955         }
2956         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2957         if (async) {
2958                 mrioc->init_cmds.is_waiting = 0;
2959                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
2960         } else {
2961                 mrioc->init_cmds.is_waiting = 1;
2962                 mrioc->init_cmds.callback = NULL;
2963                 init_completion(&mrioc->init_cmds.done);
2964         }
2965         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2966         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
2967
2968         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
2969         if (retval) {
2970                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
2971                 goto out_unlock;
2972         }
2973         if (!async) {
2974                 wait_for_completion_timeout(&mrioc->init_cmds.done,
2975                     (pe_timeout * HZ));
2976                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2977                         ioc_err(mrioc, "Issue PortEnable: command timed out\n");
2978                         retval = -1;
2979                         mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
2980                         mpi3mr_set_diagsave(mrioc);
2981                         mpi3mr_issue_reset(mrioc,
2982                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2983                             MPI3MR_RESET_FROM_PE_TIMEOUT);
2984                         mrioc->unrecoverable = 1;
2985                         goto out_unlock;
2986                 }
2987                 mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
2988         }
2989 out_unlock:
2990         mutex_unlock(&mrioc->init_cmds.mutex);
2991 out:
2992         return retval;
2993 }
2994
2995 /* Protocol type to name mapper structure*/
2996 static const struct {
2997         u8 protocol;
2998         char *name;
2999 } mpi3mr_protocols[] = {
3000         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
3001         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3002         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3003 };
3004
3005 /* Capability to name mapper structure*/
3006 static const struct {
3007         u32 capability;
3008         char *name;
3009 } mpi3mr_capabilities[] = {
3010         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3011 };
3012
3013 /**
3014  * mpi3mr_print_ioc_info - Display controller information
3015  * @mrioc: Adapter instance reference
3016  *
3017  * Display controller personalit, capability, supported
3018  * protocols etc.
3019  *
3020  * Return: Nothing
3021  */
3022 static void
3023 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3024 {
3025         int i = 0, bytes_written = 0;
3026         char personality[16];
3027         char protocol[50] = {0};
3028         char capabilities[100] = {0};
3029         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3030
3031         switch (mrioc->facts.personality) {
3032         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3033                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3034                 break;
3035         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3036                 strncpy(personality, "RAID", sizeof(personality));
3037                 break;
3038         default:
3039                 strncpy(personality, "Unknown", sizeof(personality));
3040                 break;
3041         }
3042
3043         ioc_info(mrioc, "Running in %s Personality", personality);
3044
3045         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3046             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3047             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3048
3049         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3050                 if (mrioc->facts.protocol_flags &
3051                     mpi3mr_protocols[i].protocol) {
3052                         bytes_written += scnprintf(protocol + bytes_written,
3053                                     sizeof(protocol) - bytes_written, "%s%s",
3054                                     bytes_written ? "," : "",
3055                                     mpi3mr_protocols[i].name);
3056                 }
3057         }
3058
3059         bytes_written = 0;
3060         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3061                 if (mrioc->facts.protocol_flags &
3062                     mpi3mr_capabilities[i].capability) {
3063                         bytes_written += scnprintf(capabilities + bytes_written,
3064                                     sizeof(capabilities) - bytes_written, "%s%s",
3065                                     bytes_written ? "," : "",
3066                                     mpi3mr_capabilities[i].name);
3067                 }
3068         }
3069
3070         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3071                  protocol, capabilities);
3072 }
3073
3074 /**
3075  * mpi3mr_cleanup_resources - Free PCI resources
3076  * @mrioc: Adapter instance reference
3077  *
3078  * Unmap PCI device memory and disable PCI device.
3079  *
3080  * Return: 0 on success and non-zero on failure.
3081  */
3082 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3083 {
3084         struct pci_dev *pdev = mrioc->pdev;
3085
3086         mpi3mr_cleanup_isr(mrioc);
3087
3088         if (mrioc->sysif_regs) {
3089                 iounmap((void __iomem *)mrioc->sysif_regs);
3090                 mrioc->sysif_regs = NULL;
3091         }
3092
3093         if (pci_is_enabled(pdev)) {
3094                 if (mrioc->bars)
3095                         pci_release_selected_regions(pdev, mrioc->bars);
3096                 pci_disable_device(pdev);
3097         }
3098 }
3099
3100 /**
3101  * mpi3mr_setup_resources - Enable PCI resources
3102  * @mrioc: Adapter instance reference
3103  *
3104  * Enable PCI device memory, MSI-x registers and set DMA mask.
3105  *
3106  * Return: 0 on success and non-zero on failure.
3107  */
3108 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3109 {
3110         struct pci_dev *pdev = mrioc->pdev;
3111         u32 memap_sz = 0;
3112         int i, retval = 0, capb = 0;
3113         u16 message_control;
3114         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3115             (((dma_get_required_mask(&pdev->dev) > DMA_BIT_MASK(32)) &&
3116             (sizeof(dma_addr_t) > 4)) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3117
3118         if (pci_enable_device_mem(pdev)) {
3119                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3120                 retval = -ENODEV;
3121                 goto out_failed;
3122         }
3123
3124         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3125         if (!capb) {
3126                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3127                 retval = -ENODEV;
3128                 goto out_failed;
3129         }
3130         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3131
3132         if (pci_request_selected_regions(pdev, mrioc->bars,
3133             mrioc->driver_name)) {
3134                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3135                 retval = -ENODEV;
3136                 goto out_failed;
3137         }
3138
3139         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3140                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3141                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3142                         memap_sz = pci_resource_len(pdev, i);
3143                         mrioc->sysif_regs =
3144                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3145                         break;
3146                 }
3147         }
3148
3149         pci_set_master(pdev);
3150
3151         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3152         if (retval) {
3153                 if (dma_mask != DMA_BIT_MASK(32)) {
3154                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3155                         dma_mask = DMA_BIT_MASK(32);
3156                         retval = dma_set_mask_and_coherent(&pdev->dev,
3157                             dma_mask);
3158                 }
3159                 if (retval) {
3160                         mrioc->dma_mask = 0;
3161                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3162                         goto out_failed;
3163                 }
3164         }
3165         mrioc->dma_mask = dma_mask;
3166
3167         if (!mrioc->sysif_regs) {
3168                 ioc_err(mrioc,
3169                     "Unable to map adapter memory or resource not found\n");
3170                 retval = -EINVAL;
3171                 goto out_failed;
3172         }
3173
3174         pci_read_config_word(pdev, capb + 2, &message_control);
3175         mrioc->msix_count = (message_control & 0x3FF) + 1;
3176
3177         pci_save_state(pdev);
3178
3179         pci_set_drvdata(pdev, mrioc->shost);
3180
3181         mpi3mr_ioc_disable_intr(mrioc);
3182
3183         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3184             (unsigned long long)mrioc->sysif_regs_phys,
3185             mrioc->sysif_regs, memap_sz);
3186         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3187             mrioc->msix_count);
3188         return retval;
3189
3190 out_failed:
3191         mpi3mr_cleanup_resources(mrioc);
3192         return retval;
3193 }
3194
3195 /**
3196  * mpi3mr_init_ioc - Initialize the controller
3197  * @mrioc: Adapter instance reference
3198  * @init_type: Flag to indicate is the init_type
3199  *
3200  * This the controller initialization routine, executed either
3201  * after soft reset or from pci probe callback.
3202  * Setup the required resources, memory map the controller
3203  * registers, create admin and operational reply queue pairs,
3204  * allocate required memory for reply pool, sense buffer pool,
3205  * issue IOC init request to the firmware, unmask the events and
3206  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3207  * volumes.
3208  *
3209  * Return: 0 on success and non-zero on failure.
3210  */
3211 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc, u8 init_type)
3212 {
3213         int retval = 0;
3214         enum mpi3mr_iocstate ioc_state;
3215         u64 base_info;
3216         u32 timeout;
3217         u32 ioc_status, ioc_config, i;
3218         struct mpi3_ioc_facts_data facts_data;
3219
3220         mrioc->irqpoll_sleep = MPI3MR_IRQ_POLL_SLEEP;
3221         mrioc->change_count = 0;
3222         if (init_type == MPI3MR_IT_INIT) {
3223                 mrioc->cpu_count = num_online_cpus();
3224                 retval = mpi3mr_setup_resources(mrioc);
3225                 if (retval) {
3226                         ioc_err(mrioc, "Failed to setup resources:error %d\n",
3227                             retval);
3228                         goto out_nocleanup;
3229                 }
3230         }
3231
3232         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3233         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3234
3235         ioc_info(mrioc, "SOD status %x configuration %x\n",
3236             ioc_status, ioc_config);
3237
3238         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
3239         ioc_info(mrioc, "SOD base_info %llx\n", base_info);
3240
3241         /*The timeout value is in 2sec unit, changing it to seconds*/
3242         mrioc->ready_timeout =
3243             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
3244             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
3245
3246         ioc_info(mrioc, "IOC ready timeout %d\n", mrioc->ready_timeout);
3247
3248         ioc_state = mpi3mr_get_iocstate(mrioc);
3249         ioc_info(mrioc, "IOC in %s state during detection\n",
3250             mpi3mr_iocstate_name(ioc_state));
3251
3252         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
3253             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
3254                 timeout = mrioc->ready_timeout * 10;
3255                 do {
3256                         msleep(100);
3257                 } while (--timeout);
3258
3259                 ioc_state = mpi3mr_get_iocstate(mrioc);
3260                 ioc_info(mrioc,
3261                     "IOC in %s state after waiting for reset time\n",
3262                     mpi3mr_iocstate_name(ioc_state));
3263         }
3264
3265         if (ioc_state == MRIOC_STATE_READY) {
3266                 retval = mpi3mr_issue_and_process_mur(mrioc,
3267                     MPI3MR_RESET_FROM_BRINGUP);
3268                 if (retval) {
3269                         ioc_err(mrioc, "Failed to MU reset IOC error %d\n",
3270                             retval);
3271                 }
3272                 ioc_state = mpi3mr_get_iocstate(mrioc);
3273         }
3274         if (ioc_state != MRIOC_STATE_RESET) {
3275                 mpi3mr_print_fault_info(mrioc);
3276                 retval = mpi3mr_issue_reset(mrioc,
3277                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3278                     MPI3MR_RESET_FROM_BRINGUP);
3279                 if (retval) {
3280                         ioc_err(mrioc,
3281                             "%s :Failed to soft reset IOC error %d\n",
3282                             __func__, retval);
3283                         goto out_failed;
3284                 }
3285         }
3286         ioc_state = mpi3mr_get_iocstate(mrioc);
3287         if (ioc_state != MRIOC_STATE_RESET) {
3288                 retval = -1;
3289                 ioc_err(mrioc, "Cannot bring IOC to reset state\n");
3290                 goto out_failed;
3291         }
3292
3293         retval = mpi3mr_setup_admin_qpair(mrioc);
3294         if (retval) {
3295                 ioc_err(mrioc, "Failed to setup admin Qs: error %d\n",
3296                     retval);
3297                 goto out_failed;
3298         }
3299
3300         retval = mpi3mr_bring_ioc_ready(mrioc);
3301         if (retval) {
3302                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3303                     retval);
3304                 goto out_failed;
3305         }
3306
3307         if (init_type != MPI3MR_IT_RESET) {
3308                 retval = mpi3mr_setup_isr(mrioc, 1);
3309                 if (retval) {
3310                         ioc_err(mrioc, "Failed to setup ISR error %d\n",
3311                             retval);
3312                         goto out_failed;
3313                 }
3314         } else
3315                 mpi3mr_ioc_enable_intr(mrioc);
3316
3317         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3318         if (retval) {
3319                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3320                     retval);
3321                 goto out_failed;
3322         }
3323
3324         mpi3mr_process_factsdata(mrioc, &facts_data);
3325         if (init_type == MPI3MR_IT_INIT) {
3326                 retval = mpi3mr_check_reset_dma_mask(mrioc);
3327                 if (retval) {
3328                         ioc_err(mrioc, "Resetting dma mask failed %d\n",
3329                             retval);
3330                         goto out_failed;
3331                 }
3332         }
3333
3334         mpi3mr_print_ioc_info(mrioc);
3335
3336         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3337         if (retval) {
3338                 ioc_err(mrioc,
3339                     "%s :Failed to allocated reply sense buffers %d\n",
3340                     __func__, retval);
3341                 goto out_failed;
3342         }
3343
3344         if (init_type == MPI3MR_IT_INIT) {
3345                 retval = mpi3mr_alloc_chain_bufs(mrioc);
3346                 if (retval) {
3347                         ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3348                             retval);
3349                         goto out_failed;
3350                 }
3351         }
3352
3353         retval = mpi3mr_issue_iocinit(mrioc);
3354         if (retval) {
3355                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3356                     retval);
3357                 goto out_failed;
3358         }
3359         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3360         writel(mrioc->reply_free_queue_host_index,
3361             &mrioc->sysif_regs->reply_free_host_index);
3362
3363         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3364         writel(mrioc->sbq_host_index,
3365             &mrioc->sysif_regs->sense_buffer_free_host_index);
3366
3367         if (init_type != MPI3MR_IT_RESET) {
3368                 retval = mpi3mr_setup_isr(mrioc, 0);
3369                 if (retval) {
3370                         ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3371                             retval);
3372                         goto out_failed;
3373                 }
3374         }
3375
3376         retval = mpi3mr_create_op_queues(mrioc);
3377         if (retval) {
3378                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3379                     retval);
3380                 goto out_failed;
3381         }
3382
3383         if ((init_type != MPI3MR_IT_INIT) &&
3384             (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q)) {
3385                 retval = -1;
3386                 ioc_err(mrioc,
3387                     "Cannot create minimum number of OpQueues expected:%d created:%d\n",
3388                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
3389                 goto out_failed;
3390         }
3391
3392         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3393                 mrioc->event_masks[i] = -1;
3394
3395         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3396         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3397         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3398         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3399         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3400         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3401         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3402         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3403         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3404         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3405         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3406         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3407
3408         retval = mpi3mr_issue_event_notification(mrioc);
3409         if (retval) {
3410                 ioc_err(mrioc, "Failed to issue event notification %d\n",
3411                     retval);
3412                 goto out_failed;
3413         }
3414
3415         if (init_type != MPI3MR_IT_INIT) {
3416                 ioc_info(mrioc, "Issuing Port Enable\n");
3417                 retval = mpi3mr_issue_port_enable(mrioc, 0);
3418                 if (retval) {
3419                         ioc_err(mrioc, "Failed to issue port enable %d\n",
3420                             retval);
3421                         goto out_failed;
3422                 }
3423         }
3424         return retval;
3425
3426 out_failed:
3427         if (init_type == MPI3MR_IT_INIT)
3428                 mpi3mr_cleanup_ioc(mrioc, MPI3MR_COMPLETE_CLEANUP);
3429         else
3430                 mpi3mr_cleanup_ioc(mrioc, MPI3MR_REINIT_FAILURE);
3431 out_nocleanup:
3432         return retval;
3433 }
3434
3435 /**
3436  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
3437  *                                      segments
3438  * @mrioc: Adapter instance reference
3439  * @qidx: Operational reply queue index
3440  *
3441  * Return: Nothing.
3442  */
3443 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3444 {
3445         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
3446         struct segments *segments;
3447         int i, size;
3448
3449         if (!op_reply_q->q_segments)
3450                 return;
3451
3452         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
3453         segments = op_reply_q->q_segments;
3454         for (i = 0; i < op_reply_q->num_segments; i++)
3455                 memset(segments[i].segment, 0, size);
3456 }
3457
3458 /**
3459  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
3460  *                                      segments
3461  * @mrioc: Adapter instance reference
3462  * @qidx: Operational request queue index
3463  *
3464  * Return: Nothing.
3465  */
3466 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3467 {
3468         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
3469         struct segments *segments;
3470         int i, size;
3471
3472         if (!op_req_q->q_segments)
3473                 return;
3474
3475         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
3476         segments = op_req_q->q_segments;
3477         for (i = 0; i < op_req_q->num_segments; i++)
3478                 memset(segments[i].segment, 0, size);
3479 }
3480
3481 /**
3482  * mpi3mr_memset_buffers - memset memory for a controller
3483  * @mrioc: Adapter instance reference
3484  *
3485  * clear all the memory allocated for a controller, typically
3486  * called post reset to reuse the memory allocated during the
3487  * controller init.
3488  *
3489  * Return: Nothing.
3490  */
3491 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
3492 {
3493         u16 i;
3494
3495         memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
3496         memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
3497
3498         memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
3499         memset(mrioc->host_tm_cmds.reply, 0,
3500             sizeof(*mrioc->host_tm_cmds.reply));
3501         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
3502                 memset(mrioc->dev_rmhs_cmds[i].reply, 0,
3503                     sizeof(*mrioc->dev_rmhs_cmds[i].reply));
3504         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3505         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3506
3507         for (i = 0; i < mrioc->num_queues; i++) {
3508                 mrioc->op_reply_qinfo[i].qid = 0;
3509                 mrioc->op_reply_qinfo[i].ci = 0;
3510                 mrioc->op_reply_qinfo[i].num_replies = 0;
3511                 mrioc->op_reply_qinfo[i].ephase = 0;
3512                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
3513                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
3514                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
3515
3516                 mrioc->req_qinfo[i].ci = 0;
3517                 mrioc->req_qinfo[i].pi = 0;
3518                 mrioc->req_qinfo[i].num_requests = 0;
3519                 mrioc->req_qinfo[i].qid = 0;
3520                 mrioc->req_qinfo[i].reply_qid = 0;
3521                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
3522                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
3523         }
3524 }
3525
3526 /**
3527  * mpi3mr_free_mem - Free memory allocated for a controller
3528  * @mrioc: Adapter instance reference
3529  *
3530  * Free all the memory allocated for a controller.
3531  *
3532  * Return: Nothing.
3533  */
3534 static void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
3535 {
3536         u16 i;
3537         struct mpi3mr_intr_info *intr_info;
3538
3539         if (mrioc->sense_buf_pool) {
3540                 if (mrioc->sense_buf)
3541                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
3542                             mrioc->sense_buf_dma);
3543                 dma_pool_destroy(mrioc->sense_buf_pool);
3544                 mrioc->sense_buf = NULL;
3545                 mrioc->sense_buf_pool = NULL;
3546         }
3547         if (mrioc->sense_buf_q_pool) {
3548                 if (mrioc->sense_buf_q)
3549                         dma_pool_free(mrioc->sense_buf_q_pool,
3550                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
3551                 dma_pool_destroy(mrioc->sense_buf_q_pool);
3552                 mrioc->sense_buf_q = NULL;
3553                 mrioc->sense_buf_q_pool = NULL;
3554         }
3555
3556         if (mrioc->reply_buf_pool) {
3557                 if (mrioc->reply_buf)
3558                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
3559                             mrioc->reply_buf_dma);
3560                 dma_pool_destroy(mrioc->reply_buf_pool);
3561                 mrioc->reply_buf = NULL;
3562                 mrioc->reply_buf_pool = NULL;
3563         }
3564         if (mrioc->reply_free_q_pool) {
3565                 if (mrioc->reply_free_q)
3566                         dma_pool_free(mrioc->reply_free_q_pool,
3567                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
3568                 dma_pool_destroy(mrioc->reply_free_q_pool);
3569                 mrioc->reply_free_q = NULL;
3570                 mrioc->reply_free_q_pool = NULL;
3571         }
3572
3573         for (i = 0; i < mrioc->num_op_req_q; i++)
3574                 mpi3mr_free_op_req_q_segments(mrioc, i);
3575
3576         for (i = 0; i < mrioc->num_op_reply_q; i++)
3577                 mpi3mr_free_op_reply_q_segments(mrioc, i);
3578
3579         for (i = 0; i < mrioc->intr_info_count; i++) {
3580                 intr_info = mrioc->intr_info + i;
3581                 intr_info->op_reply_q = NULL;
3582         }
3583
3584         kfree(mrioc->req_qinfo);
3585         mrioc->req_qinfo = NULL;
3586         mrioc->num_op_req_q = 0;
3587
3588         kfree(mrioc->op_reply_qinfo);
3589         mrioc->op_reply_qinfo = NULL;
3590         mrioc->num_op_reply_q = 0;
3591
3592         kfree(mrioc->init_cmds.reply);
3593         mrioc->init_cmds.reply = NULL;
3594
3595         kfree(mrioc->host_tm_cmds.reply);
3596         mrioc->host_tm_cmds.reply = NULL;
3597
3598         kfree(mrioc->removepend_bitmap);
3599         mrioc->removepend_bitmap = NULL;
3600
3601         kfree(mrioc->devrem_bitmap);
3602         mrioc->devrem_bitmap = NULL;
3603
3604         kfree(mrioc->chain_bitmap);
3605         mrioc->chain_bitmap = NULL;
3606
3607         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3608                 kfree(mrioc->dev_rmhs_cmds[i].reply);
3609                 mrioc->dev_rmhs_cmds[i].reply = NULL;
3610         }
3611
3612         if (mrioc->chain_buf_pool) {
3613                 for (i = 0; i < mrioc->chain_buf_count; i++) {
3614                         if (mrioc->chain_sgl_list[i].addr) {
3615                                 dma_pool_free(mrioc->chain_buf_pool,
3616                                     mrioc->chain_sgl_list[i].addr,
3617                                     mrioc->chain_sgl_list[i].dma_addr);
3618                                 mrioc->chain_sgl_list[i].addr = NULL;
3619                         }
3620                 }
3621                 dma_pool_destroy(mrioc->chain_buf_pool);
3622                 mrioc->chain_buf_pool = NULL;
3623         }
3624
3625         kfree(mrioc->chain_sgl_list);
3626         mrioc->chain_sgl_list = NULL;
3627
3628         if (mrioc->admin_reply_base) {
3629                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
3630                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
3631                 mrioc->admin_reply_base = NULL;
3632         }
3633         if (mrioc->admin_req_base) {
3634                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
3635                     mrioc->admin_req_base, mrioc->admin_req_dma);
3636                 mrioc->admin_req_base = NULL;
3637         }
3638 }
3639
3640 /**
3641  * mpi3mr_issue_ioc_shutdown - shutdown controller
3642  * @mrioc: Adapter instance reference
3643  *
3644  * Send shutodwn notification to the controller and wait for the
3645  * shutdown_timeout for it to be completed.
3646  *
3647  * Return: Nothing.
3648  */
3649 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
3650 {
3651         u32 ioc_config, ioc_status;
3652         u8 retval = 1;
3653         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
3654
3655         ioc_info(mrioc, "Issuing shutdown Notification\n");
3656         if (mrioc->unrecoverable) {
3657                 ioc_warn(mrioc,
3658                     "IOC is unrecoverable shutdown is not issued\n");
3659                 return;
3660         }
3661         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3662         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3663             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
3664                 ioc_info(mrioc, "shutdown already in progress\n");
3665                 return;
3666         }
3667
3668         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3669         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
3670         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN;
3671
3672         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
3673
3674         if (mrioc->facts.shutdown_timeout)
3675                 timeout = mrioc->facts.shutdown_timeout * 10;
3676
3677         do {
3678                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3679                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3680                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
3681                         retval = 0;
3682                         break;
3683                 }
3684                 msleep(100);
3685         } while (--timeout);
3686
3687         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3688         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3689
3690         if (retval) {
3691                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3692                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
3693                         ioc_warn(mrioc,
3694                             "shutdown still in progress after timeout\n");
3695         }
3696
3697         ioc_info(mrioc,
3698             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
3699             (!retval) ? "successful" : "failed", ioc_status,
3700             ioc_config);
3701 }
3702
3703 /**
3704  * mpi3mr_cleanup_ioc - Cleanup controller
3705  * @mrioc: Adapter instance reference
3706  * @reason: Cleanup reason
3707  *
3708  * controller cleanup handler, Message unit reset or soft reset
3709  * and shutdown notification is issued to the controller and the
3710  * associated memory resources are freed.
3711  *
3712  * Return: Nothing.
3713  */
3714 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc, u8 reason)
3715 {
3716         enum mpi3mr_iocstate ioc_state;
3717
3718         if (reason == MPI3MR_COMPLETE_CLEANUP)
3719                 mpi3mr_stop_watchdog(mrioc);
3720
3721         mpi3mr_ioc_disable_intr(mrioc);
3722
3723         ioc_state = mpi3mr_get_iocstate(mrioc);
3724
3725         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
3726             (ioc_state == MRIOC_STATE_READY)) {
3727                 if (mpi3mr_issue_and_process_mur(mrioc,
3728                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
3729                         mpi3mr_issue_reset(mrioc,
3730                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3731                             MPI3MR_RESET_FROM_MUR_FAILURE);
3732
3733                 if (reason != MPI3MR_REINIT_FAILURE)
3734                         mpi3mr_issue_ioc_shutdown(mrioc);
3735         }
3736
3737         if (reason == MPI3MR_COMPLETE_CLEANUP) {
3738                 mpi3mr_free_mem(mrioc);
3739                 mpi3mr_cleanup_resources(mrioc);
3740         }
3741 }
3742
3743 /**
3744  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
3745  * @mrioc: Adapter instance reference
3746  * @cmdptr: Internal command tracker
3747  *
3748  * Complete an internal driver commands with state indicating it
3749  * is completed due to reset.
3750  *
3751  * Return: Nothing.
3752  */
3753 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
3754         struct mpi3mr_drv_cmd *cmdptr)
3755 {
3756         if (cmdptr->state & MPI3MR_CMD_PENDING) {
3757                 cmdptr->state |= MPI3MR_CMD_RESET;
3758                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
3759                 if (cmdptr->is_waiting) {
3760                         complete(&cmdptr->done);
3761                         cmdptr->is_waiting = 0;
3762                 } else if (cmdptr->callback)
3763                         cmdptr->callback(mrioc, cmdptr);
3764         }
3765 }
3766
3767 /**
3768  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
3769  * @mrioc: Adapter instance reference
3770  *
3771  * Flush all internal driver commands post reset
3772  *
3773  * Return: Nothing.
3774  */
3775 static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
3776 {
3777         struct mpi3mr_drv_cmd *cmdptr;
3778         u8 i;
3779
3780         cmdptr = &mrioc->init_cmds;
3781         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3782         cmdptr = &mrioc->host_tm_cmds;
3783         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3784
3785         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3786                 cmdptr = &mrioc->dev_rmhs_cmds[i];
3787                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3788         }
3789 }
3790
3791 /**
3792  * mpi3mr_diagfault_reset_handler - Diag fault reset handler
3793  * @mrioc: Adapter instance reference
3794  * @reset_reason: Reset reason code
3795  *
3796  * This is an handler for issuing diag fault reset from the
3797  * applications through IOCTL path to stop the execution of the
3798  * controller
3799  *
3800  * Return: 0 on success, non-zero on failure.
3801  */
3802 int mpi3mr_diagfault_reset_handler(struct mpi3mr_ioc *mrioc,
3803         u32 reset_reason)
3804 {
3805         int retval = 0;
3806
3807         ioc_info(mrioc, "Entry: reason code: %s\n",
3808             mpi3mr_reset_rc_name(reset_reason));
3809         mrioc->reset_in_progress = 1;
3810
3811         mpi3mr_ioc_disable_intr(mrioc);
3812
3813         retval = mpi3mr_issue_reset(mrioc,
3814             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3815
3816         if (retval) {
3817                 ioc_err(mrioc, "The diag fault reset failed: reason %d\n",
3818                     reset_reason);
3819                 mpi3mr_ioc_enable_intr(mrioc);
3820         }
3821         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3822         mrioc->reset_in_progress = 0;
3823         return retval;
3824 }
3825
3826 /**
3827  * mpi3mr_soft_reset_handler - Reset the controller
3828  * @mrioc: Adapter instance reference
3829  * @reset_reason: Reset reason code
3830  * @snapdump: Flag to generate snapdump in firmware or not
3831  *
3832  * This is an handler for recovering controller by issuing soft
3833  * reset are diag fault reset.  This is a blocking function and
3834  * when one reset is executed if any other resets they will be
3835  * blocked. All IOCTLs/IO will be blocked during the reset. If
3836  * controller reset is successful then the controller will be
3837  * reinitalized, otherwise the controller will be marked as not
3838  * recoverable
3839  *
3840  * In snapdump bit is set, the controller is issued with diag
3841  * fault reset so that the firmware can create a snap dump and
3842  * post that the firmware will result in F000 fault and the
3843  * driver will issue soft reset to recover from that.
3844  *
3845  * Return: 0 on success, non-zero on failure.
3846  */
3847 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
3848         u32 reset_reason, u8 snapdump)
3849 {
3850         int retval = 0, i;
3851         unsigned long flags;
3852         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
3853
3854         if (mrioc->fault_dbg) {
3855                 if (snapdump)
3856                         mpi3mr_set_diagsave(mrioc);
3857                 mpi3mr_kill_ioc(mrioc, reset_reason);
3858         }
3859
3860         /*
3861          * Block new resets until the currently executing one is finished and
3862          * return the status of the existing reset for all blocked resets
3863          */
3864         if (!mutex_trylock(&mrioc->reset_mutex)) {
3865                 ioc_info(mrioc, "Another reset in progress\n");
3866                 return -1;
3867         }
3868         mrioc->reset_in_progress = 1;
3869
3870         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
3871             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
3872                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3873                         mrioc->event_masks[i] = -1;
3874
3875                 retval = mpi3mr_issue_event_notification(mrioc);
3876
3877                 if (retval) {
3878                         ioc_err(mrioc,
3879                             "Failed to turn off events prior to reset %d\n",
3880                             retval);
3881                 }
3882         }
3883
3884         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
3885
3886         mpi3mr_ioc_disable_intr(mrioc);
3887
3888         if (snapdump) {
3889                 mpi3mr_set_diagsave(mrioc);
3890                 retval = mpi3mr_issue_reset(mrioc,
3891                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3892                 if (!retval) {
3893                         do {
3894                                 host_diagnostic =
3895                                     readl(&mrioc->sysif_regs->host_diagnostic);
3896                                 if (!(host_diagnostic &
3897                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
3898                                         break;
3899                                 msleep(100);
3900                         } while (--timeout);
3901                 }
3902         }
3903
3904         retval = mpi3mr_issue_reset(mrioc,
3905             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
3906         if (retval) {
3907                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
3908                 goto out;
3909         }
3910
3911         mpi3mr_flush_delayed_rmhs_list(mrioc);
3912         mpi3mr_flush_drv_cmds(mrioc);
3913         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3914         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3915         mpi3mr_cleanup_fwevt_list(mrioc);
3916         mpi3mr_flush_host_io(mrioc);
3917         mpi3mr_invalidate_devhandles(mrioc);
3918         mpi3mr_memset_buffers(mrioc);
3919         retval = mpi3mr_init_ioc(mrioc, MPI3MR_IT_RESET);
3920         if (retval) {
3921                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
3922                     mrioc->name, reset_reason);
3923                 goto out;
3924         }
3925         ssleep(10);
3926
3927 out:
3928         if (!retval) {
3929                 mrioc->reset_in_progress = 0;
3930                 scsi_unblock_requests(mrioc->shost);
3931                 mpi3mr_rfresh_tgtdevs(mrioc);
3932                 mrioc->ts_update_counter = 0;
3933                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
3934                 if (mrioc->watchdog_work_q)
3935                         queue_delayed_work(mrioc->watchdog_work_q,
3936                             &mrioc->watchdog_work,
3937                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
3938                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
3939         } else {
3940                 mpi3mr_issue_reset(mrioc,
3941                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3942                 mrioc->unrecoverable = 1;
3943                 mrioc->reset_in_progress = 0;
3944                 retval = -1;
3945         }
3946
3947         mutex_unlock(&mrioc->reset_mutex);
3948         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3949         return retval;
3950 }