drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2021 Broadcom Inc.
   6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 #if defined(writeq) && defined(CONFIG_64BIT)
  14 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  15 {
  16         writeq(b, addr);
  17 }
  18 #else
  19 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  20 {
  21         __u64 data_out = b;
  22
  23         writel((u32)(data_out), addr);
  24         writel((u32)(data_out >> 32), (addr + 4));
  25 }
  26 #endif
  27
  28 static inline bool
  29 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  30 {
  31         u16 pi, ci, max_entries;
  32         bool is_qfull = false;
  33
  34         pi = op_req_q->pi;
  35         ci = READ_ONCE(op_req_q->ci);
  36         max_entries = op_req_q->num_requests;
  37
  38         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  39                 is_qfull = true;
  40
  41         return is_qfull;
  42 }
  43
  44 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  45 {
  46         u16 i, max_vectors;
  47
  48         max_vectors = mrioc->intr_info_count;
  49
  50         for (i = 0; i < max_vectors; i++)
  51                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  52 }
  53
  54 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  55 {
  56         mrioc->intr_enabled = 0;
  57         mpi3mr_sync_irqs(mrioc);
  58 }
  59
  60 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  61 {
  62         mrioc->intr_enabled = 1;
  63 }
  64
  65 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  66 {
  67         u16 i;
  68
  69         mpi3mr_ioc_disable_intr(mrioc);
  70
  71         if (!mrioc->intr_info)
  72                 return;
  73
  74         for (i = 0; i < mrioc->intr_info_count; i++)
  75                 free_irq(pci_irq_vector(mrioc->pdev, i),
  76                     (mrioc->intr_info + i));
  77
  78         kfree(mrioc->intr_info);
  79         mrioc->intr_info = NULL;
  80         mrioc->intr_info_count = 0;
  81         pci_free_irq_vectors(mrioc->pdev);
  82 }
  83
  84 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  85         dma_addr_t dma_addr)
  86 {
  87         struct mpi3_sge_common *sgel = paddr;
  88
  89         sgel->flags = flags;
  90         sgel->length = cpu_to_le32(length);
  91         sgel->address = cpu_to_le64(dma_addr);
  92 }
  93
  94 void mpi3mr_build_zero_len_sge(void *paddr)
  95 {
  96         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
  97
  98         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
  99 }
 100
 101 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 102         dma_addr_t phys_addr)
 103 {
 104         if (!phys_addr)
 105                 return NULL;
 106
 107         if ((phys_addr < mrioc->reply_buf_dma) ||
 108             (phys_addr > mrioc->reply_buf_dma_max_address))
 109                 return NULL;
 110
 111         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 112 }
 113
 114 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 121 }
 122
 123 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 124         u64 reply_dma)
 125 {
 126         u32 old_idx = 0;
 127         unsigned long flags;
 128
 129         spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
 130         old_idx  =  mrioc->reply_free_queue_host_index;
 131         mrioc->reply_free_queue_host_index = (
 132             (mrioc->reply_free_queue_host_index ==
 133             (mrioc->reply_free_qsz - 1)) ? 0 :
 134             (mrioc->reply_free_queue_host_index + 1));
 135         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 136         writel(mrioc->reply_free_queue_host_index,
 137             &mrioc->sysif_regs->reply_free_host_index);
 138         spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
 139 }
 140
 141 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 142         u64 sense_buf_dma)
 143 {
 144         u32 old_idx = 0;
 145         unsigned long flags;
 146
 147         spin_lock_irqsave(&mrioc->sbq_lock, flags);
 148         old_idx  =  mrioc->sbq_host_index;
 149         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 150             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 151             (mrioc->sbq_host_index + 1));
 152         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 153         writel(mrioc->sbq_host_index,
 154             &mrioc->sysif_regs->sense_buffer_free_host_index);
 155         spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
 156 }
 157
 158 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 159         struct mpi3_event_notification_reply *event_reply)
 160 {
 161         char *desc = NULL;
 162         u16 event;
 163
 164         event = event_reply->event;
 165
 166         switch (event) {
 167         case MPI3_EVENT_LOG_DATA:
 168                 desc = "Log Data";
 169                 break;
 170         case MPI3_EVENT_CHANGE:
 171                 desc = "Event Change";
 172                 break;
 173         case MPI3_EVENT_GPIO_INTERRUPT:
 174                 desc = "GPIO Interrupt";
 175                 break;
 176         case MPI3_EVENT_TEMP_THRESHOLD:
 177                 desc = "Temperature Threshold";
 178                 break;
 179         case MPI3_EVENT_CABLE_MGMT:
 180                 desc = "Cable Management";
 181                 break;
 182         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 183                 desc = "Energy Pack Change";
 184                 break;
 185         case MPI3_EVENT_DEVICE_ADDED:
 186         {
 187                 struct mpi3_device_page0 *event_data =
 188                     (struct mpi3_device_page0 *)event_reply->event_data;
 189                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 190                     event_data->dev_handle, event_data->device_form);
 191                 return;
 192         }
 193         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 194         {
 195                 struct mpi3_device_page0 *event_data =
 196                     (struct mpi3_device_page0 *)event_reply->event_data;
 197                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 198                     event_data->dev_handle, event_data->device_form);
 199                 return;
 200         }
 201         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 202         {
 203                 struct mpi3_event_data_device_status_change *event_data =
 204                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 205                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 206                     event_data->dev_handle, event_data->reason_code);
 207                 return;
 208         }
 209         case MPI3_EVENT_SAS_DISCOVERY:
 210         {
 211                 struct mpi3_event_data_sas_discovery *event_data =
 212                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 213                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 214                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 215                     "start" : "stop",
 216                     le32_to_cpu(event_data->discovery_status));
 217                 return;
 218         }
 219         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 220                 desc = "SAS Broadcast Primitive";
 221                 break;
 222         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 223                 desc = "SAS Notify Primitive";
 224                 break;
 225         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 226                 desc = "SAS Init Device Status Change";
 227                 break;
 228         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 229                 desc = "SAS Init Table Overflow";
 230                 break;
 231         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 232                 desc = "SAS Topology Change List";
 233                 break;
 234         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 235                 desc = "Enclosure Device Status Change";
 236                 break;
 237         case MPI3_EVENT_HARD_RESET_RECEIVED:
 238                 desc = "Hard Reset Received";
 239                 break;
 240         case MPI3_EVENT_SAS_PHY_COUNTER:
 241                 desc = "SAS PHY Counter";
 242                 break;
 243         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 244                 desc = "SAS Device Discovery Error";
 245                 break;
 246         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 247                 desc = "PCIE Topology Change List";
 248                 break;
 249         case MPI3_EVENT_PCIE_ENUMERATION:
 250         {
 251                 struct mpi3_event_data_pcie_enumeration *event_data =
 252                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 253                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 254                     (event_data->reason_code ==
 255                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 256                 if (event_data->enumeration_status)
 257                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 258                             le32_to_cpu(event_data->enumeration_status));
 259                 return;
 260         }
 261         case MPI3_EVENT_PREPARE_FOR_RESET:
 262                 desc = "Prepare For Reset";
 263                 break;
 264         }
 265
 266         if (!desc)
 267                 return;
 268
 269         ioc_info(mrioc, "%s\n", desc);
 270 }
 271
 272 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 273         struct mpi3_default_reply *def_reply)
 274 {
 275         struct mpi3_event_notification_reply *event_reply =
 276             (struct mpi3_event_notification_reply *)def_reply;
 277
 278         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 279         mpi3mr_print_event_data(mrioc, event_reply);
 280         mpi3mr_os_handle_events(mrioc, event_reply);
 281 }
 282
 283 static struct mpi3mr_drv_cmd *
 284 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 285         struct mpi3_default_reply *def_reply)
 286 {
 287         u16 idx;
 288
 289         switch (host_tag) {
 290         case MPI3MR_HOSTTAG_INITCMDS:
 291                 return &mrioc->init_cmds;
 292         case MPI3MR_HOSTTAG_BLK_TMS:
 293                 return &mrioc->host_tm_cmds;
 294         case MPI3MR_HOSTTAG_INVALID:
 295                 if (def_reply && def_reply->function ==
 296                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 297                         mpi3mr_handle_events(mrioc, def_reply);
 298                 return NULL;
 299         default:
 300                 break;
 301         }
 302         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 303             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 304                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 305                 return &mrioc->dev_rmhs_cmds[idx];
 306         }
 307
 308         return NULL;
 309 }
 310
 311 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 312         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 313 {
 314         u16 reply_desc_type, host_tag = 0;
 315         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 316         u32 ioc_loginfo = 0;
 317         struct mpi3_status_reply_descriptor *status_desc;
 318         struct mpi3_address_reply_descriptor *addr_desc;
 319         struct mpi3_success_reply_descriptor *success_desc;
 320         struct mpi3_default_reply *def_reply = NULL;
 321         struct mpi3mr_drv_cmd *cmdptr = NULL;
 322         struct mpi3_scsi_io_reply *scsi_reply;
 323         u8 *sense_buf = NULL;
 324
 325         *reply_dma = 0;
 326         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 327             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 328         switch (reply_desc_type) {
 329         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 330                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 331                 host_tag = le16_to_cpu(status_desc->host_tag);
 332                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 333                 if (ioc_status &
 334                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 335                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 336                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 337                 break;
 338         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 339                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 340                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 341                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 342                 if (!def_reply)
 343                         goto out;
 344                 host_tag = le16_to_cpu(def_reply->host_tag);
 345                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 346                 if (ioc_status &
 347                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 348                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 349                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 350                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 351                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 352                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 353                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 354                 }
 355                 break;
 356         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 357                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 358                 host_tag = le16_to_cpu(success_desc->host_tag);
 359                 break;
 360         default:
 361                 break;
 362         }
 363
 364         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 365         if (cmdptr) {
 366                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 367                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 368                         cmdptr->ioc_loginfo = ioc_loginfo;
 369                         cmdptr->ioc_status = ioc_status;
 370                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 371                         if (def_reply) {
 372                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 373                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 374                                     mrioc->facts.reply_sz);
 375                         }
 376                         if (cmdptr->is_waiting) {
 377                                 complete(&cmdptr->done);
 378                                 cmdptr->is_waiting = 0;
 379                         } else if (cmdptr->callback)
 380                                 cmdptr->callback(mrioc, cmdptr);
 381                 }
 382         }
 383 out:
 384         if (sense_buf)
 385                 mpi3mr_repost_sense_buf(mrioc,
 386                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 387 }
 388
 389 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 390 {
 391         u32 exp_phase = mrioc->admin_reply_ephase;
 392         u32 admin_reply_ci = mrioc->admin_reply_ci;
 393         u32 num_admin_replies = 0;
 394         u64 reply_dma = 0;
 395         struct mpi3_default_reply_descriptor *reply_desc;
 396
 397         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 398             admin_reply_ci;
 399
 400         if ((le16_to_cpu(reply_desc->reply_flags) &
 401             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 402                 return 0;
 403
 404         do {
 405                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 406                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 407                 if (reply_dma)
 408                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 409                 num_admin_replies++;
 410                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 411                         admin_reply_ci = 0;
 412                         exp_phase ^= 1;
 413                 }
 414                 reply_desc =
 415                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 416                     admin_reply_ci;
 417                 if ((le16_to_cpu(reply_desc->reply_flags) &
 418                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 419                         break;
 420         } while (1);
 421
 422         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 423         mrioc->admin_reply_ci = admin_reply_ci;
 424         mrioc->admin_reply_ephase = exp_phase;
 425
 426         return num_admin_replies;
 427 }
 428
 429 /**
 430  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 431  *      queue's consumer index from operational reply descriptor queue.
 432  * @op_reply_q: op_reply_qinfo object
 433  * @reply_ci: operational reply descriptor's queue consumer index
 434  *
 435  * Returns reply descriptor frame address
 436  */
 437 static inline struct mpi3_default_reply_descriptor *
 438 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 439 {
 440         void *segment_base_addr;
 441         struct segments *segments = op_reply_q->q_segments;
 442         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 443
 444         segment_base_addr =
 445             segments[reply_ci / op_reply_q->segment_qd].segment;
 446         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 447             (reply_ci % op_reply_q->segment_qd);
 448         return reply_desc;
 449 }
 450
 451 static int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 452         struct mpi3mr_intr_info *intr_info)
 453 {
 454         struct op_reply_qinfo *op_reply_q = intr_info->op_reply_q;
 455         struct op_req_qinfo *op_req_q;
 456         u32 exp_phase;
 457         u32 reply_ci;
 458         u32 num_op_reply = 0;
 459         u64 reply_dma = 0;
 460         struct mpi3_default_reply_descriptor *reply_desc;
 461         u16 req_q_idx = 0, reply_qidx;
 462
 463         reply_qidx = op_reply_q->qid - 1;
 464
 465         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 466                 return 0;
 467
 468         exp_phase = op_reply_q->ephase;
 469         reply_ci = op_reply_q->ci;
 470
 471         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 472         if ((le16_to_cpu(reply_desc->reply_flags) &
 473             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 474                 atomic_dec(&op_reply_q->in_use);
 475                 return 0;
 476         }
 477
 478         do {
 479                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 480                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 481
 482                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 483                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 484                     reply_qidx);
 485                 atomic_dec(&op_reply_q->pend_ios);
 486                 if (reply_dma)
 487                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 488                 num_op_reply++;
 489
 490                 if (++reply_ci == op_reply_q->num_replies) {
 491                         reply_ci = 0;
 492                         exp_phase ^= 1;
 493                 }
 494
 495                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 496
 497                 if ((le16_to_cpu(reply_desc->reply_flags) &
 498                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 499                         break;
 500                 /*
 501                  * Exit completion loop to avoid CPU lockup
 502                  * Ensure remaining completion happens from threaded ISR.
 503                  */
 504                 if (num_op_reply > mrioc->max_host_ios) {
 505                         intr_info->op_reply_q->enable_irq_poll = true;
 506                         break;
 507                 }
 508
 509         } while (1);
 510
 511         writel(reply_ci,
 512             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 513         op_reply_q->ci = reply_ci;
 514         op_reply_q->ephase = exp_phase;
 515
 516         atomic_dec(&op_reply_q->in_use);
 517         return num_op_reply;
 518 }
 519
 520 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 521 {
 522         struct mpi3mr_intr_info *intr_info = privdata;
 523         struct mpi3mr_ioc *mrioc;
 524         u16 midx;
 525         u32 num_admin_replies = 0, num_op_reply = 0;
 526
 527         if (!intr_info)
 528                 return IRQ_NONE;
 529
 530         mrioc = intr_info->mrioc;
 531
 532         if (!mrioc->intr_enabled)
 533                 return IRQ_NONE;
 534
 535         midx = intr_info->msix_index;
 536
 537         if (!midx)
 538                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 539         if (intr_info->op_reply_q)
 540                 num_op_reply = mpi3mr_process_op_reply_q(mrioc, intr_info);
 541
 542         if (num_admin_replies || num_op_reply)
 543                 return IRQ_HANDLED;
 544         else
 545                 return IRQ_NONE;
 546 }
 547
 548 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 549 {
 550         struct mpi3mr_intr_info *intr_info = privdata;
 551         struct mpi3mr_ioc *mrioc;
 552         u16 midx;
 553         int ret;
 554
 555         if (!intr_info)
 556                 return IRQ_NONE;
 557
 558         mrioc = intr_info->mrioc;
 559         midx = intr_info->msix_index;
 560         /* Call primary ISR routine */
 561         ret = mpi3mr_isr_primary(irq, privdata);
 562
 563         /*
 564          * If more IOs are expected, schedule IRQ polling thread.
 565          * Otherwise exit from ISR.
 566          */
 567         if (!intr_info->op_reply_q)
 568                 return ret;
 569
 570         if (!intr_info->op_reply_q->enable_irq_poll ||
 571             !atomic_read(&intr_info->op_reply_q->pend_ios))
 572                 return ret;
 573
 574         disable_irq_nosync(pci_irq_vector(mrioc->pdev, midx));
 575
 576         return IRQ_WAKE_THREAD;
 577 }
 578
 579 /**
 580  * mpi3mr_isr_poll - Reply queue polling routine
 581  * @irq: IRQ
 582  * @privdata: Interrupt info
 583  *
 584  * poll for pending I/O completions in a loop until pending I/Os
 585  * present or controller queue depth I/Os are processed.
 586  *
 587  * Return: IRQ_NONE or IRQ_HANDLED
 588  */
 589 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 590 {
 591         struct mpi3mr_intr_info *intr_info = privdata;
 592         struct mpi3mr_ioc *mrioc;
 593         u16 midx;
 594         u32 num_op_reply = 0;
 595
 596         if (!intr_info || !intr_info->op_reply_q)
 597                 return IRQ_NONE;
 598
 599         mrioc = intr_info->mrioc;
 600         midx = intr_info->msix_index;
 601
 602         /* Poll for pending IOs completions */
 603         do {
 604                 if (!mrioc->intr_enabled)
 605                         break;
 606
 607                 if (!midx)
 608                         mpi3mr_process_admin_reply_q(mrioc);
 609                 if (intr_info->op_reply_q)
 610                         num_op_reply +=
 611                             mpi3mr_process_op_reply_q(mrioc, intr_info);
 612
 613                 usleep_range(mrioc->irqpoll_sleep, 10 * mrioc->irqpoll_sleep);
 614
 615         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 616             (num_op_reply < mrioc->max_host_ios));
 617
 618         intr_info->op_reply_q->enable_irq_poll = false;
 619         enable_irq(pci_irq_vector(mrioc->pdev, midx));
 620
 621         return IRQ_HANDLED;
 622 }
 623
 624 /**
 625  * mpi3mr_request_irq - Request IRQ and register ISR
 626  * @mrioc: Adapter instance reference
 627  * @index: IRQ vector index
 628  *
 629  * Request threaded ISR with primary ISR and secondary
 630  *
 631  * Return: 0 on success and non zero on failures.
 632  */
 633 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 634 {
 635         struct pci_dev *pdev = mrioc->pdev;
 636         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 637         int retval = 0;
 638
 639         intr_info->mrioc = mrioc;
 640         intr_info->msix_index = index;
 641         intr_info->op_reply_q = NULL;
 642
 643         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 644             mrioc->driver_name, mrioc->id, index);
 645
 646         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 647             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 648         if (retval) {
 649                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 650                     intr_info->name, pci_irq_vector(pdev, index));
 651                 return retval;
 652         }
 653
 654         return retval;
 655 }
 656
 657 /**
 658  * mpi3mr_setup_isr - Setup ISR for the controller
 659  * @mrioc: Adapter instance reference
 660  * @setup_one: Request one IRQ or more
 661  *
 662  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 663  *
 664  * Return: 0 on success and non zero on failures.
 665  */
 666 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 667 {
 668         unsigned int irq_flags = PCI_IRQ_MSIX;
 669         int max_vectors;
 670         int retval;
 671         int i;
 672         struct irq_affinity desc = { .pre_vectors =  1};
 673
 674         mpi3mr_cleanup_isr(mrioc);
 675
 676         if (setup_one || reset_devices)
 677                 max_vectors = 1;
 678         else {
 679                 max_vectors =
 680                     min_t(int, mrioc->cpu_count + 1, mrioc->msix_count);
 681
 682                 ioc_info(mrioc,
 683                     "MSI-X vectors supported: %d, no of cores: %d,",
 684                     mrioc->msix_count, mrioc->cpu_count);
 685                 ioc_info(mrioc,
 686                     "MSI-x vectors requested: %d\n", max_vectors);
 687         }
 688
 689         irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 690
 691         mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 692         retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 693                                 1, max_vectors, irq_flags, &desc);
 694         if (retval < 0) {
 695                 ioc_err(mrioc, "Cannot alloc irq vectors\n");
 696                 goto out_failed;
 697         }
 698         if (retval != max_vectors) {
 699                 ioc_info(mrioc,
 700                     "allocated vectors (%d) are less than configured (%d)\n",
 701                     retval, max_vectors);
 702                 /*
 703                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 704                  * between Admin queue and operational queue
 705                  */
 706                 if (retval == 1)
 707                         mrioc->op_reply_q_offset = 0;
 708
 709                 max_vectors = retval;
 710         }
 711         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 712             GFP_KERNEL);
 713         if (!mrioc->intr_info) {
 714                 retval = -ENOMEM;
 715                 pci_free_irq_vectors(mrioc->pdev);
 716                 goto out_failed;
 717         }
 718         for (i = 0; i < max_vectors; i++) {
 719                 retval = mpi3mr_request_irq(mrioc, i);
 720                 if (retval) {
 721                         mrioc->intr_info_count = i;
 722                         goto out_failed;
 723                 }
 724         }
 725         mrioc->intr_info_count = max_vectors;
 726         mpi3mr_ioc_enable_intr(mrioc);
 727         return 0;
 728
 729 out_failed:
 730         mpi3mr_cleanup_isr(mrioc);
 731
 732         return retval;
 733 }
 734
 735 static const struct {
 736         enum mpi3mr_iocstate value;
 737         char *name;
 738 } mrioc_states[] = {
 739         { MRIOC_STATE_READY, "ready" },
 740         { MRIOC_STATE_FAULT, "fault" },
 741         { MRIOC_STATE_RESET, "reset" },
 742         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 743         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 744         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 745 };
 746
 747 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 748 {
 749         int i;
 750         char *name = NULL;
 751
 752         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 753                 if (mrioc_states[i].value == mrioc_state) {
 754                         name = mrioc_states[i].name;
 755                         break;
 756                 }
 757         }
 758         return name;
 759 }
 760
 761 /* Reset reason to name mapper structure*/
 762 static const struct {
 763         enum mpi3mr_reset_reason value;
 764         char *name;
 765 } mpi3mr_reset_reason_codes[] = {
 766         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 767         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 768         { MPI3MR_RESET_FROM_IOCTL, "application invocation" },
 769         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 770         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 771         { MPI3MR_RESET_FROM_IOCTL_TIMEOUT, "IOCTL timeout" },
 772         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 773         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 774         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 775         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 776         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 777         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 778         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 779         {
 780                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 781                 "create request queue timeout"
 782         },
 783         {
 784                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 785                 "create reply queue timeout"
 786         },
 787         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 788         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 789         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 790         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 791         {
 792                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 793                 "component image activation timeout"
 794         },
 795         {
 796                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 797                 "get package version timeout"
 798         },
 799         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 800         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 801 };
 802
 803 /**
 804  * mpi3mr_reset_rc_name - get reset reason code name
 805  * @reason_code: reset reason code value
 806  *
 807  * Map reset reason to an NULL terminated ASCII string
 808  *
 809  * Return: name corresponding to reset reason value or NULL.
 810  */
 811 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 812 {
 813         int i;
 814         char *name = NULL;
 815
 816         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 817                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 818                         name = mpi3mr_reset_reason_codes[i].name;
 819                         break;
 820                 }
 821         }
 822         return name;
 823 }
 824
 825 /* Reset type to name mapper structure*/
 826 static const struct {
 827         u16 reset_type;
 828         char *name;
 829 } mpi3mr_reset_types[] = {
 830         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 831         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 832 };
 833
 834 /**
 835  * mpi3mr_reset_type_name - get reset type name
 836  * @reset_type: reset type value
 837  *
 838  * Map reset type to an NULL terminated ASCII string
 839  *
 840  * Return: name corresponding to reset type value or NULL.
 841  */
 842 static const char *mpi3mr_reset_type_name(u16 reset_type)
 843 {
 844         int i;
 845         char *name = NULL;
 846
 847         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 848                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 849                         name = mpi3mr_reset_types[i].name;
 850                         break;
 851                 }
 852         }
 853         return name;
 854 }
 855
 856 /**
 857  * mpi3mr_print_fault_info - Display fault information
 858  * @mrioc: Adapter instance reference
 859  *
 860  * Display the controller fault information if there is a
 861  * controller fault.
 862  *
 863  * Return: Nothing.
 864  */
 865 static void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 866 {
 867         u32 ioc_status, code, code1, code2, code3;
 868
 869         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 870
 871         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
 872                 code = readl(&mrioc->sysif_regs->fault);
 873                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
 874                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
 875                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
 876
 877                 ioc_info(mrioc,
 878                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
 879                     code, code1, code2, code3);
 880         }
 881 }
 882
 883 /**
 884  * mpi3mr_get_iocstate - Get IOC State
 885  * @mrioc: Adapter instance reference
 886  *
 887  * Return a proper IOC state enum based on the IOC status and
 888  * IOC configuration and unrcoverable state of the controller.
 889  *
 890  * Return: Current IOC state.
 891  */
 892 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
 893 {
 894         u32 ioc_status, ioc_config;
 895         u8 ready, enabled;
 896
 897         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 898         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 899
 900         if (mrioc->unrecoverable)
 901                 return MRIOC_STATE_UNRECOVERABLE;
 902         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
 903                 return MRIOC_STATE_FAULT;
 904
 905         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
 906         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
 907
 908         if (ready && enabled)
 909                 return MRIOC_STATE_READY;
 910         if ((!ready) && (!enabled))
 911                 return MRIOC_STATE_RESET;
 912         if ((!ready) && (enabled))
 913                 return MRIOC_STATE_BECOMING_READY;
 914
 915         return MRIOC_STATE_RESET_REQUESTED;
 916 }
 917
 918 /**
 919  * mpi3mr_clear_reset_history - clear reset history
 920  * @mrioc: Adapter instance reference
 921  *
 922  * Write the reset history bit in IOC status to clear the bit,
 923  * if it is already set.
 924  *
 925  * Return: Nothing.
 926  */
 927 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
 928 {
 929         u32 ioc_status;
 930
 931         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 932         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
 933                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
 934 }
 935
 936 /**
 937  * mpi3mr_issue_and_process_mur - Message unit Reset handler
 938  * @mrioc: Adapter instance reference
 939  * @reset_reason: Reset reason code
 940  *
 941  * Issue Message unit Reset to the controller and wait for it to
 942  * be complete.
 943  *
 944  * Return: 0 on success, -1 on failure.
 945  */
 946 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
 947         u32 reset_reason)
 948 {
 949         u32 ioc_config, timeout, ioc_status;
 950         int retval = -1;
 951
 952         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
 953         if (mrioc->unrecoverable) {
 954                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
 955                 return retval;
 956         }
 957         mpi3mr_clear_reset_history(mrioc);
 958         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
 959         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 960         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
 961         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
 962
 963         timeout = mrioc->ready_timeout * 10;
 964         do {
 965                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 966                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
 967                         mpi3mr_clear_reset_history(mrioc);
 968                         ioc_config =
 969                             readl(&mrioc->sysif_regs->ioc_configuration);
 970                         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
 971                               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
 972                             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC))) {
 973                                 retval = 0;
 974                                 break;
 975                         }
 976                 }
 977                 msleep(100);
 978         } while (--timeout);
 979
 980         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 981         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 982
 983         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
 984             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
 985         return retval;
 986 }
 987
 988 /**
 989  * mpi3mr_bring_ioc_ready - Bring controller to ready state
 990  * @mrioc: Adapter instance reference
 991  *
 992  * Set Enable IOC bit in IOC configuration register and wait for
 993  * the controller to become ready.
 994  *
 995  * Return: 0 on success, -1 on failure.
 996  */
 997 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
 998 {
 999         u32 ioc_config, timeout;
1000         enum mpi3mr_iocstate current_state;
1001
1002         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1003         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1004         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1005
1006         timeout = mrioc->ready_timeout * 10;
1007         do {
1008                 current_state = mpi3mr_get_iocstate(mrioc);
1009                 if (current_state == MRIOC_STATE_READY)
1010                         return 0;
1011                 msleep(100);
1012         } while (--timeout);
1013
1014         return -1;
1015 }
1016
1017 /**
1018  * mpi3mr_soft_reset_success - Check softreset is success or not
1019  * @ioc_status: IOC status register value
1020  * @ioc_config: IOC config register value
1021  *
1022  * Check whether the soft reset is successful or not based on
1023  * IOC status and IOC config register values.
1024  *
1025  * Return: True when the soft reset is success, false otherwise.
1026  */
1027 static inline bool
1028 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1029 {
1030         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1031             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1032             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1033                 return true;
1034         return false;
1035 }
1036
1037 /**
1038  * mpi3mr_diagfault_success - Check diag fault is success or not
1039  * @mrioc: Adapter reference
1040  * @ioc_status: IOC status register value
1041  *
1042  * Check whether the controller hit diag reset fault code.
1043  *
1044  * Return: True when there is diag fault, false otherwise.
1045  */
1046 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1047         u32 ioc_status)
1048 {
1049         u32 fault;
1050
1051         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1052                 return false;
1053         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1054         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET)
1055                 return true;
1056         return false;
1057 }
1058
1059 /**
1060  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1061  * @mrioc: Adapter reference
1062  *
1063  * Set diag save bit in IOC configuration register to enable
1064  * snapdump.
1065  *
1066  * Return: Nothing.
1067  */
1068 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1069 {
1070         u32 ioc_config;
1071
1072         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1073         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1074         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1075 }
1076
1077 /**
1078  * mpi3mr_issue_reset - Issue reset to the controller
1079  * @mrioc: Adapter reference
1080  * @reset_type: Reset type
1081  * @reset_reason: Reset reason code
1082  *
1083  * Unlock the host diagnostic registers and write the specific
1084  * reset type to that, wait for reset acknowledgment from the
1085  * controller, if the reset is not successful retry for the
1086  * predefined number of times.
1087  *
1088  * Return: 0 on success, non-zero on failure.
1089  */
1090 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1091         u32 reset_reason)
1092 {
1093         int retval = -1;
1094         u8 unlock_retry_count, reset_retry_count = 0;
1095         u32 host_diagnostic, timeout, ioc_status, ioc_config;
1096
1097         pci_cfg_access_lock(mrioc->pdev);
1098         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1099             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1100                 goto out;
1101         if (mrioc->unrecoverable)
1102                 goto out;
1103 retry_reset:
1104         unlock_retry_count = 0;
1105         mpi3mr_clear_reset_history(mrioc);
1106         do {
1107                 ioc_info(mrioc,
1108                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1109                     ++unlock_retry_count);
1110                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1111                         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1112                         mrioc->unrecoverable = 1;
1113                         goto out;
1114                 }
1115
1116                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1117                     &mrioc->sysif_regs->write_sequence);
1118                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1119                     &mrioc->sysif_regs->write_sequence);
1120                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1121                     &mrioc->sysif_regs->write_sequence);
1122                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1123                     &mrioc->sysif_regs->write_sequence);
1124                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1125                     &mrioc->sysif_regs->write_sequence);
1126                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1127                     &mrioc->sysif_regs->write_sequence);
1128                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1129                     &mrioc->sysif_regs->write_sequence);
1130                 usleep_range(1000, 1100);
1131                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1132                 ioc_info(mrioc,
1133                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1134                     unlock_retry_count, host_diagnostic);
1135         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1136
1137         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1138         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1139             mpi3mr_reset_type_name(reset_type),
1140             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1141         writel(host_diagnostic | reset_type,
1142             &mrioc->sysif_regs->host_diagnostic);
1143         timeout = mrioc->ready_timeout * 10;
1144         if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) {
1145                 do {
1146                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1147                         if (ioc_status &
1148                             MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
1149                                 mpi3mr_clear_reset_history(mrioc);
1150                                 ioc_config =
1151                                     readl(&mrioc->sysif_regs->ioc_configuration);
1152                                 if (mpi3mr_soft_reset_success(ioc_status,
1153                                     ioc_config)) {
1154                                         retval = 0;
1155                                         break;
1156                                 }
1157                         }
1158                         msleep(100);
1159                 } while (--timeout);
1160                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1161                     &mrioc->sysif_regs->write_sequence);
1162         } else if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT) {
1163                 do {
1164                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1165                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1166                                 retval = 0;
1167                                 break;
1168                         }
1169                         msleep(100);
1170                 } while (--timeout);
1171                 mpi3mr_clear_reset_history(mrioc);
1172                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1173                     &mrioc->sysif_regs->write_sequence);
1174         }
1175         if (retval && ((++reset_retry_count) < MPI3MR_MAX_RESET_RETRY_COUNT)) {
1176                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1177                 ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1178                 ioc_info(mrioc,
1179                     "Base IOC Sts/Config after reset try %d is (0x%x)/(0x%x)\n",
1180                     reset_retry_count, ioc_status, ioc_config);
1181                 goto retry_reset;
1182         }
1183
1184 out:
1185         pci_cfg_access_unlock(mrioc->pdev);
1186         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1187         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1188
1189         ioc_info(mrioc,
1190             "Base IOC Sts/Config after %s reset is (0x%x)/(0x%x)\n",
1191             (!retval) ? "successful" : "failed", ioc_status,
1192             ioc_config);
1193         return retval;
1194 }
1195
1196 /**
1197  * mpi3mr_admin_request_post - Post request to admin queue
1198  * @mrioc: Adapter reference
1199  * @admin_req: MPI3 request
1200  * @admin_req_sz: Request size
1201  * @ignore_reset: Ignore reset in process
1202  *
1203  * Post the MPI3 request into admin request queue and
1204  * inform the controller, if the queue is full return
1205  * appropriate error.
1206  *
1207  * Return: 0 on success, non-zero on failure.
1208  */
1209 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1210         u16 admin_req_sz, u8 ignore_reset)
1211 {
1212         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1213         int retval = 0;
1214         unsigned long flags;
1215         u8 *areq_entry;
1216
1217         if (mrioc->unrecoverable) {
1218                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1219                 return -EFAULT;
1220         }
1221
1222         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1223         areq_pi = mrioc->admin_req_pi;
1224         areq_ci = mrioc->admin_req_ci;
1225         max_entries = mrioc->num_admin_req;
1226         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1227             (areq_pi == (max_entries - 1)))) {
1228                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1229                 retval = -EAGAIN;
1230                 goto out;
1231         }
1232         if (!ignore_reset && mrioc->reset_in_progress) {
1233                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1234                 retval = -EAGAIN;
1235                 goto out;
1236         }
1237         areq_entry = (u8 *)mrioc->admin_req_base +
1238             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1239         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1240         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1241
1242         if (++areq_pi == max_entries)
1243                 areq_pi = 0;
1244         mrioc->admin_req_pi = areq_pi;
1245
1246         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1247
1248 out:
1249         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1250
1251         return retval;
1252 }
1253
1254 /**
1255  * mpi3mr_free_op_req_q_segments - free request memory segments
1256  * @mrioc: Adapter instance reference
1257  * @q_idx: operational request queue index
1258  *
1259  * Free memory segments allocated for operational request queue
1260  *
1261  * Return: Nothing.
1262  */
1263 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1264 {
1265         u16 j;
1266         int size;
1267         struct segments *segments;
1268
1269         segments = mrioc->req_qinfo[q_idx].q_segments;
1270         if (!segments)
1271                 return;
1272
1273         if (mrioc->enable_segqueue) {
1274                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1275                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1276                         dma_free_coherent(&mrioc->pdev->dev,
1277                             MPI3MR_MAX_SEG_LIST_SIZE,
1278                             mrioc->req_qinfo[q_idx].q_segment_list,
1279                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1280                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1281                 }
1282         } else
1283                 size = mrioc->req_qinfo[q_idx].num_requests *
1284                     mrioc->facts.op_req_sz;
1285
1286         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1287                 if (!segments[j].segment)
1288                         continue;
1289                 dma_free_coherent(&mrioc->pdev->dev,
1290                     size, segments[j].segment, segments[j].segment_dma);
1291                 segments[j].segment = NULL;
1292         }
1293         kfree(mrioc->req_qinfo[q_idx].q_segments);
1294         mrioc->req_qinfo[q_idx].q_segments = NULL;
1295         mrioc->req_qinfo[q_idx].qid = 0;
1296 }
1297
1298 /**
1299  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1300  * @mrioc: Adapter instance reference
1301  * @q_idx: operational reply queue index
1302  *
1303  * Free memory segments allocated for operational reply queue
1304  *
1305  * Return: Nothing.
1306  */
1307 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1308 {
1309         u16 j;
1310         int size;
1311         struct segments *segments;
1312
1313         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1314         if (!segments)
1315                 return;
1316
1317         if (mrioc->enable_segqueue) {
1318                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1319                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1320                         dma_free_coherent(&mrioc->pdev->dev,
1321                             MPI3MR_MAX_SEG_LIST_SIZE,
1322                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1323                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1324                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1325                 }
1326         } else
1327                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1328                     mrioc->op_reply_desc_sz;
1329
1330         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1331                 if (!segments[j].segment)
1332                         continue;
1333                 dma_free_coherent(&mrioc->pdev->dev,
1334                     size, segments[j].segment, segments[j].segment_dma);
1335                 segments[j].segment = NULL;
1336         }
1337
1338         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1339         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1340         mrioc->op_reply_qinfo[q_idx].qid = 0;
1341 }
1342
1343 /**
1344  * mpi3mr_delete_op_reply_q - delete operational reply queue
1345  * @mrioc: Adapter instance reference
1346  * @qidx: operational reply queue index
1347  *
1348  * Delete operatinal reply queue by issuing MPI request
1349  * through admin queue.
1350  *
1351  * Return:  0 on success, non-zero on failure.
1352  */
1353 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1354 {
1355         struct mpi3_delete_reply_queue_request delq_req;
1356         int retval = 0;
1357         u16 reply_qid = 0, midx;
1358
1359         reply_qid = mrioc->op_reply_qinfo[qidx].qid;
1360
1361         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1362
1363         if (!reply_qid) {
1364                 retval = -1;
1365                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1366                 goto out;
1367         }
1368
1369         memset(&delq_req, 0, sizeof(delq_req));
1370         mutex_lock(&mrioc->init_cmds.mutex);
1371         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1372                 retval = -1;
1373                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1374                 mutex_unlock(&mrioc->init_cmds.mutex);
1375                 goto out;
1376         }
1377         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1378         mrioc->init_cmds.is_waiting = 1;
1379         mrioc->init_cmds.callback = NULL;
1380         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1381         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1382         delq_req.queue_id = cpu_to_le16(reply_qid);
1383
1384         init_completion(&mrioc->init_cmds.done);
1385         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1386             1);
1387         if (retval) {
1388                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1389                 goto out_unlock;
1390         }
1391         wait_for_completion_timeout(&mrioc->init_cmds.done,
1392             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1393         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1394                 ioc_err(mrioc, "Issue DelRepQ: command timed out\n");
1395                 mpi3mr_set_diagsave(mrioc);
1396                 mpi3mr_issue_reset(mrioc,
1397                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1398                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1399                 mrioc->unrecoverable = 1;
1400
1401                 retval = -1;
1402                 goto out_unlock;
1403         }
1404         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1405             != MPI3_IOCSTATUS_SUCCESS) {
1406                 ioc_err(mrioc,
1407                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1408                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1409                     mrioc->init_cmds.ioc_loginfo);
1410                 retval = -1;
1411                 goto out_unlock;
1412         }
1413         mrioc->intr_info[midx].op_reply_q = NULL;
1414
1415         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1416 out_unlock:
1417         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1418         mutex_unlock(&mrioc->init_cmds.mutex);
1419 out:
1420
1421         return retval;
1422 }
1423
1424 /**
1425  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1426  * @mrioc: Adapter instance reference
1427  * @qidx: request queue index
1428  *
1429  * Allocate segmented memory pools for operational reply
1430  * queue.
1431  *
1432  * Return: 0 on success, non-zero on failure.
1433  */
1434 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1435 {
1436         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1437         int i, size;
1438         u64 *q_segment_list_entry = NULL;
1439         struct segments *segments;
1440
1441         if (mrioc->enable_segqueue) {
1442                 op_reply_q->segment_qd =
1443                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1444
1445                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1446
1447                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1448                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1449                     GFP_KERNEL);
1450                 if (!op_reply_q->q_segment_list)
1451                         return -ENOMEM;
1452                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1453         } else {
1454                 op_reply_q->segment_qd = op_reply_q->num_replies;
1455                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1456         }
1457
1458         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1459             op_reply_q->segment_qd);
1460
1461         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1462             sizeof(struct segments), GFP_KERNEL);
1463         if (!op_reply_q->q_segments)
1464                 return -ENOMEM;
1465
1466         segments = op_reply_q->q_segments;
1467         for (i = 0; i < op_reply_q->num_segments; i++) {
1468                 segments[i].segment =
1469                     dma_alloc_coherent(&mrioc->pdev->dev,
1470                     size, &segments[i].segment_dma, GFP_KERNEL);
1471                 if (!segments[i].segment)
1472                         return -ENOMEM;
1473                 if (mrioc->enable_segqueue)
1474                         q_segment_list_entry[i] =
1475                             (unsigned long)segments[i].segment_dma;
1476         }
1477
1478         return 0;
1479 }
1480
1481 /**
1482  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1483  * @mrioc: Adapter instance reference
1484  * @qidx: request queue index
1485  *
1486  * Allocate segmented memory pools for operational request
1487  * queue.
1488  *
1489  * Return: 0 on success, non-zero on failure.
1490  */
1491 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1492 {
1493         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1494         int i, size;
1495         u64 *q_segment_list_entry = NULL;
1496         struct segments *segments;
1497
1498         if (mrioc->enable_segqueue) {
1499                 op_req_q->segment_qd =
1500                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1501
1502                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1503
1504                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1505                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1506                     GFP_KERNEL);
1507                 if (!op_req_q->q_segment_list)
1508                         return -ENOMEM;
1509                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1510
1511         } else {
1512                 op_req_q->segment_qd = op_req_q->num_requests;
1513                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1514         }
1515
1516         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1517             op_req_q->segment_qd);
1518
1519         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1520             sizeof(struct segments), GFP_KERNEL);
1521         if (!op_req_q->q_segments)
1522                 return -ENOMEM;
1523
1524         segments = op_req_q->q_segments;
1525         for (i = 0; i < op_req_q->num_segments; i++) {
1526                 segments[i].segment =
1527                     dma_alloc_coherent(&mrioc->pdev->dev,
1528                     size, &segments[i].segment_dma, GFP_KERNEL);
1529                 if (!segments[i].segment)
1530                         return -ENOMEM;
1531                 if (mrioc->enable_segqueue)
1532                         q_segment_list_entry[i] =
1533                             (unsigned long)segments[i].segment_dma;
1534         }
1535
1536         return 0;
1537 }
1538
1539 /**
1540  * mpi3mr_create_op_reply_q - create operational reply queue
1541  * @mrioc: Adapter instance reference
1542  * @qidx: operational reply queue index
1543  *
1544  * Create operatinal reply queue by issuing MPI request
1545  * through admin queue.
1546  *
1547  * Return:  0 on success, non-zero on failure.
1548  */
1549 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1550 {
1551         struct mpi3_create_reply_queue_request create_req;
1552         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1553         int retval = 0;
1554         u16 reply_qid = 0, midx;
1555
1556         reply_qid = op_reply_q->qid;
1557
1558         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1559
1560         if (reply_qid) {
1561                 retval = -1;
1562                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1563                     reply_qid);
1564
1565                 return retval;
1566         }
1567
1568         reply_qid = qidx + 1;
1569         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1570         op_reply_q->ci = 0;
1571         op_reply_q->ephase = 1;
1572         atomic_set(&op_reply_q->pend_ios, 0);
1573         atomic_set(&op_reply_q->in_use, 0);
1574         op_reply_q->enable_irq_poll = false;
1575
1576         if (!op_reply_q->q_segments) {
1577                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1578                 if (retval) {
1579                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1580                         goto out;
1581                 }
1582         }
1583
1584         memset(&create_req, 0, sizeof(create_req));
1585         mutex_lock(&mrioc->init_cmds.mutex);
1586         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1587                 retval = -1;
1588                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1589                 goto out_unlock;
1590         }
1591         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1592         mrioc->init_cmds.is_waiting = 1;
1593         mrioc->init_cmds.callback = NULL;
1594         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1595         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1596         create_req.queue_id = cpu_to_le16(reply_qid);
1597         create_req.flags = MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1598         create_req.msix_index = cpu_to_le16(mrioc->intr_info[midx].msix_index);
1599         if (mrioc->enable_segqueue) {
1600                 create_req.flags |=
1601                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1602                 create_req.base_address = cpu_to_le64(
1603                     op_reply_q->q_segment_list_dma);
1604         } else
1605                 create_req.base_address = cpu_to_le64(
1606                     op_reply_q->q_segments[0].segment_dma);
1607
1608         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1609
1610         init_completion(&mrioc->init_cmds.done);
1611         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1612             sizeof(create_req), 1);
1613         if (retval) {
1614                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1615                 goto out_unlock;
1616         }
1617         wait_for_completion_timeout(&mrioc->init_cmds.done,
1618             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1619         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1620                 ioc_err(mrioc, "CreateRepQ: command timed out\n");
1621                 mpi3mr_set_diagsave(mrioc);
1622                 mpi3mr_issue_reset(mrioc,
1623                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1624                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1625                 mrioc->unrecoverable = 1;
1626                 retval = -1;
1627                 goto out_unlock;
1628         }
1629         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1630             != MPI3_IOCSTATUS_SUCCESS) {
1631                 ioc_err(mrioc,
1632                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1633                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1634                     mrioc->init_cmds.ioc_loginfo);
1635                 retval = -1;
1636                 goto out_unlock;
1637         }
1638         op_reply_q->qid = reply_qid;
1639         mrioc->intr_info[midx].op_reply_q = op_reply_q;
1640
1641 out_unlock:
1642         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1643         mutex_unlock(&mrioc->init_cmds.mutex);
1644 out:
1645
1646         return retval;
1647 }
1648
1649 /**
1650  * mpi3mr_create_op_req_q - create operational request queue
1651  * @mrioc: Adapter instance reference
1652  * @idx: operational request queue index
1653  * @reply_qid: Reply queue ID
1654  *
1655  * Create operatinal request queue by issuing MPI request
1656  * through admin queue.
1657  *
1658  * Return:  0 on success, non-zero on failure.
1659  */
1660 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1661         u16 reply_qid)
1662 {
1663         struct mpi3_create_request_queue_request create_req;
1664         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1665         int retval = 0;
1666         u16 req_qid = 0;
1667
1668         req_qid = op_req_q->qid;
1669
1670         if (req_qid) {
1671                 retval = -1;
1672                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1673                     req_qid);
1674
1675                 return retval;
1676         }
1677         req_qid = idx + 1;
1678
1679         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1680         op_req_q->ci = 0;
1681         op_req_q->pi = 0;
1682         op_req_q->reply_qid = reply_qid;
1683         spin_lock_init(&op_req_q->q_lock);
1684
1685         if (!op_req_q->q_segments) {
1686                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1687                 if (retval) {
1688                         mpi3mr_free_op_req_q_segments(mrioc, idx);
1689                         goto out;
1690                 }
1691         }
1692
1693         memset(&create_req, 0, sizeof(create_req));
1694         mutex_lock(&mrioc->init_cmds.mutex);
1695         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1696                 retval = -1;
1697                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
1698                 goto out_unlock;
1699         }
1700         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1701         mrioc->init_cmds.is_waiting = 1;
1702         mrioc->init_cmds.callback = NULL;
1703         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1704         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
1705         create_req.queue_id = cpu_to_le16(req_qid);
1706         if (mrioc->enable_segqueue) {
1707                 create_req.flags =
1708                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1709                 create_req.base_address = cpu_to_le64(
1710                     op_req_q->q_segment_list_dma);
1711         } else
1712                 create_req.base_address = cpu_to_le64(
1713                     op_req_q->q_segments[0].segment_dma);
1714         create_req.reply_queue_id = cpu_to_le16(reply_qid);
1715         create_req.size = cpu_to_le16(op_req_q->num_requests);
1716
1717         init_completion(&mrioc->init_cmds.done);
1718         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1719             sizeof(create_req), 1);
1720         if (retval) {
1721                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
1722                 goto out_unlock;
1723         }
1724         wait_for_completion_timeout(&mrioc->init_cmds.done,
1725             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1726         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1727                 ioc_err(mrioc, "CreateReqQ: command timed out\n");
1728                 mpi3mr_set_diagsave(mrioc);
1729                 if (mpi3mr_issue_reset(mrioc,
1730                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1731                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT))
1732                         mrioc->unrecoverable = 1;
1733                 retval = -1;
1734                 goto out_unlock;
1735         }
1736         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1737             != MPI3_IOCSTATUS_SUCCESS) {
1738                 ioc_err(mrioc,
1739                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1740                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1741                     mrioc->init_cmds.ioc_loginfo);
1742                 retval = -1;
1743                 goto out_unlock;
1744         }
1745         op_req_q->qid = req_qid;
1746
1747 out_unlock:
1748         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1749         mutex_unlock(&mrioc->init_cmds.mutex);
1750 out:
1751
1752         return retval;
1753 }
1754
1755 /**
1756  * mpi3mr_create_op_queues - create operational queue pairs
1757  * @mrioc: Adapter instance reference
1758  *
1759  * Allocate memory for operational queue meta data and call
1760  * create request and reply queue functions.
1761  *
1762  * Return: 0 on success, non-zero on failures.
1763  */
1764 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
1765 {
1766         int retval = 0;
1767         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
1768
1769         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
1770             mrioc->facts.max_op_req_q);
1771
1772         msix_count_op_q =
1773             mrioc->intr_info_count - mrioc->op_reply_q_offset;
1774         if (!mrioc->num_queues)
1775                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
1776         num_queues = mrioc->num_queues;
1777         ioc_info(mrioc, "Trying to create %d Operational Q pairs\n",
1778             num_queues);
1779
1780         if (!mrioc->req_qinfo) {
1781                 mrioc->req_qinfo = kcalloc(num_queues,
1782                     sizeof(struct op_req_qinfo), GFP_KERNEL);
1783                 if (!mrioc->req_qinfo) {
1784                         retval = -1;
1785                         goto out_failed;
1786                 }
1787
1788                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
1789                     num_queues, GFP_KERNEL);
1790                 if (!mrioc->op_reply_qinfo) {
1791                         retval = -1;
1792                         goto out_failed;
1793                 }
1794         }
1795
1796         if (mrioc->enable_segqueue)
1797                 ioc_info(mrioc,
1798                     "allocating operational queues through segmented queues\n");
1799
1800         for (i = 0; i < num_queues; i++) {
1801                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
1802                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
1803                         break;
1804                 }
1805                 if (mpi3mr_create_op_req_q(mrioc, i,
1806                     mrioc->op_reply_qinfo[i].qid)) {
1807                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
1808                         mpi3mr_delete_op_reply_q(mrioc, i);
1809                         break;
1810                 }
1811         }
1812
1813         if (i == 0) {
1814                 /* Not even one queue is created successfully*/
1815                 retval = -1;
1816                 goto out_failed;
1817         }
1818         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
1819         ioc_info(mrioc, "Successfully created %d Operational Q pairs\n",
1820             mrioc->num_op_reply_q);
1821
1822         return retval;
1823 out_failed:
1824         kfree(mrioc->req_qinfo);
1825         mrioc->req_qinfo = NULL;
1826
1827         kfree(mrioc->op_reply_qinfo);
1828         mrioc->op_reply_qinfo = NULL;
1829
1830         return retval;
1831 }
1832
1833 /**
1834  * mpi3mr_op_request_post - Post request to operational queue
1835  * @mrioc: Adapter reference
1836  * @op_req_q: Operational request queue info
1837  * @req: MPI3 request
1838  *
1839  * Post the MPI3 request into operational request queue and
1840  * inform the controller, if the queue is full return
1841  * appropriate error.
1842  *
1843  * Return: 0 on success, non-zero on failure.
1844  */
1845 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
1846         struct op_req_qinfo *op_req_q, u8 *req)
1847 {
1848         u16 pi = 0, max_entries, reply_qidx = 0, midx;
1849         int retval = 0;
1850         unsigned long flags;
1851         u8 *req_entry;
1852         void *segment_base_addr;
1853         u16 req_sz = mrioc->facts.op_req_sz;
1854         struct segments *segments = op_req_q->q_segments;
1855
1856         reply_qidx = op_req_q->reply_qid - 1;
1857
1858         if (mrioc->unrecoverable)
1859                 return -EFAULT;
1860
1861         spin_lock_irqsave(&op_req_q->q_lock, flags);
1862         pi = op_req_q->pi;
1863         max_entries = op_req_q->num_requests;
1864
1865         if (mpi3mr_check_req_qfull(op_req_q)) {
1866                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
1867                     reply_qidx, mrioc->op_reply_q_offset);
1868                 mpi3mr_process_op_reply_q(mrioc, &mrioc->intr_info[midx]);
1869
1870                 if (mpi3mr_check_req_qfull(op_req_q)) {
1871                         retval = -EAGAIN;
1872                         goto out;
1873                 }
1874         }
1875
1876         if (mrioc->reset_in_progress) {
1877                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
1878                 retval = -EAGAIN;
1879                 goto out;
1880         }
1881
1882         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
1883         req_entry = (u8 *)segment_base_addr +
1884             ((pi % op_req_q->segment_qd) * req_sz);
1885
1886         memset(req_entry, 0, req_sz);
1887         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
1888
1889         if (++pi == max_entries)
1890                 pi = 0;
1891         op_req_q->pi = pi;
1892
1893         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
1894             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
1895                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
1896
1897         writel(op_req_q->pi,
1898             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
1899
1900 out:
1901         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
1902         return retval;
1903 }
1904
1905 /**
1906  * mpi3mr_sync_timestamp - Issue time stamp sync request
1907  * @mrioc: Adapter reference
1908  *
1909  * Issue IO unit control MPI request to synchornize firmware
1910  * timestamp with host time.
1911  *
1912  * Return: 0 on success, non-zero on failure.
1913  */
1914 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
1915 {
1916         ktime_t current_time;
1917         struct mpi3_iounit_control_request iou_ctrl;
1918         int retval = 0;
1919
1920         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
1921         mutex_lock(&mrioc->init_cmds.mutex);
1922         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1923                 retval = -1;
1924                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
1925                 mutex_unlock(&mrioc->init_cmds.mutex);
1926                 goto out;
1927         }
1928         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1929         mrioc->init_cmds.is_waiting = 1;
1930         mrioc->init_cmds.callback = NULL;
1931         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1932         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
1933         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
1934         current_time = ktime_get_real();
1935         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
1936
1937         init_completion(&mrioc->init_cmds.done);
1938         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
1939             sizeof(iou_ctrl), 0);
1940         if (retval) {
1941                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
1942                 goto out_unlock;
1943         }
1944
1945         wait_for_completion_timeout(&mrioc->init_cmds.done,
1946             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1947         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1948                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
1949                 mrioc->init_cmds.is_waiting = 0;
1950                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
1951                         mpi3mr_soft_reset_handler(mrioc,
1952                             MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
1953                 retval = -1;
1954                 goto out_unlock;
1955         }
1956         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1957             != MPI3_IOCSTATUS_SUCCESS) {
1958                 ioc_err(mrioc,
1959                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1960                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1961                     mrioc->init_cmds.ioc_loginfo);
1962                 retval = -1;
1963                 goto out_unlock;
1964         }
1965
1966 out_unlock:
1967         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1968         mutex_unlock(&mrioc->init_cmds.mutex);
1969
1970 out:
1971         return retval;
1972 }
1973
1974 /**
1975  * mpi3mr_watchdog_work - watchdog thread to monitor faults
1976  * @work: work struct
1977  *
1978  * Watch dog work periodically executed (1 second interval) to
1979  * monitor firmware fault and to issue periodic timer sync to
1980  * the firmware.
1981  *
1982  * Return: Nothing.
1983  */
1984 static void mpi3mr_watchdog_work(struct work_struct *work)
1985 {
1986         struct mpi3mr_ioc *mrioc =
1987             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
1988         unsigned long flags;
1989         enum mpi3mr_iocstate ioc_state;
1990         u32 fault, host_diagnostic;
1991
1992         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
1993                 mrioc->ts_update_counter = 0;
1994                 mpi3mr_sync_timestamp(mrioc);
1995         }
1996
1997         /*Check for fault state every one second and issue Soft reset*/
1998         ioc_state = mpi3mr_get_iocstate(mrioc);
1999         if (ioc_state == MRIOC_STATE_FAULT) {
2000                 fault = readl(&mrioc->sysif_regs->fault) &
2001                     MPI3_SYSIF_FAULT_CODE_MASK;
2002                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2003                 if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2004                         if (!mrioc->diagsave_timeout) {
2005                                 mpi3mr_print_fault_info(mrioc);
2006                                 ioc_warn(mrioc, "Diag save in progress\n");
2007                         }
2008                         if ((mrioc->diagsave_timeout++) <=
2009                             MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2010                                 goto schedule_work;
2011                 } else
2012                         mpi3mr_print_fault_info(mrioc);
2013                 mrioc->diagsave_timeout = 0;
2014
2015                 if (fault == MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED) {
2016                         ioc_info(mrioc,
2017                             "Factory Reset fault occurred marking controller as unrecoverable"
2018                             );
2019                         mrioc->unrecoverable = 1;
2020                         goto out;
2021                 }
2022
2023                 if ((fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) ||
2024                     (fault == MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS) ||
2025                     (mrioc->reset_in_progress))
2026                         goto out;
2027                 if (fault == MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET)
2028                         mpi3mr_soft_reset_handler(mrioc,
2029                             MPI3MR_RESET_FROM_CIACTIV_FAULT, 0);
2030                 else
2031                         mpi3mr_soft_reset_handler(mrioc,
2032                             MPI3MR_RESET_FROM_FAULT_WATCH, 0);
2033         }
2034
2035 schedule_work:
2036         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2037         if (mrioc->watchdog_work_q)
2038                 queue_delayed_work(mrioc->watchdog_work_q,
2039                     &mrioc->watchdog_work,
2040                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2041         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2042 out:
2043         return;
2044 }
2045
2046 /**
2047  * mpi3mr_start_watchdog - Start watchdog
2048  * @mrioc: Adapter instance reference
2049  *
2050  * Create and start the watchdog thread to monitor controller
2051  * faults.
2052  *
2053  * Return: Nothing.
2054  */
2055 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2056 {
2057         if (mrioc->watchdog_work_q)
2058                 return;
2059
2060         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2061         snprintf(mrioc->watchdog_work_q_name,
2062             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2063             mrioc->id);
2064         mrioc->watchdog_work_q =
2065             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2066         if (!mrioc->watchdog_work_q) {
2067                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2068                 return;
2069         }
2070
2071         if (mrioc->watchdog_work_q)
2072                 queue_delayed_work(mrioc->watchdog_work_q,
2073                     &mrioc->watchdog_work,
2074                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2075 }
2076
2077 /**
2078  * mpi3mr_stop_watchdog - Stop watchdog
2079  * @mrioc: Adapter instance reference
2080  *
2081  * Stop the watchdog thread created to monitor controller
2082  * faults.
2083  *
2084  * Return: Nothing.
2085  */
2086 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2087 {
2088         unsigned long flags;
2089         struct workqueue_struct *wq;
2090
2091         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2092         wq = mrioc->watchdog_work_q;
2093         mrioc->watchdog_work_q = NULL;
2094         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2095         if (wq) {
2096                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2097                         flush_workqueue(wq);
2098                 destroy_workqueue(wq);
2099         }
2100 }
2101
2102 /**
2103  * mpi3mr_kill_ioc - Kill the controller
2104  * @mrioc: Adapter instance reference
2105  * @reason: reason for the failure.
2106  *
2107  * If fault debug is enabled, display the fault info else issue
2108  * diag fault and freeze the system for controller debug
2109  * purpose.
2110  *
2111  * Return: Nothing.
2112  */
2113 static void mpi3mr_kill_ioc(struct mpi3mr_ioc *mrioc, u32 reason)
2114 {
2115         enum mpi3mr_iocstate ioc_state;
2116
2117         if (!mrioc->fault_dbg)
2118                 return;
2119
2120         dump_stack();
2121
2122         ioc_state = mpi3mr_get_iocstate(mrioc);
2123         if (ioc_state == MRIOC_STATE_FAULT)
2124                 mpi3mr_print_fault_info(mrioc);
2125         else {
2126                 ioc_err(mrioc, "Firmware is halted due to the reason %d\n",
2127                     reason);
2128                 mpi3mr_diagfault_reset_handler(mrioc, reason);
2129         }
2130         if (mrioc->fault_dbg == 2)
2131                 for (;;)
2132                         ;
2133         else
2134                 panic("panic in %s\n", __func__);
2135 }
2136
2137 /**
2138  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2139  * @mrioc: Adapter instance reference
2140  *
2141  * Allocate memory for admin queue pair if required and register
2142  * the admin queue with the controller.
2143  *
2144  * Return: 0 on success, non-zero on failures.
2145  */
2146 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2147 {
2148         int retval = 0;
2149         u32 num_admin_entries = 0;
2150
2151         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2152         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2153             MPI3MR_ADMIN_REQ_FRAME_SZ;
2154         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2155         mrioc->admin_req_base = NULL;
2156
2157         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2158         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2159             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2160         mrioc->admin_reply_ci = 0;
2161         mrioc->admin_reply_ephase = 1;
2162         mrioc->admin_reply_base = NULL;
2163
2164         if (!mrioc->admin_req_base) {
2165                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2166                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2167
2168                 if (!mrioc->admin_req_base) {
2169                         retval = -1;
2170                         goto out_failed;
2171                 }
2172
2173                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2174                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2175                     GFP_KERNEL);
2176
2177                 if (!mrioc->admin_reply_base) {
2178                         retval = -1;
2179                         goto out_failed;
2180                 }
2181         }
2182
2183         num_admin_entries = (mrioc->num_admin_replies << 16) |
2184             (mrioc->num_admin_req);
2185         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2186         mpi3mr_writeq(mrioc->admin_req_dma,
2187             &mrioc->sysif_regs->admin_request_queue_address);
2188         mpi3mr_writeq(mrioc->admin_reply_dma,
2189             &mrioc->sysif_regs->admin_reply_queue_address);
2190         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2191         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2192         return retval;
2193
2194 out_failed:
2195
2196         if (mrioc->admin_reply_base) {
2197                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2198                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2199                 mrioc->admin_reply_base = NULL;
2200         }
2201         if (mrioc->admin_req_base) {
2202                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2203                     mrioc->admin_req_base, mrioc->admin_req_dma);
2204                 mrioc->admin_req_base = NULL;
2205         }
2206         return retval;
2207 }
2208
2209 /**
2210  * mpi3mr_issue_iocfacts - Send IOC Facts
2211  * @mrioc: Adapter instance reference
2212  * @facts_data: Cached IOC facts data
2213  *
2214  * Issue IOC Facts MPI request through admin queue and wait for
2215  * the completion of it or time out.
2216  *
2217  * Return: 0 on success, non-zero on failures.
2218  */
2219 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2220         struct mpi3_ioc_facts_data *facts_data)
2221 {
2222         struct mpi3_ioc_facts_request iocfacts_req;
2223         void *data = NULL;
2224         dma_addr_t data_dma;
2225         u32 data_len = sizeof(*facts_data);
2226         int retval = 0;
2227         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2228
2229         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2230             GFP_KERNEL);
2231
2232         if (!data) {
2233                 retval = -1;
2234                 goto out;
2235         }
2236
2237         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2238         mutex_lock(&mrioc->init_cmds.mutex);
2239         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2240                 retval = -1;
2241                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2242                 mutex_unlock(&mrioc->init_cmds.mutex);
2243                 goto out;
2244         }
2245         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2246         mrioc->init_cmds.is_waiting = 1;
2247         mrioc->init_cmds.callback = NULL;
2248         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2249         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2250
2251         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2252             data_dma);
2253
2254         init_completion(&mrioc->init_cmds.done);
2255         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2256             sizeof(iocfacts_req), 1);
2257         if (retval) {
2258                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2259                 goto out_unlock;
2260         }
2261         wait_for_completion_timeout(&mrioc->init_cmds.done,
2262             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2263         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2264                 ioc_err(mrioc, "Issue IOCFacts: command timed out\n");
2265                 mpi3mr_set_diagsave(mrioc);
2266                 mpi3mr_issue_reset(mrioc,
2267                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2268                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2269                 mrioc->unrecoverable = 1;
2270                 retval = -1;
2271                 goto out_unlock;
2272         }
2273         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2274             != MPI3_IOCSTATUS_SUCCESS) {
2275                 ioc_err(mrioc,
2276                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2277                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2278                     mrioc->init_cmds.ioc_loginfo);
2279                 retval = -1;
2280                 goto out_unlock;
2281         }
2282         memcpy(facts_data, (u8 *)data, data_len);
2283 out_unlock:
2284         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2285         mutex_unlock(&mrioc->init_cmds.mutex);
2286
2287 out:
2288         if (data)
2289                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2290
2291         return retval;
2292 }
2293
2294 /**
2295  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2296  * @mrioc: Adapter instance reference
2297  *
2298  * Check whether the new DMA mask requested through IOCFacts by
2299  * firmware needs to be set, if so set it .
2300  *
2301  * Return: 0 on success, non-zero on failure.
2302  */
2303 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2304 {
2305         struct pci_dev *pdev = mrioc->pdev;
2306         int r;
2307         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2308
2309         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2310                 return 0;
2311
2312         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2313             mrioc->dma_mask, facts_dma_mask);
2314
2315         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2316         if (r) {
2317                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2318                     facts_dma_mask, r);
2319                 return r;
2320         }
2321         mrioc->dma_mask = facts_dma_mask;
2322         return r;
2323 }
2324
2325 /**
2326  * mpi3mr_process_factsdata - Process IOC facts data
2327  * @mrioc: Adapter instance reference
2328  * @facts_data: Cached IOC facts data
2329  *
2330  * Convert IOC facts data into cpu endianness and cache it in
2331  * the driver .
2332  *
2333  * Return: Nothing.
2334  */
2335 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2336         struct mpi3_ioc_facts_data *facts_data)
2337 {
2338         u32 ioc_config, req_sz, facts_flags;
2339
2340         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2341             (sizeof(*facts_data) / 4)) {
2342                 ioc_warn(mrioc,
2343                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2344                     sizeof(*facts_data),
2345                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2346         }
2347
2348         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2349         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2350             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2351         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2352                 ioc_err(mrioc,
2353                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2354                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2355         }
2356
2357         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2358
2359         facts_flags = le32_to_cpu(facts_data->flags);
2360         mrioc->facts.op_req_sz = req_sz;
2361         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2362             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2363             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2364
2365         mrioc->facts.ioc_num = facts_data->ioc_number;
2366         mrioc->facts.who_init = facts_data->who_init;
2367         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2368         mrioc->facts.personality = (facts_flags &
2369             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2370         mrioc->facts.dma_mask = (facts_flags &
2371             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2372             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2373         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2374         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2375         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_request);
2376         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2377         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2378         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2379         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2380         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2381         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2382         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2383         mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2384         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2385         mrioc->facts.max_pcie_switches =
2386             le16_to_cpu(facts_data->max_pcie_switches);
2387         mrioc->facts.max_sasexpanders =
2388             le16_to_cpu(facts_data->max_sas_expanders);
2389         mrioc->facts.max_sasinitiators =
2390             le16_to_cpu(facts_data->max_sas_initiators);
2391         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2392         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2393         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2394         mrioc->facts.max_op_req_q =
2395             le16_to_cpu(facts_data->max_operational_request_queues);
2396         mrioc->facts.max_op_reply_q =
2397             le16_to_cpu(facts_data->max_operational_reply_queues);
2398         mrioc->facts.ioc_capabilities =
2399             le32_to_cpu(facts_data->ioc_capabilities);
2400         mrioc->facts.fw_ver.build_num =
2401             le16_to_cpu(facts_data->fw_version.build_num);
2402         mrioc->facts.fw_ver.cust_id =
2403             le16_to_cpu(facts_data->fw_version.customer_id);
2404         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2405         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2406         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2407         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2408         mrioc->msix_count = min_t(int, mrioc->msix_count,
2409             mrioc->facts.max_msix_vectors);
2410         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2411         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2412         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2413         mrioc->facts.shutdown_timeout =
2414             le16_to_cpu(facts_data->shutdown_timeout);
2415
2416         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2417             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2418             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2419         ioc_info(mrioc,
2420             "maxreqs(%d), mindh(%d) maxvectors(%d) maxperids(%d)\n",
2421             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2422             mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
2423         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2424             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2425             mrioc->facts.sge_mod_shift);
2426         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2427             mrioc->facts.dma_mask, (facts_flags &
2428             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2429
2430         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
2431
2432         if (reset_devices)
2433                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
2434                     MPI3MR_HOST_IOS_KDUMP);
2435 }
2436
2437 /**
2438  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2439  * @mrioc: Adapter instance reference
2440  *
2441  * Allocate and initialize the reply free buffers, sense
2442  * buffers, reply free queue and sense buffer queue.
2443  *
2444  * Return: 0 on success, non-zero on failures.
2445  */
2446 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2447 {
2448         int retval = 0;
2449         u32 sz, i;
2450         dma_addr_t phy_addr;
2451
2452         if (mrioc->init_cmds.reply)
2453                 goto post_reply_sbuf;
2454
2455         mrioc->init_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2456         if (!mrioc->init_cmds.reply)
2457                 goto out_failed;
2458
2459         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2460                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->facts.reply_sz,
2461                     GFP_KERNEL);
2462                 if (!mrioc->dev_rmhs_cmds[i].reply)
2463                         goto out_failed;
2464         }
2465
2466         mrioc->host_tm_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2467         if (!mrioc->host_tm_cmds.reply)
2468                 goto out_failed;
2469
2470         mrioc->dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8;
2471         if (mrioc->facts.max_devhandle % 8)
2472                 mrioc->dev_handle_bitmap_sz++;
2473         mrioc->removepend_bitmap = kzalloc(mrioc->dev_handle_bitmap_sz,
2474             GFP_KERNEL);
2475         if (!mrioc->removepend_bitmap)
2476                 goto out_failed;
2477
2478         mrioc->devrem_bitmap_sz = MPI3MR_NUM_DEVRMCMD / 8;
2479         if (MPI3MR_NUM_DEVRMCMD % 8)
2480                 mrioc->devrem_bitmap_sz++;
2481         mrioc->devrem_bitmap = kzalloc(mrioc->devrem_bitmap_sz,
2482             GFP_KERNEL);
2483         if (!mrioc->devrem_bitmap)
2484                 goto out_failed;
2485
2486         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2487         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2488         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2489         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2490
2491         /* reply buffer pool, 16 byte align */
2492         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2493         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2494             &mrioc->pdev->dev, sz, 16, 0);
2495         if (!mrioc->reply_buf_pool) {
2496                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2497                 goto out_failed;
2498         }
2499
2500         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2501             &mrioc->reply_buf_dma);
2502         if (!mrioc->reply_buf)
2503                 goto out_failed;
2504
2505         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2506
2507         /* reply free queue, 8 byte align */
2508         sz = mrioc->reply_free_qsz * 8;
2509         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2510             &mrioc->pdev->dev, sz, 8, 0);
2511         if (!mrioc->reply_free_q_pool) {
2512                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2513                 goto out_failed;
2514         }
2515         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
2516             GFP_KERNEL, &mrioc->reply_free_q_dma);
2517         if (!mrioc->reply_free_q)
2518                 goto out_failed;
2519
2520         /* sense buffer pool,  4 byte align */
2521         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
2522         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
2523             &mrioc->pdev->dev, sz, 4, 0);
2524         if (!mrioc->sense_buf_pool) {
2525                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
2526                 goto out_failed;
2527         }
2528         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
2529             &mrioc->sense_buf_dma);
2530         if (!mrioc->sense_buf)
2531                 goto out_failed;
2532
2533         /* sense buffer queue, 8 byte align */
2534         sz = mrioc->sense_buf_q_sz * 8;
2535         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
2536             &mrioc->pdev->dev, sz, 8, 0);
2537         if (!mrioc->sense_buf_q_pool) {
2538                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
2539                 goto out_failed;
2540         }
2541         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
2542             GFP_KERNEL, &mrioc->sense_buf_q_dma);
2543         if (!mrioc->sense_buf_q)
2544                 goto out_failed;
2545
2546 post_reply_sbuf:
2547         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2548         ioc_info(mrioc,
2549             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2550             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->facts.reply_sz,
2551             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
2552         sz = mrioc->reply_free_qsz * 8;
2553         ioc_info(mrioc,
2554             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2555             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
2556             (unsigned long long)mrioc->reply_free_q_dma);
2557         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
2558         ioc_info(mrioc,
2559             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2560             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
2561             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
2562         sz = mrioc->sense_buf_q_sz * 8;
2563         ioc_info(mrioc,
2564             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2565             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
2566             (unsigned long long)mrioc->sense_buf_q_dma);
2567
2568         /* initialize Reply buffer Queue */
2569         for (i = 0, phy_addr = mrioc->reply_buf_dma;
2570             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->facts.reply_sz)
2571                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
2572         mrioc->reply_free_q[i] = cpu_to_le64(0);
2573
2574         /* initialize Sense Buffer Queue */
2575         for (i = 0, phy_addr = mrioc->sense_buf_dma;
2576             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
2577                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
2578         mrioc->sense_buf_q[i] = cpu_to_le64(0);
2579         return retval;
2580
2581 out_failed:
2582         retval = -1;
2583         return retval;
2584 }
2585
2586 /**
2587  * mpi3mr_issue_iocinit - Send IOC Init
2588  * @mrioc: Adapter instance reference
2589  *
2590  * Issue IOC Init MPI request through admin queue and wait for
2591  * the completion of it or time out.
2592  *
2593  * Return: 0 on success, non-zero on failures.
2594  */
2595 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
2596 {
2597         struct mpi3_ioc_init_request iocinit_req;
2598         struct mpi3_driver_info_layout *drv_info;
2599         dma_addr_t data_dma;
2600         u32 data_len = sizeof(*drv_info);
2601         int retval = 0;
2602         ktime_t current_time;
2603
2604         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2605             GFP_KERNEL);
2606         if (!drv_info) {
2607                 retval = -1;
2608                 goto out;
2609         }
2610         drv_info->information_length = cpu_to_le32(data_len);
2611         strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
2612         strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
2613         strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
2614         strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
2615         strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
2616         strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
2617             sizeof(drv_info->driver_release_date));
2618         drv_info->driver_capabilities = 0;
2619         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
2620             sizeof(mrioc->driver_info));
2621
2622         memset(&iocinit_req, 0, sizeof(iocinit_req));
2623         mutex_lock(&mrioc->init_cmds.mutex);
2624         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2625                 retval = -1;
2626                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
2627                 mutex_unlock(&mrioc->init_cmds.mutex);
2628                 goto out;
2629         }
2630         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2631         mrioc->init_cmds.is_waiting = 1;
2632         mrioc->init_cmds.callback = NULL;
2633         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2634         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
2635         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
2636         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
2637         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
2638         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
2639         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
2640         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
2641         iocinit_req.reply_free_queue_address =
2642             cpu_to_le64(mrioc->reply_free_q_dma);
2643         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSE_BUF_SZ);
2644         iocinit_req.sense_buffer_free_queue_depth =
2645             cpu_to_le16(mrioc->sense_buf_q_sz);
2646         iocinit_req.sense_buffer_free_queue_address =
2647             cpu_to_le64(mrioc->sense_buf_q_dma);
2648         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
2649
2650         current_time = ktime_get_real();
2651         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
2652
2653         init_completion(&mrioc->init_cmds.done);
2654         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
2655             sizeof(iocinit_req), 1);
2656         if (retval) {
2657                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
2658                 goto out_unlock;
2659         }
2660         wait_for_completion_timeout(&mrioc->init_cmds.done,
2661             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2662         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2663                 mpi3mr_set_diagsave(mrioc);
2664                 mpi3mr_issue_reset(mrioc,
2665                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2666                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
2667                 mrioc->unrecoverable = 1;
2668                 ioc_err(mrioc, "Issue IOCInit: command timed out\n");
2669                 retval = -1;
2670                 goto out_unlock;
2671         }
2672         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2673             != MPI3_IOCSTATUS_SUCCESS) {
2674                 ioc_err(mrioc,
2675                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2676                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2677                     mrioc->init_cmds.ioc_loginfo);
2678                 retval = -1;
2679                 goto out_unlock;
2680         }
2681
2682 out_unlock:
2683         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2684         mutex_unlock(&mrioc->init_cmds.mutex);
2685
2686 out:
2687         if (drv_info)
2688                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
2689                     data_dma);
2690
2691         return retval;
2692 }
2693
2694 /**
2695  * mpi3mr_unmask_events - Unmask events in event mask bitmap
2696  * @mrioc: Adapter instance reference
2697  * @event: MPI event ID
2698  *
2699  * Un mask the specific event by resetting the event_mask
2700  * bitmap.
2701  *
2702  * Return: 0 on success, non-zero on failures.
2703  */
2704 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
2705 {
2706         u32 desired_event;
2707         u8 word;
2708
2709         if (event >= 128)
2710                 return;
2711
2712         desired_event = (1 << (event % 32));
2713         word = event / 32;
2714
2715         mrioc->event_masks[word] &= ~desired_event;
2716 }
2717
2718 /**
2719  * mpi3mr_issue_event_notification - Send event notification
2720  * @mrioc: Adapter instance reference
2721  *
2722  * Issue event notification MPI request through admin queue and
2723  * wait for the completion of it or time out.
2724  *
2725  * Return: 0 on success, non-zero on failures.
2726  */
2727 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
2728 {
2729         struct mpi3_event_notification_request evtnotify_req;
2730         int retval = 0;
2731         u8 i;
2732
2733         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
2734         mutex_lock(&mrioc->init_cmds.mutex);
2735         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2736                 retval = -1;
2737                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
2738                 mutex_unlock(&mrioc->init_cmds.mutex);
2739                 goto out;
2740         }
2741         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2742         mrioc->init_cmds.is_waiting = 1;
2743         mrioc->init_cmds.callback = NULL;
2744         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2745         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
2746         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2747                 evtnotify_req.event_masks[i] =
2748                     cpu_to_le32(mrioc->event_masks[i]);
2749         init_completion(&mrioc->init_cmds.done);
2750         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
2751             sizeof(evtnotify_req), 1);
2752         if (retval) {
2753                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
2754                 goto out_unlock;
2755         }
2756         wait_for_completion_timeout(&mrioc->init_cmds.done,
2757             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2758         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2759                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2760                 mpi3mr_set_diagsave(mrioc);
2761                 mpi3mr_issue_reset(mrioc,
2762                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2763                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
2764                 mrioc->unrecoverable = 1;
2765                 retval = -1;
2766                 goto out_unlock;
2767         }
2768         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2769             != MPI3_IOCSTATUS_SUCCESS) {
2770                 ioc_err(mrioc,
2771                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2772                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2773                     mrioc->init_cmds.ioc_loginfo);
2774                 retval = -1;
2775                 goto out_unlock;
2776         }
2777
2778 out_unlock:
2779         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2780         mutex_unlock(&mrioc->init_cmds.mutex);
2781 out:
2782         return retval;
2783 }
2784
2785 /**
2786  * mpi3mr_send_event_ack - Send event acknowledgment
2787  * @mrioc: Adapter instance reference
2788  * @event: MPI3 event ID
2789  * @event_ctx: Event context
2790  *
2791  * Send event acknowledgment through admin queue and wait for
2792  * it to complete.
2793  *
2794  * Return: 0 on success, non-zero on failures.
2795  */
2796 int mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
2797         u32 event_ctx)
2798 {
2799         struct mpi3_event_ack_request evtack_req;
2800         int retval = 0;
2801
2802         memset(&evtack_req, 0, sizeof(evtack_req));
2803         mutex_lock(&mrioc->init_cmds.mutex);
2804         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2805                 retval = -1;
2806                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
2807                 mutex_unlock(&mrioc->init_cmds.mutex);
2808                 goto out;
2809         }
2810         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2811         mrioc->init_cmds.is_waiting = 1;
2812         mrioc->init_cmds.callback = NULL;
2813         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2814         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
2815         evtack_req.event = event;
2816         evtack_req.event_context = cpu_to_le32(event_ctx);
2817
2818         init_completion(&mrioc->init_cmds.done);
2819         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
2820             sizeof(evtack_req), 1);
2821         if (retval) {
2822                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
2823                 goto out_unlock;
2824         }
2825         wait_for_completion_timeout(&mrioc->init_cmds.done,
2826             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2827         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2828                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2829                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2830                         mpi3mr_soft_reset_handler(mrioc,
2831                             MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
2832                 retval = -1;
2833                 goto out_unlock;
2834         }
2835         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2836             != MPI3_IOCSTATUS_SUCCESS) {
2837                 ioc_err(mrioc,
2838                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2839                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2840                     mrioc->init_cmds.ioc_loginfo);
2841                 retval = -1;
2842                 goto out_unlock;
2843         }
2844
2845 out_unlock:
2846         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2847         mutex_unlock(&mrioc->init_cmds.mutex);
2848 out:
2849         return retval;
2850 }
2851
2852 /**
2853  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
2854  * @mrioc: Adapter instance reference
2855  *
2856  * Allocate chain buffers and set a bitmap to indicate free
2857  * chain buffers. Chain buffers are used to pass the SGE
2858  * information along with MPI3 SCSI IO requests for host I/O.
2859  *
2860  * Return: 0 on success, non-zero on failure
2861  */
2862 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
2863 {
2864         int retval = 0;
2865         u32 sz, i;
2866         u16 num_chains;
2867
2868         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
2869
2870         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
2871             | SHOST_DIX_TYPE1_PROTECTION
2872             | SHOST_DIX_TYPE2_PROTECTION
2873             | SHOST_DIX_TYPE3_PROTECTION))
2874                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
2875
2876         mrioc->chain_buf_count = num_chains;
2877         sz = sizeof(struct chain_element) * num_chains;
2878         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
2879         if (!mrioc->chain_sgl_list)
2880                 goto out_failed;
2881
2882         sz = MPI3MR_PAGE_SIZE_4K;
2883         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
2884             &mrioc->pdev->dev, sz, 16, 0);
2885         if (!mrioc->chain_buf_pool) {
2886                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
2887                 goto out_failed;
2888         }
2889
2890         for (i = 0; i < num_chains; i++) {
2891                 mrioc->chain_sgl_list[i].addr =
2892                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
2893                     &mrioc->chain_sgl_list[i].dma_addr);
2894
2895                 if (!mrioc->chain_sgl_list[i].addr)
2896                         goto out_failed;
2897         }
2898         mrioc->chain_bitmap_sz = num_chains / 8;
2899         if (num_chains % 8)
2900                 mrioc->chain_bitmap_sz++;
2901         mrioc->chain_bitmap = kzalloc(mrioc->chain_bitmap_sz, GFP_KERNEL);
2902         if (!mrioc->chain_bitmap)
2903                 goto out_failed;
2904         return retval;
2905 out_failed:
2906         retval = -1;
2907         return retval;
2908 }
2909
2910 /**
2911  * mpi3mr_port_enable_complete - Mark port enable complete
2912  * @mrioc: Adapter instance reference
2913  * @drv_cmd: Internal command tracker
2914  *
2915  * Call back for asynchronous port enable request sets the
2916  * driver command to indicate port enable request is complete.
2917  *
2918  * Return: Nothing
2919  */
2920 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
2921         struct mpi3mr_drv_cmd *drv_cmd)
2922 {
2923         drv_cmd->state = MPI3MR_CMD_NOTUSED;
2924         drv_cmd->callback = NULL;
2925         mrioc->scan_failed = drv_cmd->ioc_status;
2926         mrioc->scan_started = 0;
2927 }
2928
2929 /**
2930  * mpi3mr_issue_port_enable - Issue Port Enable
2931  * @mrioc: Adapter instance reference
2932  * @async: Flag to wait for completion or not
2933  *
2934  * Issue Port Enable MPI request through admin queue and if the
2935  * async flag is not set wait for the completion of the port
2936  * enable or time out.
2937  *
2938  * Return: 0 on success, non-zero on failures.
2939  */
2940 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
2941 {
2942         struct mpi3_port_enable_request pe_req;
2943         int retval = 0;
2944         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
2945
2946         memset(&pe_req, 0, sizeof(pe_req));
2947         mutex_lock(&mrioc->init_cmds.mutex);
2948         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2949                 retval = -1;
2950                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
2951                 mutex_unlock(&mrioc->init_cmds.mutex);
2952                 goto out;
2953         }
2954         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2955         if (async) {
2956                 mrioc->init_cmds.is_waiting = 0;
2957                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
2958         } else {
2959                 mrioc->init_cmds.is_waiting = 1;
2960                 mrioc->init_cmds.callback = NULL;
2961                 init_completion(&mrioc->init_cmds.done);
2962         }
2963         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2964         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
2965
2966         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
2967         if (retval) {
2968                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
2969                 goto out_unlock;
2970         }
2971         if (!async) {
2972                 wait_for_completion_timeout(&mrioc->init_cmds.done,
2973                     (pe_timeout * HZ));
2974                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2975                         ioc_err(mrioc, "Issue PortEnable: command timed out\n");
2976                         retval = -1;
2977                         mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
2978                         mpi3mr_set_diagsave(mrioc);
2979                         mpi3mr_issue_reset(mrioc,
2980                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2981                             MPI3MR_RESET_FROM_PE_TIMEOUT);
2982                         mrioc->unrecoverable = 1;
2983                         goto out_unlock;
2984                 }
2985                 mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
2986         }
2987 out_unlock:
2988         mutex_unlock(&mrioc->init_cmds.mutex);
2989 out:
2990         return retval;
2991 }
2992
2993 /* Protocol type to name mapper structure*/
2994 static const struct {
2995         u8 protocol;
2996         char *name;
2997 } mpi3mr_protocols[] = {
2998         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
2999         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3000         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3001 };
3002
3003 /* Capability to name mapper structure*/
3004 static const struct {
3005         u32 capability;
3006         char *name;
3007 } mpi3mr_capabilities[] = {
3008         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3009 };
3010
3011 /**
3012  * mpi3mr_print_ioc_info - Display controller information
3013  * @mrioc: Adapter instance reference
3014  *
3015  * Display controller personalit, capability, supported
3016  * protocols etc.
3017  *
3018  * Return: Nothing
3019  */
3020 static void
3021 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3022 {
3023         int i = 0, bytes_written = 0;
3024         char personality[16];
3025         char protocol[50] = {0};
3026         char capabilities[100] = {0};
3027         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3028
3029         switch (mrioc->facts.personality) {
3030         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3031                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3032                 break;
3033         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3034                 strncpy(personality, "RAID", sizeof(personality));
3035                 break;
3036         default:
3037                 strncpy(personality, "Unknown", sizeof(personality));
3038                 break;
3039         }
3040
3041         ioc_info(mrioc, "Running in %s Personality", personality);
3042
3043         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3044             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3045             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3046
3047         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3048                 if (mrioc->facts.protocol_flags &
3049                     mpi3mr_protocols[i].protocol) {
3050                         bytes_written += scnprintf(protocol + bytes_written,
3051                                     sizeof(protocol) - bytes_written, "%s%s",
3052                                     bytes_written ? "," : "",
3053                                     mpi3mr_protocols[i].name);
3054                 }
3055         }
3056
3057         bytes_written = 0;
3058         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3059                 if (mrioc->facts.protocol_flags &
3060                     mpi3mr_capabilities[i].capability) {
3061                         bytes_written += scnprintf(capabilities + bytes_written,
3062                                     sizeof(capabilities) - bytes_written, "%s%s",
3063                                     bytes_written ? "," : "",
3064                                     mpi3mr_capabilities[i].name);
3065                 }
3066         }
3067
3068         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3069                  protocol, capabilities);
3070 }
3071
3072 /**
3073  * mpi3mr_cleanup_resources - Free PCI resources
3074  * @mrioc: Adapter instance reference
3075  *
3076  * Unmap PCI device memory and disable PCI device.
3077  *
3078  * Return: 0 on success and non-zero on failure.
3079  */
3080 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3081 {
3082         struct pci_dev *pdev = mrioc->pdev;
3083
3084         mpi3mr_cleanup_isr(mrioc);
3085
3086         if (mrioc->sysif_regs) {
3087                 iounmap((void __iomem *)mrioc->sysif_regs);
3088                 mrioc->sysif_regs = NULL;
3089         }
3090
3091         if (pci_is_enabled(pdev)) {
3092                 if (mrioc->bars)
3093                         pci_release_selected_regions(pdev, mrioc->bars);
3094                 pci_disable_device(pdev);
3095         }
3096 }
3097
3098 /**
3099  * mpi3mr_setup_resources - Enable PCI resources
3100  * @mrioc: Adapter instance reference
3101  *
3102  * Enable PCI device memory, MSI-x registers and set DMA mask.
3103  *
3104  * Return: 0 on success and non-zero on failure.
3105  */
3106 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3107 {
3108         struct pci_dev *pdev = mrioc->pdev;
3109         u32 memap_sz = 0;
3110         int i, retval = 0, capb = 0;
3111         u16 message_control;
3112         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3113             (((dma_get_required_mask(&pdev->dev) > DMA_BIT_MASK(32)) &&
3114             (sizeof(dma_addr_t) > 4)) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3115
3116         if (pci_enable_device_mem(pdev)) {
3117                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3118                 retval = -ENODEV;
3119                 goto out_failed;
3120         }
3121
3122         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3123         if (!capb) {
3124                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3125                 retval = -ENODEV;
3126                 goto out_failed;
3127         }
3128         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3129
3130         if (pci_request_selected_regions(pdev, mrioc->bars,
3131             mrioc->driver_name)) {
3132                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3133                 retval = -ENODEV;
3134                 goto out_failed;
3135         }
3136
3137         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3138                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3139                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3140                         memap_sz = pci_resource_len(pdev, i);
3141                         mrioc->sysif_regs =
3142                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3143                         break;
3144                 }
3145         }
3146
3147         pci_set_master(pdev);
3148
3149         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3150         if (retval) {
3151                 if (dma_mask != DMA_BIT_MASK(32)) {
3152                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3153                         dma_mask = DMA_BIT_MASK(32);
3154                         retval = dma_set_mask_and_coherent(&pdev->dev,
3155                             dma_mask);
3156                 }
3157                 if (retval) {
3158                         mrioc->dma_mask = 0;
3159                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3160                         goto out_failed;
3161                 }
3162         }
3163         mrioc->dma_mask = dma_mask;
3164
3165         if (!mrioc->sysif_regs) {
3166                 ioc_err(mrioc,
3167                     "Unable to map adapter memory or resource not found\n");
3168                 retval = -EINVAL;
3169                 goto out_failed;
3170         }
3171
3172         pci_read_config_word(pdev, capb + 2, &message_control);
3173         mrioc->msix_count = (message_control & 0x3FF) + 1;
3174
3175         pci_save_state(pdev);
3176
3177         pci_set_drvdata(pdev, mrioc->shost);
3178
3179         mpi3mr_ioc_disable_intr(mrioc);
3180
3181         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3182             (unsigned long long)mrioc->sysif_regs_phys,
3183             mrioc->sysif_regs, memap_sz);
3184         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3185             mrioc->msix_count);
3186         return retval;
3187
3188 out_failed:
3189         mpi3mr_cleanup_resources(mrioc);
3190         return retval;
3191 }
3192
3193 /**
3194  * mpi3mr_init_ioc - Initialize the controller
3195  * @mrioc: Adapter instance reference
3196  * @init_type: Flag to indicate is the init_type
3197  *
3198  * This the controller initialization routine, executed either
3199  * after soft reset or from pci probe callback.
3200  * Setup the required resources, memory map the controller
3201  * registers, create admin and operational reply queue pairs,
3202  * allocate required memory for reply pool, sense buffer pool,
3203  * issue IOC init request to the firmware, unmask the events and
3204  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3205  * volumes.
3206  *
3207  * Return: 0 on success and non-zero on failure.
3208  */
3209 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc, u8 init_type)
3210 {
3211         int retval = 0;
3212         enum mpi3mr_iocstate ioc_state;
3213         u64 base_info;
3214         u32 timeout;
3215         u32 ioc_status, ioc_config, i;
3216         struct mpi3_ioc_facts_data facts_data;
3217
3218         mrioc->irqpoll_sleep = MPI3MR_IRQ_POLL_SLEEP;
3219         mrioc->change_count = 0;
3220         if (init_type == MPI3MR_IT_INIT) {
3221                 mrioc->cpu_count = num_online_cpus();
3222                 retval = mpi3mr_setup_resources(mrioc);
3223                 if (retval) {
3224                         ioc_err(mrioc, "Failed to setup resources:error %d\n",
3225                             retval);
3226                         goto out_nocleanup;
3227                 }
3228         }
3229
3230         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3231         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3232
3233         ioc_info(mrioc, "SOD status %x configuration %x\n",
3234             ioc_status, ioc_config);
3235
3236         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
3237         ioc_info(mrioc, "SOD base_info %llx\n", base_info);
3238
3239         /*The timeout value is in 2sec unit, changing it to seconds*/
3240         mrioc->ready_timeout =
3241             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
3242             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
3243
3244         ioc_info(mrioc, "IOC ready timeout %d\n", mrioc->ready_timeout);
3245
3246         ioc_state = mpi3mr_get_iocstate(mrioc);
3247         ioc_info(mrioc, "IOC in %s state during detection\n",
3248             mpi3mr_iocstate_name(ioc_state));
3249
3250         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
3251             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
3252                 timeout = mrioc->ready_timeout * 10;
3253                 do {
3254                         msleep(100);
3255                 } while (--timeout);
3256
3257                 ioc_state = mpi3mr_get_iocstate(mrioc);
3258                 ioc_info(mrioc,
3259                     "IOC in %s state after waiting for reset time\n",
3260                     mpi3mr_iocstate_name(ioc_state));
3261         }
3262
3263         if (ioc_state == MRIOC_STATE_READY) {
3264                 retval = mpi3mr_issue_and_process_mur(mrioc,
3265                     MPI3MR_RESET_FROM_BRINGUP);
3266                 if (retval) {
3267                         ioc_err(mrioc, "Failed to MU reset IOC error %d\n",
3268                             retval);
3269                 }
3270                 ioc_state = mpi3mr_get_iocstate(mrioc);
3271         }
3272         if (ioc_state != MRIOC_STATE_RESET) {
3273                 mpi3mr_print_fault_info(mrioc);
3274                 retval = mpi3mr_issue_reset(mrioc,
3275                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3276                     MPI3MR_RESET_FROM_BRINGUP);
3277                 if (retval) {
3278                         ioc_err(mrioc,
3279                             "%s :Failed to soft reset IOC error %d\n",
3280                             __func__, retval);
3281                         goto out_failed;
3282                 }
3283         }
3284         ioc_state = mpi3mr_get_iocstate(mrioc);
3285         if (ioc_state != MRIOC_STATE_RESET) {
3286                 retval = -1;
3287                 ioc_err(mrioc, "Cannot bring IOC to reset state\n");
3288                 goto out_failed;
3289         }
3290
3291         retval = mpi3mr_setup_admin_qpair(mrioc);
3292         if (retval) {
3293                 ioc_err(mrioc, "Failed to setup admin Qs: error %d\n",
3294                     retval);
3295                 goto out_failed;
3296         }
3297
3298         retval = mpi3mr_bring_ioc_ready(mrioc);
3299         if (retval) {
3300                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3301                     retval);
3302                 goto out_failed;
3303         }
3304
3305         if (init_type != MPI3MR_IT_RESET) {
3306                 retval = mpi3mr_setup_isr(mrioc, 1);
3307                 if (retval) {
3308                         ioc_err(mrioc, "Failed to setup ISR error %d\n",
3309                             retval);
3310                         goto out_failed;
3311                 }
3312         } else
3313                 mpi3mr_ioc_enable_intr(mrioc);
3314
3315         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3316         if (retval) {
3317                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3318                     retval);
3319                 goto out_failed;
3320         }
3321
3322         mpi3mr_process_factsdata(mrioc, &facts_data);
3323         if (init_type == MPI3MR_IT_INIT) {
3324                 retval = mpi3mr_check_reset_dma_mask(mrioc);
3325                 if (retval) {
3326                         ioc_err(mrioc, "Resetting dma mask failed %d\n",
3327                             retval);
3328                         goto out_failed;
3329                 }
3330         }
3331
3332         mpi3mr_print_ioc_info(mrioc);
3333
3334         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3335         if (retval) {
3336                 ioc_err(mrioc,
3337                     "%s :Failed to allocated reply sense buffers %d\n",
3338                     __func__, retval);
3339                 goto out_failed;
3340         }
3341
3342         if (init_type == MPI3MR_IT_INIT) {
3343                 retval = mpi3mr_alloc_chain_bufs(mrioc);
3344                 if (retval) {
3345                         ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3346                             retval);
3347                         goto out_failed;
3348                 }
3349         }
3350
3351         retval = mpi3mr_issue_iocinit(mrioc);
3352         if (retval) {
3353                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3354                     retval);
3355                 goto out_failed;
3356         }
3357         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3358         writel(mrioc->reply_free_queue_host_index,
3359             &mrioc->sysif_regs->reply_free_host_index);
3360
3361         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3362         writel(mrioc->sbq_host_index,
3363             &mrioc->sysif_regs->sense_buffer_free_host_index);
3364
3365         if (init_type != MPI3MR_IT_RESET) {
3366                 retval = mpi3mr_setup_isr(mrioc, 0);
3367                 if (retval) {
3368                         ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3369                             retval);
3370                         goto out_failed;
3371                 }
3372         }
3373
3374         retval = mpi3mr_create_op_queues(mrioc);
3375         if (retval) {
3376                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3377                     retval);
3378                 goto out_failed;
3379         }
3380
3381         if ((init_type != MPI3MR_IT_INIT) &&
3382             (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q)) {
3383                 retval = -1;
3384                 ioc_err(mrioc,
3385                     "Cannot create minimum number of OpQueues expected:%d created:%d\n",
3386                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
3387                 goto out_failed;
3388         }
3389
3390         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3391                 mrioc->event_masks[i] = -1;
3392
3393         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3394         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3395         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3396         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3397         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3398         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3399         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3400         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3401         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3402         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3403         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3404         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3405
3406         retval = mpi3mr_issue_event_notification(mrioc);
3407         if (retval) {
3408                 ioc_err(mrioc, "Failed to issue event notification %d\n",
3409                     retval);
3410                 goto out_failed;
3411         }
3412
3413         if (init_type != MPI3MR_IT_INIT) {
3414                 ioc_info(mrioc, "Issuing Port Enable\n");
3415                 retval = mpi3mr_issue_port_enable(mrioc, 0);
3416                 if (retval) {
3417                         ioc_err(mrioc, "Failed to issue port enable %d\n",
3418                             retval);
3419                         goto out_failed;
3420                 }
3421         }
3422         return retval;
3423
3424 out_failed:
3425         if (init_type == MPI3MR_IT_INIT)
3426                 mpi3mr_cleanup_ioc(mrioc, MPI3MR_COMPLETE_CLEANUP);
3427         else
3428                 mpi3mr_cleanup_ioc(mrioc, MPI3MR_REINIT_FAILURE);
3429 out_nocleanup:
3430         return retval;
3431 }
3432
3433 /**
3434  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
3435  *                                      segments
3436  * @mrioc: Adapter instance reference
3437  * @qidx: Operational reply queue index
3438  *
3439  * Return: Nothing.
3440  */
3441 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3442 {
3443         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
3444         struct segments *segments;
3445         int i, size;
3446
3447         if (!op_reply_q->q_segments)
3448                 return;
3449
3450         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
3451         segments = op_reply_q->q_segments;
3452         for (i = 0; i < op_reply_q->num_segments; i++)
3453                 memset(segments[i].segment, 0, size);
3454 }
3455
3456 /**
3457  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
3458  *                                      segments
3459  * @mrioc: Adapter instance reference
3460  * @qidx: Operational request queue index
3461  *
3462  * Return: Nothing.
3463  */
3464 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3465 {
3466         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
3467         struct segments *segments;
3468         int i, size;
3469
3470         if (!op_req_q->q_segments)
3471                 return;
3472
3473         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
3474         segments = op_req_q->q_segments;
3475         for (i = 0; i < op_req_q->num_segments; i++)
3476                 memset(segments[i].segment, 0, size);
3477 }
3478
3479 /**
3480  * mpi3mr_memset_buffers - memset memory for a controller
3481  * @mrioc: Adapter instance reference
3482  *
3483  * clear all the memory allocated for a controller, typically
3484  * called post reset to reuse the memory allocated during the
3485  * controller init.
3486  *
3487  * Return: Nothing.
3488  */
3489 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
3490 {
3491         u16 i;
3492
3493         memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
3494         memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
3495
3496         memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
3497         memset(mrioc->host_tm_cmds.reply, 0,
3498             sizeof(*mrioc->host_tm_cmds.reply));
3499         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
3500                 memset(mrioc->dev_rmhs_cmds[i].reply, 0,
3501                     sizeof(*mrioc->dev_rmhs_cmds[i].reply));
3502         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3503         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3504
3505         for (i = 0; i < mrioc->num_queues; i++) {
3506                 mrioc->op_reply_qinfo[i].qid = 0;
3507                 mrioc->op_reply_qinfo[i].ci = 0;
3508                 mrioc->op_reply_qinfo[i].num_replies = 0;
3509                 mrioc->op_reply_qinfo[i].ephase = 0;
3510                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
3511                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
3512                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
3513
3514                 mrioc->req_qinfo[i].ci = 0;
3515                 mrioc->req_qinfo[i].pi = 0;
3516                 mrioc->req_qinfo[i].num_requests = 0;
3517                 mrioc->req_qinfo[i].qid = 0;
3518                 mrioc->req_qinfo[i].reply_qid = 0;
3519                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
3520                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
3521         }
3522 }
3523
3524 /**
3525  * mpi3mr_free_mem - Free memory allocated for a controller
3526  * @mrioc: Adapter instance reference
3527  *
3528  * Free all the memory allocated for a controller.
3529  *
3530  * Return: Nothing.
3531  */
3532 static void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
3533 {
3534         u16 i;
3535         struct mpi3mr_intr_info *intr_info;
3536
3537         if (mrioc->sense_buf_pool) {
3538                 if (mrioc->sense_buf)
3539                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
3540                             mrioc->sense_buf_dma);
3541                 dma_pool_destroy(mrioc->sense_buf_pool);
3542                 mrioc->sense_buf = NULL;
3543                 mrioc->sense_buf_pool = NULL;
3544         }
3545         if (mrioc->sense_buf_q_pool) {
3546                 if (mrioc->sense_buf_q)
3547                         dma_pool_free(mrioc->sense_buf_q_pool,
3548                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
3549                 dma_pool_destroy(mrioc->sense_buf_q_pool);
3550                 mrioc->sense_buf_q = NULL;
3551                 mrioc->sense_buf_q_pool = NULL;
3552         }
3553
3554         if (mrioc->reply_buf_pool) {
3555                 if (mrioc->reply_buf)
3556                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
3557                             mrioc->reply_buf_dma);
3558                 dma_pool_destroy(mrioc->reply_buf_pool);
3559                 mrioc->reply_buf = NULL;
3560                 mrioc->reply_buf_pool = NULL;
3561         }
3562         if (mrioc->reply_free_q_pool) {
3563                 if (mrioc->reply_free_q)
3564                         dma_pool_free(mrioc->reply_free_q_pool,
3565                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
3566                 dma_pool_destroy(mrioc->reply_free_q_pool);
3567                 mrioc->reply_free_q = NULL;
3568                 mrioc->reply_free_q_pool = NULL;
3569         }
3570
3571         for (i = 0; i < mrioc->num_op_req_q; i++)
3572                 mpi3mr_free_op_req_q_segments(mrioc, i);
3573
3574         for (i = 0; i < mrioc->num_op_reply_q; i++)
3575                 mpi3mr_free_op_reply_q_segments(mrioc, i);
3576
3577         for (i = 0; i < mrioc->intr_info_count; i++) {
3578                 intr_info = mrioc->intr_info + i;
3579                 intr_info->op_reply_q = NULL;
3580         }
3581
3582         kfree(mrioc->req_qinfo);
3583         mrioc->req_qinfo = NULL;
3584         mrioc->num_op_req_q = 0;
3585
3586         kfree(mrioc->op_reply_qinfo);
3587         mrioc->op_reply_qinfo = NULL;
3588         mrioc->num_op_reply_q = 0;
3589
3590         kfree(mrioc->init_cmds.reply);
3591         mrioc->init_cmds.reply = NULL;
3592
3593         kfree(mrioc->host_tm_cmds.reply);
3594         mrioc->host_tm_cmds.reply = NULL;
3595
3596         kfree(mrioc->removepend_bitmap);
3597         mrioc->removepend_bitmap = NULL;
3598
3599         kfree(mrioc->devrem_bitmap);
3600         mrioc->devrem_bitmap = NULL;
3601
3602         kfree(mrioc->chain_bitmap);
3603         mrioc->chain_bitmap = NULL;
3604
3605         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3606                 kfree(mrioc->dev_rmhs_cmds[i].reply);
3607                 mrioc->dev_rmhs_cmds[i].reply = NULL;
3608         }
3609
3610         if (mrioc->chain_buf_pool) {
3611                 for (i = 0; i < mrioc->chain_buf_count; i++) {
3612                         if (mrioc->chain_sgl_list[i].addr) {
3613                                 dma_pool_free(mrioc->chain_buf_pool,
3614                                     mrioc->chain_sgl_list[i].addr,
3615                                     mrioc->chain_sgl_list[i].dma_addr);
3616                                 mrioc->chain_sgl_list[i].addr = NULL;
3617                         }
3618                 }
3619                 dma_pool_destroy(mrioc->chain_buf_pool);
3620                 mrioc->chain_buf_pool = NULL;
3621         }
3622
3623         kfree(mrioc->chain_sgl_list);
3624         mrioc->chain_sgl_list = NULL;
3625
3626         if (mrioc->admin_reply_base) {
3627                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
3628                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
3629                 mrioc->admin_reply_base = NULL;
3630         }
3631         if (mrioc->admin_req_base) {
3632                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
3633                     mrioc->admin_req_base, mrioc->admin_req_dma);
3634                 mrioc->admin_req_base = NULL;
3635         }
3636 }
3637
3638 /**
3639  * mpi3mr_issue_ioc_shutdown - shutdown controller
3640  * @mrioc: Adapter instance reference
3641  *
3642  * Send shutodwn notification to the controller and wait for the
3643  * shutdown_timeout for it to be completed.
3644  *
3645  * Return: Nothing.
3646  */
3647 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
3648 {
3649         u32 ioc_config, ioc_status;
3650         u8 retval = 1;
3651         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
3652
3653         ioc_info(mrioc, "Issuing shutdown Notification\n");
3654         if (mrioc->unrecoverable) {
3655                 ioc_warn(mrioc,
3656                     "IOC is unrecoverable shutdown is not issued\n");
3657                 return;
3658         }
3659         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3660         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3661             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
3662                 ioc_info(mrioc, "shutdown already in progress\n");
3663                 return;
3664         }
3665
3666         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3667         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
3668         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
3669
3670         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
3671
3672         if (mrioc->facts.shutdown_timeout)
3673                 timeout = mrioc->facts.shutdown_timeout * 10;
3674
3675         do {
3676                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3677                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3678                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
3679                         retval = 0;
3680                         break;
3681                 }
3682                 msleep(100);
3683         } while (--timeout);
3684
3685         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3686         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3687
3688         if (retval) {
3689                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3690                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
3691                         ioc_warn(mrioc,
3692                             "shutdown still in progress after timeout\n");
3693         }
3694
3695         ioc_info(mrioc,
3696             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
3697             (!retval) ? "successful" : "failed", ioc_status,
3698             ioc_config);
3699 }
3700
3701 /**
3702  * mpi3mr_cleanup_ioc - Cleanup controller
3703  * @mrioc: Adapter instance reference
3704  * @reason: Cleanup reason
3705  *
3706  * controller cleanup handler, Message unit reset or soft reset
3707  * and shutdown notification is issued to the controller and the
3708  * associated memory resources are freed.
3709  *
3710  * Return: Nothing.
3711  */
3712 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc, u8 reason)
3713 {
3714         enum mpi3mr_iocstate ioc_state;
3715
3716         if (reason == MPI3MR_COMPLETE_CLEANUP)
3717                 mpi3mr_stop_watchdog(mrioc);
3718
3719         mpi3mr_ioc_disable_intr(mrioc);
3720
3721         ioc_state = mpi3mr_get_iocstate(mrioc);
3722
3723         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
3724             (ioc_state == MRIOC_STATE_READY)) {
3725                 if (mpi3mr_issue_and_process_mur(mrioc,
3726                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
3727                         mpi3mr_issue_reset(mrioc,
3728                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3729                             MPI3MR_RESET_FROM_MUR_FAILURE);
3730
3731                 if (reason != MPI3MR_REINIT_FAILURE)
3732                         mpi3mr_issue_ioc_shutdown(mrioc);
3733         }
3734
3735         if (reason == MPI3MR_COMPLETE_CLEANUP) {
3736                 mpi3mr_free_mem(mrioc);
3737                 mpi3mr_cleanup_resources(mrioc);
3738         }
3739 }
3740
3741 /**
3742  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
3743  * @mrioc: Adapter instance reference
3744  * @cmdptr: Internal command tracker
3745  *
3746  * Complete an internal driver commands with state indicating it
3747  * is completed due to reset.
3748  *
3749  * Return: Nothing.
3750  */
3751 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
3752         struct mpi3mr_drv_cmd *cmdptr)
3753 {
3754         if (cmdptr->state & MPI3MR_CMD_PENDING) {
3755                 cmdptr->state |= MPI3MR_CMD_RESET;
3756                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
3757                 if (cmdptr->is_waiting) {
3758                         complete(&cmdptr->done);
3759                         cmdptr->is_waiting = 0;
3760                 } else if (cmdptr->callback)
3761                         cmdptr->callback(mrioc, cmdptr);
3762         }
3763 }
3764
3765 /**
3766  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
3767  * @mrioc: Adapter instance reference
3768  *
3769  * Flush all internal driver commands post reset
3770  *
3771  * Return: Nothing.
3772  */
3773 static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
3774 {
3775         struct mpi3mr_drv_cmd *cmdptr;
3776         u8 i;
3777
3778         cmdptr = &mrioc->init_cmds;
3779         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3780         cmdptr = &mrioc->host_tm_cmds;
3781         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3782
3783         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3784                 cmdptr = &mrioc->dev_rmhs_cmds[i];
3785                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3786         }
3787 }
3788
3789 /**
3790  * mpi3mr_diagfault_reset_handler - Diag fault reset handler
3791  * @mrioc: Adapter instance reference
3792  * @reset_reason: Reset reason code
3793  *
3794  * This is an handler for issuing diag fault reset from the
3795  * applications through IOCTL path to stop the execution of the
3796  * controller
3797  *
3798  * Return: 0 on success, non-zero on failure.
3799  */
3800 int mpi3mr_diagfault_reset_handler(struct mpi3mr_ioc *mrioc,
3801         u32 reset_reason)
3802 {
3803         int retval = 0;
3804
3805         ioc_info(mrioc, "Entry: reason code: %s\n",
3806             mpi3mr_reset_rc_name(reset_reason));
3807         mrioc->reset_in_progress = 1;
3808
3809         mpi3mr_ioc_disable_intr(mrioc);
3810
3811         retval = mpi3mr_issue_reset(mrioc,
3812             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3813
3814         if (retval) {
3815                 ioc_err(mrioc, "The diag fault reset failed: reason %d\n",
3816                     reset_reason);
3817                 mpi3mr_ioc_enable_intr(mrioc);
3818         }
3819         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3820         mrioc->reset_in_progress = 0;
3821         return retval;
3822 }
3823
3824 /**
3825  * mpi3mr_soft_reset_handler - Reset the controller
3826  * @mrioc: Adapter instance reference
3827  * @reset_reason: Reset reason code
3828  * @snapdump: Flag to generate snapdump in firmware or not
3829  *
3830  * This is an handler for recovering controller by issuing soft
3831  * reset are diag fault reset.  This is a blocking function and
3832  * when one reset is executed if any other resets they will be
3833  * blocked. All IOCTLs/IO will be blocked during the reset. If
3834  * controller reset is successful then the controller will be
3835  * reinitalized, otherwise the controller will be marked as not
3836  * recoverable
3837  *
3838  * In snapdump bit is set, the controller is issued with diag
3839  * fault reset so that the firmware can create a snap dump and
3840  * post that the firmware will result in F000 fault and the
3841  * driver will issue soft reset to recover from that.
3842  *
3843  * Return: 0 on success, non-zero on failure.
3844  */
3845 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
3846         u32 reset_reason, u8 snapdump)
3847 {
3848         int retval = 0, i;
3849         unsigned long flags;
3850         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
3851
3852         if (mrioc->fault_dbg) {
3853                 if (snapdump)
3854                         mpi3mr_set_diagsave(mrioc);
3855                 mpi3mr_kill_ioc(mrioc, reset_reason);
3856         }
3857
3858         /*
3859          * Block new resets until the currently executing one is finished and
3860          * return the status of the existing reset for all blocked resets
3861          */
3862         if (!mutex_trylock(&mrioc->reset_mutex)) {
3863                 ioc_info(mrioc, "Another reset in progress\n");
3864                 return -1;
3865         }
3866         mrioc->reset_in_progress = 1;
3867
3868         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
3869             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
3870                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3871                         mrioc->event_masks[i] = -1;
3872
3873                 retval = mpi3mr_issue_event_notification(mrioc);
3874
3875                 if (retval) {
3876                         ioc_err(mrioc,
3877                             "Failed to turn off events prior to reset %d\n",
3878                             retval);
3879                 }
3880         }
3881
3882         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
3883
3884         mpi3mr_ioc_disable_intr(mrioc);
3885
3886         if (snapdump) {
3887                 mpi3mr_set_diagsave(mrioc);
3888                 retval = mpi3mr_issue_reset(mrioc,
3889                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3890                 if (!retval) {
3891                         do {
3892                                 host_diagnostic =
3893                                     readl(&mrioc->sysif_regs->host_diagnostic);
3894                                 if (!(host_diagnostic &
3895                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
3896                                         break;
3897                                 msleep(100);
3898                         } while (--timeout);
3899                 }
3900         }
3901
3902         retval = mpi3mr_issue_reset(mrioc,
3903             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
3904         if (retval) {
3905                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
3906                 goto out;
3907         }
3908
3909         mpi3mr_flush_delayed_rmhs_list(mrioc);
3910         mpi3mr_flush_drv_cmds(mrioc);
3911         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3912         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3913         mpi3mr_cleanup_fwevt_list(mrioc);
3914         mpi3mr_flush_host_io(mrioc);
3915         mpi3mr_invalidate_devhandles(mrioc);
3916         mpi3mr_memset_buffers(mrioc);
3917         retval = mpi3mr_init_ioc(mrioc, MPI3MR_IT_RESET);
3918         if (retval) {
3919                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
3920                     mrioc->name, reset_reason);
3921                 goto out;
3922         }
3923         ssleep(10);
3924
3925 out:
3926         if (!retval) {
3927                 mrioc->reset_in_progress = 0;
3928                 scsi_unblock_requests(mrioc->shost);
3929                 mpi3mr_rfresh_tgtdevs(mrioc);
3930                 mrioc->ts_update_counter = 0;
3931                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
3932                 if (mrioc->watchdog_work_q)
3933                         queue_delayed_work(mrioc->watchdog_work_q,
3934                             &mrioc->watchdog_work,
3935                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
3936                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
3937         } else {
3938                 mpi3mr_issue_reset(mrioc,
3939                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3940                 mrioc->unrecoverable = 1;
3941                 mrioc->reset_in_progress = 0;
3942                 retval = -1;
3943         }
3944
3945         mutex_unlock(&mrioc->reset_mutex);
3946         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3947         return retval;
3948 }