drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2022 Broadcom Inc.
   6  *  (mailto: [email protected])
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 static int
  14 mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type, u32 reset_reason);
  15 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc);
  16 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
  17         struct mpi3_ioc_facts_data *facts_data);
  18 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
  19         struct mpi3mr_drv_cmd *drv_cmd);
  20
  21 static int poll_queues;
  22 module_param(poll_queues, int, 0444);
  23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
  24
  25 #if defined(writeq) && defined(CONFIG_64BIT)
  26 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  27 {
  28         writeq(b, addr);
  29 }
  30 #else
  31 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  32 {
  33         __u64 data_out = b;
  34
  35         writel((u32)(data_out), addr);
  36         writel((u32)(data_out >> 32), (addr + 4));
  37 }
  38 #endif
  39
  40 static inline bool
  41 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  42 {
  43         u16 pi, ci, max_entries;
  44         bool is_qfull = false;
  45
  46         pi = op_req_q->pi;
  47         ci = READ_ONCE(op_req_q->ci);
  48         max_entries = op_req_q->num_requests;
  49
  50         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  51                 is_qfull = true;
  52
  53         return is_qfull;
  54 }
  55
  56 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  57 {
  58         u16 i, max_vectors;
  59
  60         max_vectors = mrioc->intr_info_count;
  61
  62         for (i = 0; i < max_vectors; i++)
  63                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  64 }
  65
  66 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  67 {
  68         mrioc->intr_enabled = 0;
  69         mpi3mr_sync_irqs(mrioc);
  70 }
  71
  72 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  73 {
  74         mrioc->intr_enabled = 1;
  75 }
  76
  77 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  78 {
  79         u16 i;
  80
  81         mpi3mr_ioc_disable_intr(mrioc);
  82
  83         if (!mrioc->intr_info)
  84                 return;
  85
  86         for (i = 0; i < mrioc->intr_info_count; i++)
  87                 free_irq(pci_irq_vector(mrioc->pdev, i),
  88                     (mrioc->intr_info + i));
  89
  90         kfree(mrioc->intr_info);
  91         mrioc->intr_info = NULL;
  92         mrioc->intr_info_count = 0;
  93         mrioc->is_intr_info_set = false;
  94         pci_free_irq_vectors(mrioc->pdev);
  95 }
  96
  97 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  98         dma_addr_t dma_addr)
  99 {
 100         struct mpi3_sge_common *sgel = paddr;
 101
 102         sgel->flags = flags;
 103         sgel->length = cpu_to_le32(length);
 104         sgel->address = cpu_to_le64(dma_addr);
 105 }
 106
 107 void mpi3mr_build_zero_len_sge(void *paddr)
 108 {
 109         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
 110
 111         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
 112 }
 113
 114 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         if ((phys_addr < mrioc->reply_buf_dma) ||
 121             (phys_addr > mrioc->reply_buf_dma_max_address))
 122                 return NULL;
 123
 124         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 125 }
 126
 127 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 128         dma_addr_t phys_addr)
 129 {
 130         if (!phys_addr)
 131                 return NULL;
 132
 133         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 134 }
 135
 136 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 137         u64 reply_dma)
 138 {
 139         u32 old_idx = 0;
 140         unsigned long flags;
 141
 142         spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
 143         old_idx  =  mrioc->reply_free_queue_host_index;
 144         mrioc->reply_free_queue_host_index = (
 145             (mrioc->reply_free_queue_host_index ==
 146             (mrioc->reply_free_qsz - 1)) ? 0 :
 147             (mrioc->reply_free_queue_host_index + 1));
 148         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 149         writel(mrioc->reply_free_queue_host_index,
 150             &mrioc->sysif_regs->reply_free_host_index);
 151         spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
 152 }
 153
 154 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 155         u64 sense_buf_dma)
 156 {
 157         u32 old_idx = 0;
 158         unsigned long flags;
 159
 160         spin_lock_irqsave(&mrioc->sbq_lock, flags);
 161         old_idx  =  mrioc->sbq_host_index;
 162         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 163             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 164             (mrioc->sbq_host_index + 1));
 165         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 166         writel(mrioc->sbq_host_index,
 167             &mrioc->sysif_regs->sense_buffer_free_host_index);
 168         spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
 169 }
 170
 171 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 172         struct mpi3_event_notification_reply *event_reply)
 173 {
 174         char *desc = NULL;
 175         u16 event;
 176
 177         event = event_reply->event;
 178
 179         switch (event) {
 180         case MPI3_EVENT_LOG_DATA:
 181                 desc = "Log Data";
 182                 break;
 183         case MPI3_EVENT_CHANGE:
 184                 desc = "Event Change";
 185                 break;
 186         case MPI3_EVENT_GPIO_INTERRUPT:
 187                 desc = "GPIO Interrupt";
 188                 break;
 189         case MPI3_EVENT_CABLE_MGMT:
 190                 desc = "Cable Management";
 191                 break;
 192         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 193                 desc = "Energy Pack Change";
 194                 break;
 195         case MPI3_EVENT_DEVICE_ADDED:
 196         {
 197                 struct mpi3_device_page0 *event_data =
 198                     (struct mpi3_device_page0 *)event_reply->event_data;
 199                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 200                     event_data->dev_handle, event_data->device_form);
 201                 return;
 202         }
 203         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 204         {
 205                 struct mpi3_device_page0 *event_data =
 206                     (struct mpi3_device_page0 *)event_reply->event_data;
 207                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 208                     event_data->dev_handle, event_data->device_form);
 209                 return;
 210         }
 211         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 212         {
 213                 struct mpi3_event_data_device_status_change *event_data =
 214                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 215                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 216                     event_data->dev_handle, event_data->reason_code);
 217                 return;
 218         }
 219         case MPI3_EVENT_SAS_DISCOVERY:
 220         {
 221                 struct mpi3_event_data_sas_discovery *event_data =
 222                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 223                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 224                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 225                     "start" : "stop",
 226                     le32_to_cpu(event_data->discovery_status));
 227                 return;
 228         }
 229         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 230                 desc = "SAS Broadcast Primitive";
 231                 break;
 232         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 233                 desc = "SAS Notify Primitive";
 234                 break;
 235         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 236                 desc = "SAS Init Device Status Change";
 237                 break;
 238         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 239                 desc = "SAS Init Table Overflow";
 240                 break;
 241         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 242                 desc = "SAS Topology Change List";
 243                 break;
 244         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 245                 desc = "Enclosure Device Status Change";
 246                 break;
 247         case MPI3_EVENT_ENCL_DEVICE_ADDED:
 248                 desc = "Enclosure Added";
 249                 break;
 250         case MPI3_EVENT_HARD_RESET_RECEIVED:
 251                 desc = "Hard Reset Received";
 252                 break;
 253         case MPI3_EVENT_SAS_PHY_COUNTER:
 254                 desc = "SAS PHY Counter";
 255                 break;
 256         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 257                 desc = "SAS Device Discovery Error";
 258                 break;
 259         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 260                 desc = "PCIE Topology Change List";
 261                 break;
 262         case MPI3_EVENT_PCIE_ENUMERATION:
 263         {
 264                 struct mpi3_event_data_pcie_enumeration *event_data =
 265                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 266                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 267                     (event_data->reason_code ==
 268                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 269                 if (event_data->enumeration_status)
 270                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 271                             le32_to_cpu(event_data->enumeration_status));
 272                 return;
 273         }
 274         case MPI3_EVENT_PREPARE_FOR_RESET:
 275                 desc = "Prepare For Reset";
 276                 break;
 277         }
 278
 279         if (!desc)
 280                 return;
 281
 282         ioc_info(mrioc, "%s\n", desc);
 283 }
 284
 285 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 286         struct mpi3_default_reply *def_reply)
 287 {
 288         struct mpi3_event_notification_reply *event_reply =
 289             (struct mpi3_event_notification_reply *)def_reply;
 290
 291         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 292         mpi3mr_print_event_data(mrioc, event_reply);
 293         mpi3mr_os_handle_events(mrioc, event_reply);
 294 }
 295
 296 static struct mpi3mr_drv_cmd *
 297 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 298         struct mpi3_default_reply *def_reply)
 299 {
 300         u16 idx;
 301
 302         switch (host_tag) {
 303         case MPI3MR_HOSTTAG_INITCMDS:
 304                 return &mrioc->init_cmds;
 305         case MPI3MR_HOSTTAG_CFG_CMDS:
 306                 return &mrioc->cfg_cmds;
 307         case MPI3MR_HOSTTAG_BSG_CMDS:
 308                 return &mrioc->bsg_cmds;
 309         case MPI3MR_HOSTTAG_BLK_TMS:
 310                 return &mrioc->host_tm_cmds;
 311         case MPI3MR_HOSTTAG_PEL_ABORT:
 312                 return &mrioc->pel_abort_cmd;
 313         case MPI3MR_HOSTTAG_PEL_WAIT:
 314                 return &mrioc->pel_cmds;
 315         case MPI3MR_HOSTTAG_TRANSPORT_CMDS:
 316                 return &mrioc->transport_cmds;
 317         case MPI3MR_HOSTTAG_INVALID:
 318                 if (def_reply && def_reply->function ==
 319                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 320                         mpi3mr_handle_events(mrioc, def_reply);
 321                 return NULL;
 322         default:
 323                 break;
 324         }
 325         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 326             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 327                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 328                 return &mrioc->dev_rmhs_cmds[idx];
 329         }
 330
 331         if (host_tag >= MPI3MR_HOSTTAG_EVTACKCMD_MIN &&
 332             host_tag <= MPI3MR_HOSTTAG_EVTACKCMD_MAX) {
 333                 idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
 334                 return &mrioc->evtack_cmds[idx];
 335         }
 336
 337         return NULL;
 338 }
 339
 340 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 341         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 342 {
 343         u16 reply_desc_type, host_tag = 0;
 344         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 345         u32 ioc_loginfo = 0;
 346         struct mpi3_status_reply_descriptor *status_desc;
 347         struct mpi3_address_reply_descriptor *addr_desc;
 348         struct mpi3_success_reply_descriptor *success_desc;
 349         struct mpi3_default_reply *def_reply = NULL;
 350         struct mpi3mr_drv_cmd *cmdptr = NULL;
 351         struct mpi3_scsi_io_reply *scsi_reply;
 352         u8 *sense_buf = NULL;
 353
 354         *reply_dma = 0;
 355         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 356             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 357         switch (reply_desc_type) {
 358         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 359                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 360                 host_tag = le16_to_cpu(status_desc->host_tag);
 361                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 362                 if (ioc_status &
 363                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 364                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 365                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 366                 break;
 367         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 368                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 369                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 370                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 371                 if (!def_reply)
 372                         goto out;
 373                 host_tag = le16_to_cpu(def_reply->host_tag);
 374                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 375                 if (ioc_status &
 376                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 377                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 378                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 379                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 380                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 381                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 382                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 383                 }
 384                 break;
 385         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 386                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 387                 host_tag = le16_to_cpu(success_desc->host_tag);
 388                 break;
 389         default:
 390                 break;
 391         }
 392
 393         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 394         if (cmdptr) {
 395                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 396                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 397                         cmdptr->ioc_loginfo = ioc_loginfo;
 398                         cmdptr->ioc_status = ioc_status;
 399                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 400                         if (def_reply) {
 401                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 402                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 403                                     mrioc->reply_sz);
 404                         }
 405                         if (cmdptr->is_waiting) {
 406                                 complete(&cmdptr->done);
 407                                 cmdptr->is_waiting = 0;
 408                         } else if (cmdptr->callback)
 409                                 cmdptr->callback(mrioc, cmdptr);
 410                 }
 411         }
 412 out:
 413         if (sense_buf)
 414                 mpi3mr_repost_sense_buf(mrioc,
 415                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 416 }
 417
 418 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 419 {
 420         u32 exp_phase = mrioc->admin_reply_ephase;
 421         u32 admin_reply_ci = mrioc->admin_reply_ci;
 422         u32 num_admin_replies = 0;
 423         u64 reply_dma = 0;
 424         struct mpi3_default_reply_descriptor *reply_desc;
 425
 426         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 427             admin_reply_ci;
 428
 429         if ((le16_to_cpu(reply_desc->reply_flags) &
 430             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 431                 return 0;
 432
 433         do {
 434                 if (mrioc->unrecoverable)
 435                         break;
 436
 437                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 438                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 439                 if (reply_dma)
 440                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 441                 num_admin_replies++;
 442                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 443                         admin_reply_ci = 0;
 444                         exp_phase ^= 1;
 445                 }
 446                 reply_desc =
 447                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 448                     admin_reply_ci;
 449                 if ((le16_to_cpu(reply_desc->reply_flags) &
 450                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 451                         break;
 452         } while (1);
 453
 454         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 455         mrioc->admin_reply_ci = admin_reply_ci;
 456         mrioc->admin_reply_ephase = exp_phase;
 457
 458         return num_admin_replies;
 459 }
 460
 461 /**
 462  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 463  *      queue's consumer index from operational reply descriptor queue.
 464  * @op_reply_q: op_reply_qinfo object
 465  * @reply_ci: operational reply descriptor's queue consumer index
 466  *
 467  * Returns reply descriptor frame address
 468  */
 469 static inline struct mpi3_default_reply_descriptor *
 470 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 471 {
 472         void *segment_base_addr;
 473         struct segments *segments = op_reply_q->q_segments;
 474         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 475
 476         segment_base_addr =
 477             segments[reply_ci / op_reply_q->segment_qd].segment;
 478         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 479             (reply_ci % op_reply_q->segment_qd);
 480         return reply_desc;
 481 }
 482
 483 /**
 484  * mpi3mr_process_op_reply_q - Operational reply queue handler
 485  * @mrioc: Adapter instance reference
 486  * @op_reply_q: Operational reply queue info
 487  *
 488  * Checks the specific operational reply queue and drains the
 489  * reply queue entries until the queue is empty and process the
 490  * individual reply descriptors.
 491  *
 492  * Return: 0 if queue is already processed,or number of reply
 493  *          descriptors processed.
 494  */
 495 int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 496         struct op_reply_qinfo *op_reply_q)
 497 {
 498         struct op_req_qinfo *op_req_q;
 499         u32 exp_phase;
 500         u32 reply_ci;
 501         u32 num_op_reply = 0;
 502         u64 reply_dma = 0;
 503         struct mpi3_default_reply_descriptor *reply_desc;
 504         u16 req_q_idx = 0, reply_qidx;
 505
 506         reply_qidx = op_reply_q->qid - 1;
 507
 508         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 509                 return 0;
 510
 511         exp_phase = op_reply_q->ephase;
 512         reply_ci = op_reply_q->ci;
 513
 514         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 515         if ((le16_to_cpu(reply_desc->reply_flags) &
 516             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 517                 atomic_dec(&op_reply_q->in_use);
 518                 return 0;
 519         }
 520
 521         do {
 522                 if (mrioc->unrecoverable)
 523                         break;
 524
 525                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 526                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 527
 528                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 529                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 530                     reply_qidx);
 531                 atomic_dec(&op_reply_q->pend_ios);
 532                 if (reply_dma)
 533                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 534                 num_op_reply++;
 535
 536                 if (++reply_ci == op_reply_q->num_replies) {
 537                         reply_ci = 0;
 538                         exp_phase ^= 1;
 539                 }
 540
 541                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 542
 543                 if ((le16_to_cpu(reply_desc->reply_flags) &
 544                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 545                         break;
 546 #ifndef CONFIG_PREEMPT_RT
 547                 /*
 548                  * Exit completion loop to avoid CPU lockup
 549                  * Ensure remaining completion happens from threaded ISR.
 550                  */
 551                 if (num_op_reply > mrioc->max_host_ios) {
 552                         op_reply_q->enable_irq_poll = true;
 553                         break;
 554                 }
 555 #endif
 556         } while (1);
 557
 558         writel(reply_ci,
 559             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 560         op_reply_q->ci = reply_ci;
 561         op_reply_q->ephase = exp_phase;
 562
 563         atomic_dec(&op_reply_q->in_use);
 564         return num_op_reply;
 565 }
 566
 567 /**
 568  * mpi3mr_blk_mq_poll - Operational reply queue handler
 569  * @shost: SCSI Host reference
 570  * @queue_num: Request queue number (w.r.t OS it is hardware context number)
 571  *
 572  * Checks the specific operational reply queue and drains the
 573  * reply queue entries until the queue is empty and process the
 574  * individual reply descriptors.
 575  *
 576  * Return: 0 if queue is already processed,or number of reply
 577  *          descriptors processed.
 578  */
 579 int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
 580 {
 581         int num_entries = 0;
 582         struct mpi3mr_ioc *mrioc;
 583
 584         mrioc = (struct mpi3mr_ioc *)shost->hostdata;
 585
 586         if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||
 587             mrioc->unrecoverable))
 588                 return 0;
 589
 590         num_entries = mpi3mr_process_op_reply_q(mrioc,
 591                         &mrioc->op_reply_qinfo[queue_num]);
 592
 593         return num_entries;
 594 }
 595
 596 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 597 {
 598         struct mpi3mr_intr_info *intr_info = privdata;
 599         struct mpi3mr_ioc *mrioc;
 600         u16 midx;
 601         u32 num_admin_replies = 0, num_op_reply = 0;
 602
 603         if (!intr_info)
 604                 return IRQ_NONE;
 605
 606         mrioc = intr_info->mrioc;
 607
 608         if (!mrioc->intr_enabled)
 609                 return IRQ_NONE;
 610
 611         midx = intr_info->msix_index;
 612
 613         if (!midx)
 614                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 615         if (intr_info->op_reply_q)
 616                 num_op_reply = mpi3mr_process_op_reply_q(mrioc,
 617                     intr_info->op_reply_q);
 618
 619         if (num_admin_replies || num_op_reply)
 620                 return IRQ_HANDLED;
 621         else
 622                 return IRQ_NONE;
 623 }
 624
 625 #ifndef CONFIG_PREEMPT_RT
 626
 627 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 628 {
 629         struct mpi3mr_intr_info *intr_info = privdata;
 630         int ret;
 631
 632         if (!intr_info)
 633                 return IRQ_NONE;
 634
 635         /* Call primary ISR routine */
 636         ret = mpi3mr_isr_primary(irq, privdata);
 637
 638         /*
 639          * If more IOs are expected, schedule IRQ polling thread.
 640          * Otherwise exit from ISR.
 641          */
 642         if (!intr_info->op_reply_q)
 643                 return ret;
 644
 645         if (!intr_info->op_reply_q->enable_irq_poll ||
 646             !atomic_read(&intr_info->op_reply_q->pend_ios))
 647                 return ret;
 648
 649         disable_irq_nosync(intr_info->os_irq);
 650
 651         return IRQ_WAKE_THREAD;
 652 }
 653
 654 /**
 655  * mpi3mr_isr_poll - Reply queue polling routine
 656  * @irq: IRQ
 657  * @privdata: Interrupt info
 658  *
 659  * poll for pending I/O completions in a loop until pending I/Os
 660  * present or controller queue depth I/Os are processed.
 661  *
 662  * Return: IRQ_NONE or IRQ_HANDLED
 663  */
 664 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 665 {
 666         struct mpi3mr_intr_info *intr_info = privdata;
 667         struct mpi3mr_ioc *mrioc;
 668         u16 midx;
 669         u32 num_op_reply = 0;
 670
 671         if (!intr_info || !intr_info->op_reply_q)
 672                 return IRQ_NONE;
 673
 674         mrioc = intr_info->mrioc;
 675         midx = intr_info->msix_index;
 676
 677         /* Poll for pending IOs completions */
 678         do {
 679                 if (!mrioc->intr_enabled || mrioc->unrecoverable)
 680                         break;
 681
 682                 if (!midx)
 683                         mpi3mr_process_admin_reply_q(mrioc);
 684                 if (intr_info->op_reply_q)
 685                         num_op_reply +=
 686                             mpi3mr_process_op_reply_q(mrioc,
 687                                 intr_info->op_reply_q);
 688
 689                 usleep_range(MPI3MR_IRQ_POLL_SLEEP, 10 * MPI3MR_IRQ_POLL_SLEEP);
 690
 691         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 692             (num_op_reply < mrioc->max_host_ios));
 693
 694         intr_info->op_reply_q->enable_irq_poll = false;
 695         enable_irq(intr_info->os_irq);
 696
 697         return IRQ_HANDLED;
 698 }
 699
 700 #endif
 701
 702 /**
 703  * mpi3mr_request_irq - Request IRQ and register ISR
 704  * @mrioc: Adapter instance reference
 705  * @index: IRQ vector index
 706  *
 707  * Request threaded ISR with primary ISR and secondary
 708  *
 709  * Return: 0 on success and non zero on failures.
 710  */
 711 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 712 {
 713         struct pci_dev *pdev = mrioc->pdev;
 714         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 715         int retval = 0;
 716
 717         intr_info->mrioc = mrioc;
 718         intr_info->msix_index = index;
 719         intr_info->op_reply_q = NULL;
 720
 721         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 722             mrioc->driver_name, mrioc->id, index);
 723
 724 #ifndef CONFIG_PREEMPT_RT
 725         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 726             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 727 #else
 728         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr_primary,
 729             NULL, IRQF_SHARED, intr_info->name, intr_info);
 730 #endif
 731         if (retval) {
 732                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 733                     intr_info->name, pci_irq_vector(pdev, index));
 734                 return retval;
 735         }
 736
 737         intr_info->os_irq = pci_irq_vector(pdev, index);
 738         return retval;
 739 }
 740
 741 static void mpi3mr_calc_poll_queues(struct mpi3mr_ioc *mrioc, u16 max_vectors)
 742 {
 743         if (!mrioc->requested_poll_qcount)
 744                 return;
 745
 746         /* Reserved for Admin and Default Queue */
 747         if (max_vectors > 2 &&
 748                 (mrioc->requested_poll_qcount < max_vectors - 2)) {
 749                 ioc_info(mrioc,
 750                     "enabled polled queues (%d) msix (%d)\n",
 751                     mrioc->requested_poll_qcount, max_vectors);
 752         } else {
 753                 ioc_info(mrioc,
 754                     "disabled polled queues (%d) msix (%d) because of no resources for default queue\n",
 755                     mrioc->requested_poll_qcount, max_vectors);
 756                 mrioc->requested_poll_qcount = 0;
 757         }
 758 }
 759
 760 /**
 761  * mpi3mr_setup_isr - Setup ISR for the controller
 762  * @mrioc: Adapter instance reference
 763  * @setup_one: Request one IRQ or more
 764  *
 765  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 766  *
 767  * Return: 0 on success and non zero on failures.
 768  */
 769 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 770 {
 771         unsigned int irq_flags = PCI_IRQ_MSIX;
 772         int max_vectors, min_vec;
 773         int retval;
 774         int i;
 775         struct irq_affinity desc = { .pre_vectors =  1, .post_vectors = 1 };
 776
 777         if (mrioc->is_intr_info_set)
 778                 return 0;
 779
 780         mpi3mr_cleanup_isr(mrioc);
 781
 782         if (setup_one || reset_devices) {
 783                 max_vectors = 1;
 784                 retval = pci_alloc_irq_vectors(mrioc->pdev,
 785                     1, max_vectors, irq_flags);
 786                 if (retval < 0) {
 787                         ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 788                             retval);
 789                         goto out_failed;
 790                 }
 791         } else {
 792                 max_vectors =
 793                     min_t(int, mrioc->cpu_count + 1 +
 794                         mrioc->requested_poll_qcount, mrioc->msix_count);
 795
 796                 mpi3mr_calc_poll_queues(mrioc, max_vectors);
 797
 798                 ioc_info(mrioc,
 799                     "MSI-X vectors supported: %d, no of cores: %d,",
 800                     mrioc->msix_count, mrioc->cpu_count);
 801                 ioc_info(mrioc,
 802                     "MSI-x vectors requested: %d poll_queues %d\n",
 803                     max_vectors, mrioc->requested_poll_qcount);
 804
 805                 desc.post_vectors = mrioc->requested_poll_qcount;
 806                 min_vec = desc.pre_vectors + desc.post_vectors;
 807                 irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 808
 809                 retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 810                         min_vec, max_vectors, irq_flags, &desc);
 811
 812                 if (retval < 0) {
 813                         ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 814                             retval);
 815                         goto out_failed;
 816                 }
 817
 818
 819                 /*
 820                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 821                  * between Admin queue and operational queue
 822                  */
 823                 if (retval == min_vec)
 824                         mrioc->op_reply_q_offset = 0;
 825                 else if (retval != (max_vectors)) {
 826                         ioc_info(mrioc,
 827                             "allocated vectors (%d) are less than configured (%d)\n",
 828                             retval, max_vectors);
 829                 }
 830
 831                 max_vectors = retval;
 832                 mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 833
 834                 mpi3mr_calc_poll_queues(mrioc, max_vectors);
 835
 836         }
 837
 838         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 839             GFP_KERNEL);
 840         if (!mrioc->intr_info) {
 841                 retval = -ENOMEM;
 842                 pci_free_irq_vectors(mrioc->pdev);
 843                 goto out_failed;
 844         }
 845         for (i = 0; i < max_vectors; i++) {
 846                 retval = mpi3mr_request_irq(mrioc, i);
 847                 if (retval) {
 848                         mrioc->intr_info_count = i;
 849                         goto out_failed;
 850                 }
 851         }
 852         if (reset_devices || !setup_one)
 853                 mrioc->is_intr_info_set = true;
 854         mrioc->intr_info_count = max_vectors;
 855         mpi3mr_ioc_enable_intr(mrioc);
 856         return 0;
 857
 858 out_failed:
 859         mpi3mr_cleanup_isr(mrioc);
 860
 861         return retval;
 862 }
 863
 864 static const struct {
 865         enum mpi3mr_iocstate value;
 866         char *name;
 867 } mrioc_states[] = {
 868         { MRIOC_STATE_READY, "ready" },
 869         { MRIOC_STATE_FAULT, "fault" },
 870         { MRIOC_STATE_RESET, "reset" },
 871         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 872         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 873         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 874 };
 875
 876 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 877 {
 878         int i;
 879         char *name = NULL;
 880
 881         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 882                 if (mrioc_states[i].value == mrioc_state) {
 883                         name = mrioc_states[i].name;
 884                         break;
 885                 }
 886         }
 887         return name;
 888 }
 889
 890 /* Reset reason to name mapper structure*/
 891 static const struct {
 892         enum mpi3mr_reset_reason value;
 893         char *name;
 894 } mpi3mr_reset_reason_codes[] = {
 895         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 896         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 897         { MPI3MR_RESET_FROM_APP, "application invocation" },
 898         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 899         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 900         { MPI3MR_RESET_FROM_APP_TIMEOUT, "application command timeout" },
 901         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 902         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 903         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 904         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 905         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 906         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 907         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 908         {
 909                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 910                 "create request queue timeout"
 911         },
 912         {
 913                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 914                 "create reply queue timeout"
 915         },
 916         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 917         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 918         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 919         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 920         {
 921                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 922                 "component image activation timeout"
 923         },
 924         {
 925                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 926                 "get package version timeout"
 927         },
 928         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 929         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 930         { MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" },
 931         { MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"},
 932         { MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" },
 933 };
 934
 935 /**
 936  * mpi3mr_reset_rc_name - get reset reason code name
 937  * @reason_code: reset reason code value
 938  *
 939  * Map reset reason to an NULL terminated ASCII string
 940  *
 941  * Return: name corresponding to reset reason value or NULL.
 942  */
 943 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 944 {
 945         int i;
 946         char *name = NULL;
 947
 948         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 949                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 950                         name = mpi3mr_reset_reason_codes[i].name;
 951                         break;
 952                 }
 953         }
 954         return name;
 955 }
 956
 957 /* Reset type to name mapper structure*/
 958 static const struct {
 959         u16 reset_type;
 960         char *name;
 961 } mpi3mr_reset_types[] = {
 962         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 963         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 964 };
 965
 966 /**
 967  * mpi3mr_reset_type_name - get reset type name
 968  * @reset_type: reset type value
 969  *
 970  * Map reset type to an NULL terminated ASCII string
 971  *
 972  * Return: name corresponding to reset type value or NULL.
 973  */
 974 static const char *mpi3mr_reset_type_name(u16 reset_type)
 975 {
 976         int i;
 977         char *name = NULL;
 978
 979         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 980                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 981                         name = mpi3mr_reset_types[i].name;
 982                         break;
 983                 }
 984         }
 985         return name;
 986 }
 987
 988 /**
 989  * mpi3mr_print_fault_info - Display fault information
 990  * @mrioc: Adapter instance reference
 991  *
 992  * Display the controller fault information if there is a
 993  * controller fault.
 994  *
 995  * Return: Nothing.
 996  */
 997 void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 998 {
 999         u32 ioc_status, code, code1, code2, code3;
1000
1001         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1002
1003         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1004                 code = readl(&mrioc->sysif_regs->fault);
1005                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
1006                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
1007                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
1008
1009                 ioc_info(mrioc,
1010                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
1011                     code, code1, code2, code3);
1012         }
1013 }
1014
1015 /**
1016  * mpi3mr_get_iocstate - Get IOC State
1017  * @mrioc: Adapter instance reference
1018  *
1019  * Return a proper IOC state enum based on the IOC status and
1020  * IOC configuration and unrcoverable state of the controller.
1021  *
1022  * Return: Current IOC state.
1023  */
1024 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
1025 {
1026         u32 ioc_status, ioc_config;
1027         u8 ready, enabled;
1028
1029         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1030         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1031
1032         if (mrioc->unrecoverable)
1033                 return MRIOC_STATE_UNRECOVERABLE;
1034         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
1035                 return MRIOC_STATE_FAULT;
1036
1037         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
1038         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
1039
1040         if (ready && enabled)
1041                 return MRIOC_STATE_READY;
1042         if ((!ready) && (!enabled))
1043                 return MRIOC_STATE_RESET;
1044         if ((!ready) && (enabled))
1045                 return MRIOC_STATE_BECOMING_READY;
1046
1047         return MRIOC_STATE_RESET_REQUESTED;
1048 }
1049
1050 /**
1051  * mpi3mr_clear_reset_history - clear reset history
1052  * @mrioc: Adapter instance reference
1053  *
1054  * Write the reset history bit in IOC status to clear the bit,
1055  * if it is already set.
1056  *
1057  * Return: Nothing.
1058  */
1059 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
1060 {
1061         u32 ioc_status;
1062
1063         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1064         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1065                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
1066 }
1067
1068 /**
1069  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1070  * @mrioc: Adapter instance reference
1071  * @reset_reason: Reset reason code
1072  *
1073  * Issue Message unit Reset to the controller and wait for it to
1074  * be complete.
1075  *
1076  * Return: 0 on success, -1 on failure.
1077  */
1078 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
1079         u32 reset_reason)
1080 {
1081         u32 ioc_config, timeout, ioc_status;
1082         int retval = -1;
1083
1084         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
1085         if (mrioc->unrecoverable) {
1086                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
1087                 return retval;
1088         }
1089         mpi3mr_clear_reset_history(mrioc);
1090         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1091         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1092         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1093         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1094
1095         timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1096         do {
1097                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1098                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
1099                         mpi3mr_clear_reset_history(mrioc);
1100                         break;
1101                 }
1102                 if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1103                         mpi3mr_print_fault_info(mrioc);
1104                         break;
1105                 }
1106                 msleep(100);
1107         } while (--timeout);
1108
1109         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1110         if (timeout && !((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1111               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1112               (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1113                 retval = 0;
1114
1115         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
1116             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
1117         return retval;
1118 }
1119
1120 /**
1121  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1122  * during reset/resume
1123  * @mrioc: Adapter instance reference
1124  *
1125  * Return zero if the new IOCFacts parameters value is compatible with
1126  * older values else return -EPERM
1127  */
1128 static int
1129 mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc)
1130 {
1131         void *removepend_bitmap;
1132
1133         if (mrioc->facts.reply_sz > mrioc->reply_sz) {
1134                 ioc_err(mrioc,
1135                     "cannot increase reply size from %d to %d\n",
1136                     mrioc->reply_sz, mrioc->facts.reply_sz);
1137                 return -EPERM;
1138         }
1139
1140         if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {
1141                 ioc_err(mrioc,
1142                     "cannot reduce number of operational reply queues from %d to %d\n",
1143                     mrioc->num_op_reply_q,
1144                     mrioc->facts.max_op_reply_q);
1145                 return -EPERM;
1146         }
1147
1148         if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {
1149                 ioc_err(mrioc,
1150                     "cannot reduce number of operational request queues from %d to %d\n",
1151                     mrioc->num_op_req_q, mrioc->facts.max_op_req_q);
1152                 return -EPERM;
1153         }
1154
1155         if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &
1156             MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED))
1157                 ioc_err(mrioc,
1158                     "critical error: multipath capability is enabled at the\n"
1159                     "\tcontroller while sas transport support is enabled at the\n"
1160                     "\tdriver, please reboot the system or reload the driver\n");
1161
1162         if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {
1163                 removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,
1164                                                   GFP_KERNEL);
1165                 if (!removepend_bitmap) {
1166                         ioc_err(mrioc,
1167                                 "failed to increase removepend_bitmap bits from %d to %d\n",
1168                                 mrioc->dev_handle_bitmap_bits,
1169                                 mrioc->facts.max_devhandle);
1170                         return -EPERM;
1171                 }
1172                 bitmap_free(mrioc->removepend_bitmap);
1173                 mrioc->removepend_bitmap = removepend_bitmap;
1174                 ioc_info(mrioc,
1175                          "increased bits of dev_handle_bitmap from %d to %d\n",
1176                          mrioc->dev_handle_bitmap_bits,
1177                          mrioc->facts.max_devhandle);
1178                 mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
1179         }
1180
1181         return 0;
1182 }
1183
1184 /**
1185  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1186  * @mrioc: Adapter instance reference
1187  *
1188  * Set Enable IOC bit in IOC configuration register and wait for
1189  * the controller to become ready.
1190  *
1191  * Return: 0 on success, appropriate error on failure.
1192  */
1193 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
1194 {
1195         u32 ioc_config, ioc_status, timeout;
1196         int retval = 0;
1197         enum mpi3mr_iocstate ioc_state;
1198         u64 base_info;
1199
1200         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1201         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1202         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
1203         ioc_info(mrioc, "ioc_status(0x%08x), ioc_config(0x%08x), ioc_info(0x%016llx) at the bringup\n",
1204             ioc_status, ioc_config, base_info);
1205
1206         /*The timeout value is in 2sec unit, changing it to seconds*/
1207         mrioc->ready_timeout =
1208             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
1209             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
1210
1211         ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);
1212
1213         ioc_state = mpi3mr_get_iocstate(mrioc);
1214         ioc_info(mrioc, "controller is in %s state during detection\n",
1215             mpi3mr_iocstate_name(ioc_state));
1216
1217         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
1218             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
1219                 timeout = mrioc->ready_timeout * 10;
1220                 do {
1221                         msleep(100);
1222                 } while (--timeout);
1223
1224                 if (!pci_device_is_present(mrioc->pdev)) {
1225                         mrioc->unrecoverable = 1;
1226                         ioc_err(mrioc,
1227                             "controller is not present while waiting to reset\n");
1228                         retval = -1;
1229                         goto out_device_not_present;
1230                 }
1231
1232                 ioc_state = mpi3mr_get_iocstate(mrioc);
1233                 ioc_info(mrioc,
1234                     "controller is in %s state after waiting to reset\n",
1235                     mpi3mr_iocstate_name(ioc_state));
1236         }
1237
1238         if (ioc_state == MRIOC_STATE_READY) {
1239                 ioc_info(mrioc, "issuing message unit reset (MUR) to bring to reset state\n");
1240                 retval = mpi3mr_issue_and_process_mur(mrioc,
1241                     MPI3MR_RESET_FROM_BRINGUP);
1242                 ioc_state = mpi3mr_get_iocstate(mrioc);
1243                 if (retval)
1244                         ioc_err(mrioc,
1245                             "message unit reset failed with error %d current state %s\n",
1246                             retval, mpi3mr_iocstate_name(ioc_state));
1247         }
1248         if (ioc_state != MRIOC_STATE_RESET) {
1249                 mpi3mr_print_fault_info(mrioc);
1250                 ioc_info(mrioc, "issuing soft reset to bring to reset state\n");
1251                 retval = mpi3mr_issue_reset(mrioc,
1252                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
1253                     MPI3MR_RESET_FROM_BRINGUP);
1254                 if (retval) {
1255                         ioc_err(mrioc,
1256                             "soft reset failed with error %d\n", retval);
1257                         goto out_failed;
1258                 }
1259         }
1260         ioc_state = mpi3mr_get_iocstate(mrioc);
1261         if (ioc_state != MRIOC_STATE_RESET) {
1262                 ioc_err(mrioc,
1263                     "cannot bring controller to reset state, current state: %s\n",
1264                     mpi3mr_iocstate_name(ioc_state));
1265                 goto out_failed;
1266         }
1267         mpi3mr_clear_reset_history(mrioc);
1268         retval = mpi3mr_setup_admin_qpair(mrioc);
1269         if (retval) {
1270                 ioc_err(mrioc, "failed to setup admin queues: error %d\n",
1271                     retval);
1272                 goto out_failed;
1273         }
1274
1275         ioc_info(mrioc, "bringing controller to ready state\n");
1276         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1277         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1278         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1279
1280         timeout = mrioc->ready_timeout * 10;
1281         do {
1282                 ioc_state = mpi3mr_get_iocstate(mrioc);
1283                 if (ioc_state == MRIOC_STATE_READY) {
1284                         ioc_info(mrioc,
1285                             "successfully transitioned to %s state\n",
1286                             mpi3mr_iocstate_name(ioc_state));
1287                         return 0;
1288                 }
1289                 if (!pci_device_is_present(mrioc->pdev)) {
1290                         mrioc->unrecoverable = 1;
1291                         ioc_err(mrioc,
1292                             "controller is not present at the bringup\n");
1293                         retval = -1;
1294                         goto out_device_not_present;
1295                 }
1296                 msleep(100);
1297         } while (--timeout);
1298
1299 out_failed:
1300         ioc_state = mpi3mr_get_iocstate(mrioc);
1301         ioc_err(mrioc,
1302             "failed to bring to ready state,  current state: %s\n",
1303             mpi3mr_iocstate_name(ioc_state));
1304 out_device_not_present:
1305         return retval;
1306 }
1307
1308 /**
1309  * mpi3mr_soft_reset_success - Check softreset is success or not
1310  * @ioc_status: IOC status register value
1311  * @ioc_config: IOC config register value
1312  *
1313  * Check whether the soft reset is successful or not based on
1314  * IOC status and IOC config register values.
1315  *
1316  * Return: True when the soft reset is success, false otherwise.
1317  */
1318 static inline bool
1319 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1320 {
1321         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1322             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1323                 return true;
1324         return false;
1325 }
1326
1327 /**
1328  * mpi3mr_diagfault_success - Check diag fault is success or not
1329  * @mrioc: Adapter reference
1330  * @ioc_status: IOC status register value
1331  *
1332  * Check whether the controller hit diag reset fault code.
1333  *
1334  * Return: True when there is diag fault, false otherwise.
1335  */
1336 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1337         u32 ioc_status)
1338 {
1339         u32 fault;
1340
1341         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1342                 return false;
1343         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1344         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) {
1345                 mpi3mr_print_fault_info(mrioc);
1346                 return true;
1347         }
1348         return false;
1349 }
1350
1351 /**
1352  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1353  * @mrioc: Adapter reference
1354  *
1355  * Set diag save bit in IOC configuration register to enable
1356  * snapdump.
1357  *
1358  * Return: Nothing.
1359  */
1360 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1361 {
1362         u32 ioc_config;
1363
1364         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1365         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1366         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1367 }
1368
1369 /**
1370  * mpi3mr_issue_reset - Issue reset to the controller
1371  * @mrioc: Adapter reference
1372  * @reset_type: Reset type
1373  * @reset_reason: Reset reason code
1374  *
1375  * Unlock the host diagnostic registers and write the specific
1376  * reset type to that, wait for reset acknowledgment from the
1377  * controller, if the reset is not successful retry for the
1378  * predefined number of times.
1379  *
1380  * Return: 0 on success, non-zero on failure.
1381  */
1382 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1383         u32 reset_reason)
1384 {
1385         int retval = -1;
1386         u8 unlock_retry_count = 0;
1387         u32 host_diagnostic, ioc_status, ioc_config;
1388         u32 timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1389
1390         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1391             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1392                 return retval;
1393         if (mrioc->unrecoverable)
1394                 return retval;
1395         if (reset_reason == MPI3MR_RESET_FROM_FIRMWARE) {
1396                 retval = 0;
1397                 return retval;
1398         }
1399
1400         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1401             mpi3mr_reset_type_name(reset_type),
1402             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1403
1404         mpi3mr_clear_reset_history(mrioc);
1405         do {
1406                 ioc_info(mrioc,
1407                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1408                     ++unlock_retry_count);
1409                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1410                         ioc_err(mrioc,
1411                             "%s reset failed due to unlock failure, host_diagnostic(0x%08x)\n",
1412                             mpi3mr_reset_type_name(reset_type),
1413                             host_diagnostic);
1414                         mrioc->unrecoverable = 1;
1415                         return retval;
1416                 }
1417
1418                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1419                     &mrioc->sysif_regs->write_sequence);
1420                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1421                     &mrioc->sysif_regs->write_sequence);
1422                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1423                     &mrioc->sysif_regs->write_sequence);
1424                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1425                     &mrioc->sysif_regs->write_sequence);
1426                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1427                     &mrioc->sysif_regs->write_sequence);
1428                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1429                     &mrioc->sysif_regs->write_sequence);
1430                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1431                     &mrioc->sysif_regs->write_sequence);
1432                 usleep_range(1000, 1100);
1433                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1434                 ioc_info(mrioc,
1435                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1436                     unlock_retry_count, host_diagnostic);
1437         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1438
1439         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1440         writel(host_diagnostic | reset_type,
1441             &mrioc->sysif_regs->host_diagnostic);
1442         switch (reset_type) {
1443         case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET:
1444                 do {
1445                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1446                         ioc_config =
1447                             readl(&mrioc->sysif_regs->ioc_configuration);
1448                         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1449                             && mpi3mr_soft_reset_success(ioc_status, ioc_config)
1450                             ) {
1451                                 mpi3mr_clear_reset_history(mrioc);
1452                                 retval = 0;
1453                                 break;
1454                         }
1455                         msleep(100);
1456                 } while (--timeout);
1457                 mpi3mr_print_fault_info(mrioc);
1458                 break;
1459         case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT:
1460                 do {
1461                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1462                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1463                                 retval = 0;
1464                                 break;
1465                         }
1466                         msleep(100);
1467                 } while (--timeout);
1468                 break;
1469         default:
1470                 break;
1471         }
1472
1473         writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1474             &mrioc->sysif_regs->write_sequence);
1475
1476         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1477         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1478         ioc_info(mrioc,
1479             "ioc_status/ioc_onfig after %s reset is (0x%x)/(0x%x)\n",
1480             (!retval)?"successful":"failed", ioc_status,
1481             ioc_config);
1482         if (retval)
1483                 mrioc->unrecoverable = 1;
1484         return retval;
1485 }
1486
1487 /**
1488  * mpi3mr_admin_request_post - Post request to admin queue
1489  * @mrioc: Adapter reference
1490  * @admin_req: MPI3 request
1491  * @admin_req_sz: Request size
1492  * @ignore_reset: Ignore reset in process
1493  *
1494  * Post the MPI3 request into admin request queue and
1495  * inform the controller, if the queue is full return
1496  * appropriate error.
1497  *
1498  * Return: 0 on success, non-zero on failure.
1499  */
1500 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1501         u16 admin_req_sz, u8 ignore_reset)
1502 {
1503         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1504         int retval = 0;
1505         unsigned long flags;
1506         u8 *areq_entry;
1507
1508         if (mrioc->unrecoverable) {
1509                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1510                 return -EFAULT;
1511         }
1512
1513         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1514         areq_pi = mrioc->admin_req_pi;
1515         areq_ci = mrioc->admin_req_ci;
1516         max_entries = mrioc->num_admin_req;
1517         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1518             (areq_pi == (max_entries - 1)))) {
1519                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1520                 retval = -EAGAIN;
1521                 goto out;
1522         }
1523         if (!ignore_reset && mrioc->reset_in_progress) {
1524                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1525                 retval = -EAGAIN;
1526                 goto out;
1527         }
1528         areq_entry = (u8 *)mrioc->admin_req_base +
1529             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1530         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1531         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1532
1533         if (++areq_pi == max_entries)
1534                 areq_pi = 0;
1535         mrioc->admin_req_pi = areq_pi;
1536
1537         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1538
1539 out:
1540         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1541
1542         return retval;
1543 }
1544
1545 /**
1546  * mpi3mr_free_op_req_q_segments - free request memory segments
1547  * @mrioc: Adapter instance reference
1548  * @q_idx: operational request queue index
1549  *
1550  * Free memory segments allocated for operational request queue
1551  *
1552  * Return: Nothing.
1553  */
1554 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1555 {
1556         u16 j;
1557         int size;
1558         struct segments *segments;
1559
1560         segments = mrioc->req_qinfo[q_idx].q_segments;
1561         if (!segments)
1562                 return;
1563
1564         if (mrioc->enable_segqueue) {
1565                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1566                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1567                         dma_free_coherent(&mrioc->pdev->dev,
1568                             MPI3MR_MAX_SEG_LIST_SIZE,
1569                             mrioc->req_qinfo[q_idx].q_segment_list,
1570                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1571                         mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1572                 }
1573         } else
1574                 size = mrioc->req_qinfo[q_idx].segment_qd *
1575                     mrioc->facts.op_req_sz;
1576
1577         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1578                 if (!segments[j].segment)
1579                         continue;
1580                 dma_free_coherent(&mrioc->pdev->dev,
1581                     size, segments[j].segment, segments[j].segment_dma);
1582                 segments[j].segment = NULL;
1583         }
1584         kfree(mrioc->req_qinfo[q_idx].q_segments);
1585         mrioc->req_qinfo[q_idx].q_segments = NULL;
1586         mrioc->req_qinfo[q_idx].qid = 0;
1587 }
1588
1589 /**
1590  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1591  * @mrioc: Adapter instance reference
1592  * @q_idx: operational reply queue index
1593  *
1594  * Free memory segments allocated for operational reply queue
1595  *
1596  * Return: Nothing.
1597  */
1598 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1599 {
1600         u16 j;
1601         int size;
1602         struct segments *segments;
1603
1604         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1605         if (!segments)
1606                 return;
1607
1608         if (mrioc->enable_segqueue) {
1609                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1610                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1611                         dma_free_coherent(&mrioc->pdev->dev,
1612                             MPI3MR_MAX_SEG_LIST_SIZE,
1613                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1614                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1615                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1616                 }
1617         } else
1618                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1619                     mrioc->op_reply_desc_sz;
1620
1621         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1622                 if (!segments[j].segment)
1623                         continue;
1624                 dma_free_coherent(&mrioc->pdev->dev,
1625                     size, segments[j].segment, segments[j].segment_dma);
1626                 segments[j].segment = NULL;
1627         }
1628
1629         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1630         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1631         mrioc->op_reply_qinfo[q_idx].qid = 0;
1632 }
1633
1634 /**
1635  * mpi3mr_delete_op_reply_q - delete operational reply queue
1636  * @mrioc: Adapter instance reference
1637  * @qidx: operational reply queue index
1638  *
1639  * Delete operatinal reply queue by issuing MPI request
1640  * through admin queue.
1641  *
1642  * Return:  0 on success, non-zero on failure.
1643  */
1644 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1645 {
1646         struct mpi3_delete_reply_queue_request delq_req;
1647         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1648         int retval = 0;
1649         u16 reply_qid = 0, midx;
1650
1651         reply_qid = op_reply_q->qid;
1652
1653         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1654
1655         if (!reply_qid) {
1656                 retval = -1;
1657                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1658                 goto out;
1659         }
1660
1661         (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :
1662             mrioc->active_poll_qcount--;
1663
1664         memset(&delq_req, 0, sizeof(delq_req));
1665         mutex_lock(&mrioc->init_cmds.mutex);
1666         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1667                 retval = -1;
1668                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1669                 mutex_unlock(&mrioc->init_cmds.mutex);
1670                 goto out;
1671         }
1672         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1673         mrioc->init_cmds.is_waiting = 1;
1674         mrioc->init_cmds.callback = NULL;
1675         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1676         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1677         delq_req.queue_id = cpu_to_le16(reply_qid);
1678
1679         init_completion(&mrioc->init_cmds.done);
1680         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1681             1);
1682         if (retval) {
1683                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1684                 goto out_unlock;
1685         }
1686         wait_for_completion_timeout(&mrioc->init_cmds.done,
1687             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1688         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1689                 ioc_err(mrioc, "delete reply queue timed out\n");
1690                 mpi3mr_check_rh_fault_ioc(mrioc,
1691                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1692                 retval = -1;
1693                 goto out_unlock;
1694         }
1695         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1696             != MPI3_IOCSTATUS_SUCCESS) {
1697                 ioc_err(mrioc,
1698                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1699                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1700                     mrioc->init_cmds.ioc_loginfo);
1701                 retval = -1;
1702                 goto out_unlock;
1703         }
1704         mrioc->intr_info[midx].op_reply_q = NULL;
1705
1706         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1707 out_unlock:
1708         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1709         mutex_unlock(&mrioc->init_cmds.mutex);
1710 out:
1711
1712         return retval;
1713 }
1714
1715 /**
1716  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1717  * @mrioc: Adapter instance reference
1718  * @qidx: request queue index
1719  *
1720  * Allocate segmented memory pools for operational reply
1721  * queue.
1722  *
1723  * Return: 0 on success, non-zero on failure.
1724  */
1725 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1726 {
1727         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1728         int i, size;
1729         u64 *q_segment_list_entry = NULL;
1730         struct segments *segments;
1731
1732         if (mrioc->enable_segqueue) {
1733                 op_reply_q->segment_qd =
1734                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1735
1736                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1737
1738                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1739                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1740                     GFP_KERNEL);
1741                 if (!op_reply_q->q_segment_list)
1742                         return -ENOMEM;
1743                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1744         } else {
1745                 op_reply_q->segment_qd = op_reply_q->num_replies;
1746                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1747         }
1748
1749         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1750             op_reply_q->segment_qd);
1751
1752         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1753             sizeof(struct segments), GFP_KERNEL);
1754         if (!op_reply_q->q_segments)
1755                 return -ENOMEM;
1756
1757         segments = op_reply_q->q_segments;
1758         for (i = 0; i < op_reply_q->num_segments; i++) {
1759                 segments[i].segment =
1760                     dma_alloc_coherent(&mrioc->pdev->dev,
1761                     size, &segments[i].segment_dma, GFP_KERNEL);
1762                 if (!segments[i].segment)
1763                         return -ENOMEM;
1764                 if (mrioc->enable_segqueue)
1765                         q_segment_list_entry[i] =
1766                             (unsigned long)segments[i].segment_dma;
1767         }
1768
1769         return 0;
1770 }
1771
1772 /**
1773  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1774  * @mrioc: Adapter instance reference
1775  * @qidx: request queue index
1776  *
1777  * Allocate segmented memory pools for operational request
1778  * queue.
1779  *
1780  * Return: 0 on success, non-zero on failure.
1781  */
1782 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1783 {
1784         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1785         int i, size;
1786         u64 *q_segment_list_entry = NULL;
1787         struct segments *segments;
1788
1789         if (mrioc->enable_segqueue) {
1790                 op_req_q->segment_qd =
1791                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1792
1793                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1794
1795                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1796                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1797                     GFP_KERNEL);
1798                 if (!op_req_q->q_segment_list)
1799                         return -ENOMEM;
1800                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1801
1802         } else {
1803                 op_req_q->segment_qd = op_req_q->num_requests;
1804                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1805         }
1806
1807         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1808             op_req_q->segment_qd);
1809
1810         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1811             sizeof(struct segments), GFP_KERNEL);
1812         if (!op_req_q->q_segments)
1813                 return -ENOMEM;
1814
1815         segments = op_req_q->q_segments;
1816         for (i = 0; i < op_req_q->num_segments; i++) {
1817                 segments[i].segment =
1818                     dma_alloc_coherent(&mrioc->pdev->dev,
1819                     size, &segments[i].segment_dma, GFP_KERNEL);
1820                 if (!segments[i].segment)
1821                         return -ENOMEM;
1822                 if (mrioc->enable_segqueue)
1823                         q_segment_list_entry[i] =
1824                             (unsigned long)segments[i].segment_dma;
1825         }
1826
1827         return 0;
1828 }
1829
1830 /**
1831  * mpi3mr_create_op_reply_q - create operational reply queue
1832  * @mrioc: Adapter instance reference
1833  * @qidx: operational reply queue index
1834  *
1835  * Create operatinal reply queue by issuing MPI request
1836  * through admin queue.
1837  *
1838  * Return:  0 on success, non-zero on failure.
1839  */
1840 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1841 {
1842         struct mpi3_create_reply_queue_request create_req;
1843         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1844         int retval = 0;
1845         u16 reply_qid = 0, midx;
1846
1847         reply_qid = op_reply_q->qid;
1848
1849         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1850
1851         if (reply_qid) {
1852                 retval = -1;
1853                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1854                     reply_qid);
1855
1856                 return retval;
1857         }
1858
1859         reply_qid = qidx + 1;
1860         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1861         if (!mrioc->pdev->revision)
1862                 op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
1863         op_reply_q->ci = 0;
1864         op_reply_q->ephase = 1;
1865         atomic_set(&op_reply_q->pend_ios, 0);
1866         atomic_set(&op_reply_q->in_use, 0);
1867         op_reply_q->enable_irq_poll = false;
1868
1869         if (!op_reply_q->q_segments) {
1870                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1871                 if (retval) {
1872                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1873                         goto out;
1874                 }
1875         }
1876
1877         memset(&create_req, 0, sizeof(create_req));
1878         mutex_lock(&mrioc->init_cmds.mutex);
1879         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1880                 retval = -1;
1881                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1882                 goto out_unlock;
1883         }
1884         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1885         mrioc->init_cmds.is_waiting = 1;
1886         mrioc->init_cmds.callback = NULL;
1887         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1888         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1889         create_req.queue_id = cpu_to_le16(reply_qid);
1890
1891         if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))
1892                 op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;
1893         else
1894                 op_reply_q->qtype = MPI3MR_POLL_QUEUE;
1895
1896         if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {
1897                 create_req.flags =
1898                         MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1899                 create_req.msix_index =
1900                         cpu_to_le16(mrioc->intr_info[midx].msix_index);
1901         } else {
1902                 create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);
1903                 ioc_info(mrioc, "create reply queue(polled): for qid(%d), midx(%d)\n",
1904                         reply_qid, midx);
1905                 if (!mrioc->active_poll_qcount)
1906                         disable_irq_nosync(pci_irq_vector(mrioc->pdev,
1907                             mrioc->intr_info_count - 1));
1908         }
1909
1910         if (mrioc->enable_segqueue) {
1911                 create_req.flags |=
1912                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1913                 create_req.base_address = cpu_to_le64(
1914                     op_reply_q->q_segment_list_dma);
1915         } else
1916                 create_req.base_address = cpu_to_le64(
1917                     op_reply_q->q_segments[0].segment_dma);
1918
1919         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1920
1921         init_completion(&mrioc->init_cmds.done);
1922         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1923             sizeof(create_req), 1);
1924         if (retval) {
1925                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1926                 goto out_unlock;
1927         }
1928         wait_for_completion_timeout(&mrioc->init_cmds.done,
1929             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1930         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1931                 ioc_err(mrioc, "create reply queue timed out\n");
1932                 mpi3mr_check_rh_fault_ioc(mrioc,
1933                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1934                 retval = -1;
1935                 goto out_unlock;
1936         }
1937         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1938             != MPI3_IOCSTATUS_SUCCESS) {
1939                 ioc_err(mrioc,
1940                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1941                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1942                     mrioc->init_cmds.ioc_loginfo);
1943                 retval = -1;
1944                 goto out_unlock;
1945         }
1946         op_reply_q->qid = reply_qid;
1947         if (midx < mrioc->intr_info_count)
1948                 mrioc->intr_info[midx].op_reply_q = op_reply_q;
1949
1950         (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :
1951             mrioc->active_poll_qcount++;
1952
1953 out_unlock:
1954         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1955         mutex_unlock(&mrioc->init_cmds.mutex);
1956 out:
1957
1958         return retval;
1959 }
1960
1961 /**
1962  * mpi3mr_create_op_req_q - create operational request queue
1963  * @mrioc: Adapter instance reference
1964  * @idx: operational request queue index
1965  * @reply_qid: Reply queue ID
1966  *
1967  * Create operatinal request queue by issuing MPI request
1968  * through admin queue.
1969  *
1970  * Return:  0 on success, non-zero on failure.
1971  */
1972 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1973         u16 reply_qid)
1974 {
1975         struct mpi3_create_request_queue_request create_req;
1976         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1977         int retval = 0;
1978         u16 req_qid = 0;
1979
1980         req_qid = op_req_q->qid;
1981
1982         if (req_qid) {
1983                 retval = -1;
1984                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1985                     req_qid);
1986
1987                 return retval;
1988         }
1989         req_qid = idx + 1;
1990
1991         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1992         op_req_q->ci = 0;
1993         op_req_q->pi = 0;
1994         op_req_q->reply_qid = reply_qid;
1995         spin_lock_init(&op_req_q->q_lock);
1996
1997         if (!op_req_q->q_segments) {
1998                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1999                 if (retval) {
2000                         mpi3mr_free_op_req_q_segments(mrioc, idx);
2001                         goto out;
2002                 }
2003         }
2004
2005         memset(&create_req, 0, sizeof(create_req));
2006         mutex_lock(&mrioc->init_cmds.mutex);
2007         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2008                 retval = -1;
2009                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
2010                 goto out_unlock;
2011         }
2012         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2013         mrioc->init_cmds.is_waiting = 1;
2014         mrioc->init_cmds.callback = NULL;
2015         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2016         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
2017         create_req.queue_id = cpu_to_le16(req_qid);
2018         if (mrioc->enable_segqueue) {
2019                 create_req.flags =
2020                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
2021                 create_req.base_address = cpu_to_le64(
2022                     op_req_q->q_segment_list_dma);
2023         } else
2024                 create_req.base_address = cpu_to_le64(
2025                     op_req_q->q_segments[0].segment_dma);
2026         create_req.reply_queue_id = cpu_to_le16(reply_qid);
2027         create_req.size = cpu_to_le16(op_req_q->num_requests);
2028
2029         init_completion(&mrioc->init_cmds.done);
2030         retval = mpi3mr_admin_request_post(mrioc, &create_req,
2031             sizeof(create_req), 1);
2032         if (retval) {
2033                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
2034                 goto out_unlock;
2035         }
2036         wait_for_completion_timeout(&mrioc->init_cmds.done,
2037             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2038         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2039                 ioc_err(mrioc, "create request queue timed out\n");
2040                 mpi3mr_check_rh_fault_ioc(mrioc,
2041                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT);
2042                 retval = -1;
2043                 goto out_unlock;
2044         }
2045         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2046             != MPI3_IOCSTATUS_SUCCESS) {
2047                 ioc_err(mrioc,
2048                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2049                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2050                     mrioc->init_cmds.ioc_loginfo);
2051                 retval = -1;
2052                 goto out_unlock;
2053         }
2054         op_req_q->qid = req_qid;
2055
2056 out_unlock:
2057         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2058         mutex_unlock(&mrioc->init_cmds.mutex);
2059 out:
2060
2061         return retval;
2062 }
2063
2064 /**
2065  * mpi3mr_create_op_queues - create operational queue pairs
2066  * @mrioc: Adapter instance reference
2067  *
2068  * Allocate memory for operational queue meta data and call
2069  * create request and reply queue functions.
2070  *
2071  * Return: 0 on success, non-zero on failures.
2072  */
2073 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
2074 {
2075         int retval = 0;
2076         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
2077
2078         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
2079             mrioc->facts.max_op_req_q);
2080
2081         msix_count_op_q =
2082             mrioc->intr_info_count - mrioc->op_reply_q_offset;
2083         if (!mrioc->num_queues)
2084                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
2085         /*
2086          * During reset set the num_queues to the number of queues
2087          * that was set before the reset.
2088          */
2089         num_queues = mrioc->num_op_reply_q ?
2090             mrioc->num_op_reply_q : mrioc->num_queues;
2091         ioc_info(mrioc, "trying to create %d operational queue pairs\n",
2092             num_queues);
2093
2094         if (!mrioc->req_qinfo) {
2095                 mrioc->req_qinfo = kcalloc(num_queues,
2096                     sizeof(struct op_req_qinfo), GFP_KERNEL);
2097                 if (!mrioc->req_qinfo) {
2098                         retval = -1;
2099                         goto out_failed;
2100                 }
2101
2102                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
2103                     num_queues, GFP_KERNEL);
2104                 if (!mrioc->op_reply_qinfo) {
2105                         retval = -1;
2106                         goto out_failed;
2107                 }
2108         }
2109
2110         if (mrioc->enable_segqueue)
2111                 ioc_info(mrioc,
2112                     "allocating operational queues through segmented queues\n");
2113
2114         for (i = 0; i < num_queues; i++) {
2115                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
2116                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
2117                         break;
2118                 }
2119                 if (mpi3mr_create_op_req_q(mrioc, i,
2120                     mrioc->op_reply_qinfo[i].qid)) {
2121                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
2122                         mpi3mr_delete_op_reply_q(mrioc, i);
2123                         break;
2124                 }
2125         }
2126
2127         if (i == 0) {
2128                 /* Not even one queue is created successfully*/
2129                 retval = -1;
2130                 goto out_failed;
2131         }
2132         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
2133         ioc_info(mrioc,
2134             "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",
2135             mrioc->num_op_reply_q, mrioc->default_qcount,
2136             mrioc->active_poll_qcount);
2137
2138         return retval;
2139 out_failed:
2140         kfree(mrioc->req_qinfo);
2141         mrioc->req_qinfo = NULL;
2142
2143         kfree(mrioc->op_reply_qinfo);
2144         mrioc->op_reply_qinfo = NULL;
2145
2146         return retval;
2147 }
2148
2149 /**
2150  * mpi3mr_op_request_post - Post request to operational queue
2151  * @mrioc: Adapter reference
2152  * @op_req_q: Operational request queue info
2153  * @req: MPI3 request
2154  *
2155  * Post the MPI3 request into operational request queue and
2156  * inform the controller, if the queue is full return
2157  * appropriate error.
2158  *
2159  * Return: 0 on success, non-zero on failure.
2160  */
2161 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
2162         struct op_req_qinfo *op_req_q, u8 *req)
2163 {
2164         u16 pi = 0, max_entries, reply_qidx = 0, midx;
2165         int retval = 0;
2166         unsigned long flags;
2167         u8 *req_entry;
2168         void *segment_base_addr;
2169         u16 req_sz = mrioc->facts.op_req_sz;
2170         struct segments *segments = op_req_q->q_segments;
2171
2172         reply_qidx = op_req_q->reply_qid - 1;
2173
2174         if (mrioc->unrecoverable)
2175                 return -EFAULT;
2176
2177         spin_lock_irqsave(&op_req_q->q_lock, flags);
2178         pi = op_req_q->pi;
2179         max_entries = op_req_q->num_requests;
2180
2181         if (mpi3mr_check_req_qfull(op_req_q)) {
2182                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
2183                     reply_qidx, mrioc->op_reply_q_offset);
2184                 mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
2185
2186                 if (mpi3mr_check_req_qfull(op_req_q)) {
2187                         retval = -EAGAIN;
2188                         goto out;
2189                 }
2190         }
2191
2192         if (mrioc->reset_in_progress) {
2193                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
2194                 retval = -EAGAIN;
2195                 goto out;
2196         }
2197
2198         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
2199         req_entry = (u8 *)segment_base_addr +
2200             ((pi % op_req_q->segment_qd) * req_sz);
2201
2202         memset(req_entry, 0, req_sz);
2203         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
2204
2205         if (++pi == max_entries)
2206                 pi = 0;
2207         op_req_q->pi = pi;
2208
2209 #ifndef CONFIG_PREEMPT_RT
2210         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
2211             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
2212                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
2213 #else
2214         atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);
2215 #endif
2216
2217         writel(op_req_q->pi,
2218             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
2219
2220 out:
2221         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
2222         return retval;
2223 }
2224
2225 /**
2226  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2227  * controller
2228  * @mrioc: Adapter instance reference
2229  * @reason_code: reason code for the fault.
2230  *
2231  * This routine will save snapdump and fault the controller with
2232  * the given reason code if it is not already in the fault or
2233  * not asynchronosuly reset. This will be used to handle
2234  * initilaization time faults/resets/timeout as in those cases
2235  * immediate soft reset invocation is not required.
2236  *
2237  * Return:  None.
2238  */
2239 void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
2240 {
2241         u32 ioc_status, host_diagnostic, timeout;
2242
2243         if (mrioc->unrecoverable) {
2244                 ioc_err(mrioc, "controller is unrecoverable\n");
2245                 return;
2246         }
2247
2248         if (!pci_device_is_present(mrioc->pdev)) {
2249                 mrioc->unrecoverable = 1;
2250                 ioc_err(mrioc, "controller is not present\n");
2251                 return;
2252         }
2253
2254         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2255         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
2256             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
2257                 mpi3mr_print_fault_info(mrioc);
2258                 return;
2259         }
2260         mpi3mr_set_diagsave(mrioc);
2261         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2262             reason_code);
2263         timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
2264         do {
2265                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2266                 if (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
2267                         break;
2268                 msleep(100);
2269         } while (--timeout);
2270 }
2271
2272 /**
2273  * mpi3mr_sync_timestamp - Issue time stamp sync request
2274  * @mrioc: Adapter reference
2275  *
2276  * Issue IO unit control MPI request to synchornize firmware
2277  * timestamp with host time.
2278  *
2279  * Return: 0 on success, non-zero on failure.
2280  */
2281 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
2282 {
2283         ktime_t current_time;
2284         struct mpi3_iounit_control_request iou_ctrl;
2285         int retval = 0;
2286
2287         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2288         mutex_lock(&mrioc->init_cmds.mutex);
2289         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2290                 retval = -1;
2291                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
2292                 mutex_unlock(&mrioc->init_cmds.mutex);
2293                 goto out;
2294         }
2295         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2296         mrioc->init_cmds.is_waiting = 1;
2297         mrioc->init_cmds.callback = NULL;
2298         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2299         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2300         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
2301         current_time = ktime_get_real();
2302         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
2303
2304         init_completion(&mrioc->init_cmds.done);
2305         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
2306             sizeof(iou_ctrl), 0);
2307         if (retval) {
2308                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
2309                 goto out_unlock;
2310         }
2311
2312         wait_for_completion_timeout(&mrioc->init_cmds.done,
2313             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2314         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2315                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
2316                 mrioc->init_cmds.is_waiting = 0;
2317                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2318                         mpi3mr_soft_reset_handler(mrioc,
2319                             MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
2320                 retval = -1;
2321                 goto out_unlock;
2322         }
2323         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2324             != MPI3_IOCSTATUS_SUCCESS) {
2325                 ioc_err(mrioc,
2326                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2327                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2328                     mrioc->init_cmds.ioc_loginfo);
2329                 retval = -1;
2330                 goto out_unlock;
2331         }
2332
2333 out_unlock:
2334         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2335         mutex_unlock(&mrioc->init_cmds.mutex);
2336
2337 out:
2338         return retval;
2339 }
2340
2341 /**
2342  * mpi3mr_print_pkg_ver - display controller fw package version
2343  * @mrioc: Adapter reference
2344  *
2345  * Retrieve firmware package version from the component image
2346  * header of the controller flash and display it.
2347  *
2348  * Return: 0 on success and non-zero on failure.
2349  */
2350 static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
2351 {
2352         struct mpi3_ci_upload_request ci_upload;
2353         int retval = -1;
2354         void *data = NULL;
2355         dma_addr_t data_dma;
2356         struct mpi3_ci_manifest_mpi *manifest;
2357         u32 data_len = sizeof(struct mpi3_ci_manifest_mpi);
2358         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2359
2360         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2361             GFP_KERNEL);
2362         if (!data)
2363                 return -ENOMEM;
2364
2365         memset(&ci_upload, 0, sizeof(ci_upload));
2366         mutex_lock(&mrioc->init_cmds.mutex);
2367         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2368                 ioc_err(mrioc, "sending get package version failed due to command in use\n");
2369                 mutex_unlock(&mrioc->init_cmds.mutex);
2370                 goto out;
2371         }
2372         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2373         mrioc->init_cmds.is_waiting = 1;
2374         mrioc->init_cmds.callback = NULL;
2375         ci_upload.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2376         ci_upload.function = MPI3_FUNCTION_CI_UPLOAD;
2377         ci_upload.msg_flags = MPI3_CI_UPLOAD_MSGFLAGS_LOCATION_PRIMARY;
2378         ci_upload.signature1 = cpu_to_le32(MPI3_IMAGE_HEADER_SIGNATURE1_MANIFEST);
2379         ci_upload.image_offset = cpu_to_le32(MPI3_IMAGE_HEADER_SIZE);
2380         ci_upload.segment_size = cpu_to_le32(data_len);
2381
2382         mpi3mr_add_sg_single(&ci_upload.sgl, sgl_flags, data_len,
2383             data_dma);
2384         init_completion(&mrioc->init_cmds.done);
2385         retval = mpi3mr_admin_request_post(mrioc, &ci_upload,
2386             sizeof(ci_upload), 1);
2387         if (retval) {
2388                 ioc_err(mrioc, "posting get package version failed\n");
2389                 goto out_unlock;
2390         }
2391         wait_for_completion_timeout(&mrioc->init_cmds.done,
2392             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2393         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2394                 ioc_err(mrioc, "get package version timed out\n");
2395                 mpi3mr_check_rh_fault_ioc(mrioc,
2396                     MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT);
2397                 retval = -1;
2398                 goto out_unlock;
2399         }
2400         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2401             == MPI3_IOCSTATUS_SUCCESS) {
2402                 manifest = (struct mpi3_ci_manifest_mpi *) data;
2403                 if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {
2404                         ioc_info(mrioc,
2405                             "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",
2406                             manifest->package_version.gen_major,
2407                             manifest->package_version.gen_minor,
2408                             manifest->package_version.phase_major,
2409                             manifest->package_version.phase_minor,
2410                             manifest->package_version.customer_id,
2411                             manifest->package_version.build_num);
2412                 }
2413         }
2414         retval = 0;
2415 out_unlock:
2416         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2417         mutex_unlock(&mrioc->init_cmds.mutex);
2418
2419 out:
2420         if (data)
2421                 dma_free_coherent(&mrioc->pdev->dev, data_len, data,
2422                     data_dma);
2423         return retval;
2424 }
2425
2426 /**
2427  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2428  * @work: work struct
2429  *
2430  * Watch dog work periodically executed (1 second interval) to
2431  * monitor firmware fault and to issue periodic timer sync to
2432  * the firmware.
2433  *
2434  * Return: Nothing.
2435  */
2436 static void mpi3mr_watchdog_work(struct work_struct *work)
2437 {
2438         struct mpi3mr_ioc *mrioc =
2439             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
2440         unsigned long flags;
2441         enum mpi3mr_iocstate ioc_state;
2442         u32 fault, host_diagnostic, ioc_status;
2443         u32 reset_reason = MPI3MR_RESET_FROM_FAULT_WATCH;
2444
2445         if (mrioc->reset_in_progress)
2446                 return;
2447
2448         if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {
2449                 ioc_err(mrioc, "watchdog could not detect the controller\n");
2450                 mrioc->unrecoverable = 1;
2451         }
2452
2453         if (mrioc->unrecoverable) {
2454                 ioc_err(mrioc,
2455                     "flush pending commands for unrecoverable controller\n");
2456                 mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
2457                 return;
2458         }
2459
2460         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
2461                 mrioc->ts_update_counter = 0;
2462                 mpi3mr_sync_timestamp(mrioc);
2463         }
2464
2465         if ((mrioc->prepare_for_reset) &&
2466             ((mrioc->prepare_for_reset_timeout_counter++) >=
2467              MPI3MR_PREPARE_FOR_RESET_TIMEOUT)) {
2468                 mpi3mr_soft_reset_handler(mrioc,
2469                     MPI3MR_RESET_FROM_CIACTVRST_TIMER, 1);
2470                 return;
2471         }
2472
2473         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2474         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
2475                 mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_FIRMWARE, 0);
2476                 return;
2477         }
2478
2479         /*Check for fault state every one second and issue Soft reset*/
2480         ioc_state = mpi3mr_get_iocstate(mrioc);
2481         if (ioc_state != MRIOC_STATE_FAULT)
2482                 goto schedule_work;
2483
2484         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
2485         host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2486         if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2487                 if (!mrioc->diagsave_timeout) {
2488                         mpi3mr_print_fault_info(mrioc);
2489                         ioc_warn(mrioc, "diag save in progress\n");
2490                 }
2491                 if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2492                         goto schedule_work;
2493         }
2494
2495         mpi3mr_print_fault_info(mrioc);
2496         mrioc->diagsave_timeout = 0;
2497
2498         switch (fault) {
2499         case MPI3_SYSIF_FAULT_CODE_COMPLETE_RESET_NEEDED:
2500         case MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED:
2501                 ioc_warn(mrioc,
2502                     "controller requires system power cycle, marking controller as unrecoverable\n");
2503                 mrioc->unrecoverable = 1;
2504                 goto schedule_work;
2505         case MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS:
2506                 return;
2507         case MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET:
2508                 reset_reason = MPI3MR_RESET_FROM_CIACTIV_FAULT;
2509                 break;
2510         default:
2511                 break;
2512         }
2513         mpi3mr_soft_reset_handler(mrioc, reset_reason, 0);
2514         return;
2515
2516 schedule_work:
2517         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2518         if (mrioc->watchdog_work_q)
2519                 queue_delayed_work(mrioc->watchdog_work_q,
2520                     &mrioc->watchdog_work,
2521                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2522         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2523         return;
2524 }
2525
2526 /**
2527  * mpi3mr_start_watchdog - Start watchdog
2528  * @mrioc: Adapter instance reference
2529  *
2530  * Create and start the watchdog thread to monitor controller
2531  * faults.
2532  *
2533  * Return: Nothing.
2534  */
2535 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2536 {
2537         if (mrioc->watchdog_work_q)
2538                 return;
2539
2540         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2541         snprintf(mrioc->watchdog_work_q_name,
2542             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2543             mrioc->id);
2544         mrioc->watchdog_work_q =
2545             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2546         if (!mrioc->watchdog_work_q) {
2547                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2548                 return;
2549         }
2550
2551         if (mrioc->watchdog_work_q)
2552                 queue_delayed_work(mrioc->watchdog_work_q,
2553                     &mrioc->watchdog_work,
2554                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2555 }
2556
2557 /**
2558  * mpi3mr_stop_watchdog - Stop watchdog
2559  * @mrioc: Adapter instance reference
2560  *
2561  * Stop the watchdog thread created to monitor controller
2562  * faults.
2563  *
2564  * Return: Nothing.
2565  */
2566 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2567 {
2568         unsigned long flags;
2569         struct workqueue_struct *wq;
2570
2571         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2572         wq = mrioc->watchdog_work_q;
2573         mrioc->watchdog_work_q = NULL;
2574         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2575         if (wq) {
2576                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2577                         flush_workqueue(wq);
2578                 destroy_workqueue(wq);
2579         }
2580 }
2581
2582 /**
2583  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2584  * @mrioc: Adapter instance reference
2585  *
2586  * Allocate memory for admin queue pair if required and register
2587  * the admin queue with the controller.
2588  *
2589  * Return: 0 on success, non-zero on failures.
2590  */
2591 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2592 {
2593         int retval = 0;
2594         u32 num_admin_entries = 0;
2595
2596         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2597         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2598             MPI3MR_ADMIN_REQ_FRAME_SZ;
2599         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2600         mrioc->admin_req_base = NULL;
2601
2602         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2603         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2604             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2605         mrioc->admin_reply_ci = 0;
2606         mrioc->admin_reply_ephase = 1;
2607         mrioc->admin_reply_base = NULL;
2608
2609         if (!mrioc->admin_req_base) {
2610                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2611                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2612
2613                 if (!mrioc->admin_req_base) {
2614                         retval = -1;
2615                         goto out_failed;
2616                 }
2617
2618                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2619                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2620                     GFP_KERNEL);
2621
2622                 if (!mrioc->admin_reply_base) {
2623                         retval = -1;
2624                         goto out_failed;
2625                 }
2626         }
2627
2628         num_admin_entries = (mrioc->num_admin_replies << 16) |
2629             (mrioc->num_admin_req);
2630         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2631         mpi3mr_writeq(mrioc->admin_req_dma,
2632             &mrioc->sysif_regs->admin_request_queue_address);
2633         mpi3mr_writeq(mrioc->admin_reply_dma,
2634             &mrioc->sysif_regs->admin_reply_queue_address);
2635         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2636         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2637         return retval;
2638
2639 out_failed:
2640
2641         if (mrioc->admin_reply_base) {
2642                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2643                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2644                 mrioc->admin_reply_base = NULL;
2645         }
2646         if (mrioc->admin_req_base) {
2647                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2648                     mrioc->admin_req_base, mrioc->admin_req_dma);
2649                 mrioc->admin_req_base = NULL;
2650         }
2651         return retval;
2652 }
2653
2654 /**
2655  * mpi3mr_issue_iocfacts - Send IOC Facts
2656  * @mrioc: Adapter instance reference
2657  * @facts_data: Cached IOC facts data
2658  *
2659  * Issue IOC Facts MPI request through admin queue and wait for
2660  * the completion of it or time out.
2661  *
2662  * Return: 0 on success, non-zero on failures.
2663  */
2664 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2665         struct mpi3_ioc_facts_data *facts_data)
2666 {
2667         struct mpi3_ioc_facts_request iocfacts_req;
2668         void *data = NULL;
2669         dma_addr_t data_dma;
2670         u32 data_len = sizeof(*facts_data);
2671         int retval = 0;
2672         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2673
2674         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2675             GFP_KERNEL);
2676
2677         if (!data) {
2678                 retval = -1;
2679                 goto out;
2680         }
2681
2682         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2683         mutex_lock(&mrioc->init_cmds.mutex);
2684         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2685                 retval = -1;
2686                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2687                 mutex_unlock(&mrioc->init_cmds.mutex);
2688                 goto out;
2689         }
2690         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2691         mrioc->init_cmds.is_waiting = 1;
2692         mrioc->init_cmds.callback = NULL;
2693         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2694         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2695
2696         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2697             data_dma);
2698
2699         init_completion(&mrioc->init_cmds.done);
2700         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2701             sizeof(iocfacts_req), 1);
2702         if (retval) {
2703                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2704                 goto out_unlock;
2705         }
2706         wait_for_completion_timeout(&mrioc->init_cmds.done,
2707             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2708         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2709                 ioc_err(mrioc, "ioc_facts timed out\n");
2710                 mpi3mr_check_rh_fault_ioc(mrioc,
2711                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2712                 retval = -1;
2713                 goto out_unlock;
2714         }
2715         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2716             != MPI3_IOCSTATUS_SUCCESS) {
2717                 ioc_err(mrioc,
2718                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2719                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2720                     mrioc->init_cmds.ioc_loginfo);
2721                 retval = -1;
2722                 goto out_unlock;
2723         }
2724         memcpy(facts_data, (u8 *)data, data_len);
2725         mpi3mr_process_factsdata(mrioc, facts_data);
2726 out_unlock:
2727         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2728         mutex_unlock(&mrioc->init_cmds.mutex);
2729
2730 out:
2731         if (data)
2732                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2733
2734         return retval;
2735 }
2736
2737 /**
2738  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2739  * @mrioc: Adapter instance reference
2740  *
2741  * Check whether the new DMA mask requested through IOCFacts by
2742  * firmware needs to be set, if so set it .
2743  *
2744  * Return: 0 on success, non-zero on failure.
2745  */
2746 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2747 {
2748         struct pci_dev *pdev = mrioc->pdev;
2749         int r;
2750         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2751
2752         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2753                 return 0;
2754
2755         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2756             mrioc->dma_mask, facts_dma_mask);
2757
2758         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2759         if (r) {
2760                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2761                     facts_dma_mask, r);
2762                 return r;
2763         }
2764         mrioc->dma_mask = facts_dma_mask;
2765         return r;
2766 }
2767
2768 /**
2769  * mpi3mr_process_factsdata - Process IOC facts data
2770  * @mrioc: Adapter instance reference
2771  * @facts_data: Cached IOC facts data
2772  *
2773  * Convert IOC facts data into cpu endianness and cache it in
2774  * the driver .
2775  *
2776  * Return: Nothing.
2777  */
2778 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2779         struct mpi3_ioc_facts_data *facts_data)
2780 {
2781         u32 ioc_config, req_sz, facts_flags;
2782
2783         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2784             (sizeof(*facts_data) / 4)) {
2785                 ioc_warn(mrioc,
2786                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2787                     sizeof(*facts_data),
2788                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2789         }
2790
2791         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2792         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2793             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2794         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2795                 ioc_err(mrioc,
2796                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2797                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2798         }
2799
2800         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2801
2802         facts_flags = le32_to_cpu(facts_data->flags);
2803         mrioc->facts.op_req_sz = req_sz;
2804         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2805             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2806             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2807
2808         mrioc->facts.ioc_num = facts_data->ioc_number;
2809         mrioc->facts.who_init = facts_data->who_init;
2810         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2811         mrioc->facts.personality = (facts_flags &
2812             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2813         mrioc->facts.dma_mask = (facts_flags &
2814             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2815             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2816         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2817         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2818         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
2819         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2820         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2821         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2822         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2823         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2824         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2825         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2826         mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2827         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2828         mrioc->facts.max_pcie_switches =
2829             le16_to_cpu(facts_data->max_pcie_switches);
2830         mrioc->facts.max_sasexpanders =
2831             le16_to_cpu(facts_data->max_sas_expanders);
2832         mrioc->facts.max_sasinitiators =
2833             le16_to_cpu(facts_data->max_sas_initiators);
2834         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2835         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2836         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2837         mrioc->facts.max_op_req_q =
2838             le16_to_cpu(facts_data->max_operational_request_queues);
2839         mrioc->facts.max_op_reply_q =
2840             le16_to_cpu(facts_data->max_operational_reply_queues);
2841         mrioc->facts.ioc_capabilities =
2842             le32_to_cpu(facts_data->ioc_capabilities);
2843         mrioc->facts.fw_ver.build_num =
2844             le16_to_cpu(facts_data->fw_version.build_num);
2845         mrioc->facts.fw_ver.cust_id =
2846             le16_to_cpu(facts_data->fw_version.customer_id);
2847         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2848         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2849         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2850         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2851         mrioc->msix_count = min_t(int, mrioc->msix_count,
2852             mrioc->facts.max_msix_vectors);
2853         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2854         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2855         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2856         mrioc->facts.shutdown_timeout =
2857             le16_to_cpu(facts_data->shutdown_timeout);
2858
2859         mrioc->facts.max_dev_per_tg =
2860             facts_data->max_devices_per_throttle_group;
2861         mrioc->facts.io_throttle_data_length =
2862             le16_to_cpu(facts_data->io_throttle_data_length);
2863         mrioc->facts.max_io_throttle_group =
2864             le16_to_cpu(facts_data->max_io_throttle_group);
2865         mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);
2866         mrioc->facts.io_throttle_high =
2867             le16_to_cpu(facts_data->io_throttle_high);
2868
2869         /* Store in 512b block count */
2870         if (mrioc->facts.io_throttle_data_length)
2871                 mrioc->io_throttle_data_length =
2872                     (mrioc->facts.io_throttle_data_length * 2 * 4);
2873         else
2874                 /* set the length to 1MB + 1K to disable throttle */
2875                 mrioc->io_throttle_data_length = MPI3MR_MAX_SECTORS + 2;
2876
2877         mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);
2878         mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);
2879
2880         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2881             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2882             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2883         ioc_info(mrioc,
2884             "maxreqs(%d), mindh(%d) maxvectors(%d) maxperids(%d)\n",
2885             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2886             mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
2887         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2888             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2889             mrioc->facts.sge_mod_shift);
2890         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2891             mrioc->facts.dma_mask, (facts_flags &
2892             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2893         ioc_info(mrioc,
2894             "max_dev_per_throttle_group(%d), max_throttle_groups(%d)\n",
2895             mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);
2896         ioc_info(mrioc,
2897            "io_throttle_data_len(%dKiB), io_throttle_high(%dMiB), io_throttle_low(%dMiB)\n",
2898            mrioc->facts.io_throttle_data_length * 4,
2899            mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);
2900 }
2901
2902 /**
2903  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2904  * @mrioc: Adapter instance reference
2905  *
2906  * Allocate and initialize the reply free buffers, sense
2907  * buffers, reply free queue and sense buffer queue.
2908  *
2909  * Return: 0 on success, non-zero on failures.
2910  */
2911 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2912 {
2913         int retval = 0;
2914         u32 sz, i;
2915
2916         if (mrioc->init_cmds.reply)
2917                 return retval;
2918
2919         mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2920         if (!mrioc->init_cmds.reply)
2921                 goto out_failed;
2922
2923         mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2924         if (!mrioc->bsg_cmds.reply)
2925                 goto out_failed;
2926
2927         mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2928         if (!mrioc->transport_cmds.reply)
2929                 goto out_failed;
2930
2931         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2932                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,
2933                     GFP_KERNEL);
2934                 if (!mrioc->dev_rmhs_cmds[i].reply)
2935                         goto out_failed;
2936         }
2937
2938         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
2939                 mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,
2940                     GFP_KERNEL);
2941                 if (!mrioc->evtack_cmds[i].reply)
2942                         goto out_failed;
2943         }
2944
2945         mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2946         if (!mrioc->host_tm_cmds.reply)
2947                 goto out_failed;
2948
2949         mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2950         if (!mrioc->pel_cmds.reply)
2951                 goto out_failed;
2952
2953         mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2954         if (!mrioc->pel_abort_cmd.reply)
2955                 goto out_failed;
2956
2957         mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
2958         mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,
2959                                                  GFP_KERNEL);
2960         if (!mrioc->removepend_bitmap)
2961                 goto out_failed;
2962
2963         mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);
2964         if (!mrioc->devrem_bitmap)
2965                 goto out_failed;
2966
2967         mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,
2968                                                   GFP_KERNEL);
2969         if (!mrioc->evtack_cmds_bitmap)
2970                 goto out_failed;
2971
2972         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2973         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2974         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2975         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2976
2977         /* reply buffer pool, 16 byte align */
2978         sz = mrioc->num_reply_bufs * mrioc->reply_sz;
2979         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2980             &mrioc->pdev->dev, sz, 16, 0);
2981         if (!mrioc->reply_buf_pool) {
2982                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2983                 goto out_failed;
2984         }
2985
2986         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2987             &mrioc->reply_buf_dma);
2988         if (!mrioc->reply_buf)
2989                 goto out_failed;
2990
2991         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2992
2993         /* reply free queue, 8 byte align */
2994         sz = mrioc->reply_free_qsz * 8;
2995         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2996             &mrioc->pdev->dev, sz, 8, 0);
2997         if (!mrioc->reply_free_q_pool) {
2998                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2999                 goto out_failed;
3000         }
3001         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
3002             GFP_KERNEL, &mrioc->reply_free_q_dma);
3003         if (!mrioc->reply_free_q)
3004                 goto out_failed;
3005
3006         /* sense buffer pool,  4 byte align */
3007         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3008         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
3009             &mrioc->pdev->dev, sz, 4, 0);
3010         if (!mrioc->sense_buf_pool) {
3011                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
3012                 goto out_failed;
3013         }
3014         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
3015             &mrioc->sense_buf_dma);
3016         if (!mrioc->sense_buf)
3017                 goto out_failed;
3018
3019         /* sense buffer queue, 8 byte align */
3020         sz = mrioc->sense_buf_q_sz * 8;
3021         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
3022             &mrioc->pdev->dev, sz, 8, 0);
3023         if (!mrioc->sense_buf_q_pool) {
3024                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
3025                 goto out_failed;
3026         }
3027         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
3028             GFP_KERNEL, &mrioc->sense_buf_q_dma);
3029         if (!mrioc->sense_buf_q)
3030                 goto out_failed;
3031
3032         return retval;
3033
3034 out_failed:
3035         retval = -1;
3036         return retval;
3037 }
3038
3039 /**
3040  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3041  * buffers
3042  * @mrioc: Adapter instance reference
3043  *
3044  * Helper function to initialize reply and sense buffers along
3045  * with some debug prints.
3046  *
3047  * Return:  None.
3048  */
3049 static void mpimr_initialize_reply_sbuf_queues(struct mpi3mr_ioc *mrioc)
3050 {
3051         u32 sz, i;
3052         dma_addr_t phy_addr;
3053
3054         sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3055         ioc_info(mrioc,
3056             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3057             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,
3058             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
3059         sz = mrioc->reply_free_qsz * 8;
3060         ioc_info(mrioc,
3061             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3062             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
3063             (unsigned long long)mrioc->reply_free_q_dma);
3064         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3065         ioc_info(mrioc,
3066             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3067             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
3068             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
3069         sz = mrioc->sense_buf_q_sz * 8;
3070         ioc_info(mrioc,
3071             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3072             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
3073             (unsigned long long)mrioc->sense_buf_q_dma);
3074
3075         /* initialize Reply buffer Queue */
3076         for (i = 0, phy_addr = mrioc->reply_buf_dma;
3077             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)
3078                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
3079         mrioc->reply_free_q[i] = cpu_to_le64(0);
3080
3081         /* initialize Sense Buffer Queue */
3082         for (i = 0, phy_addr = mrioc->sense_buf_dma;
3083             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
3084                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
3085         mrioc->sense_buf_q[i] = cpu_to_le64(0);
3086 }
3087
3088 /**
3089  * mpi3mr_issue_iocinit - Send IOC Init
3090  * @mrioc: Adapter instance reference
3091  *
3092  * Issue IOC Init MPI request through admin queue and wait for
3093  * the completion of it or time out.
3094  *
3095  * Return: 0 on success, non-zero on failures.
3096  */
3097 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
3098 {
3099         struct mpi3_ioc_init_request iocinit_req;
3100         struct mpi3_driver_info_layout *drv_info;
3101         dma_addr_t data_dma;
3102         u32 data_len = sizeof(*drv_info);
3103         int retval = 0;
3104         ktime_t current_time;
3105
3106         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
3107             GFP_KERNEL);
3108         if (!drv_info) {
3109                 retval = -1;
3110                 goto out;
3111         }
3112         mpimr_initialize_reply_sbuf_queues(mrioc);
3113
3114         drv_info->information_length = cpu_to_le32(data_len);
3115         strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
3116         strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
3117         strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
3118         strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
3119         strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
3120         strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
3121             sizeof(drv_info->driver_release_date));
3122         drv_info->driver_capabilities = 0;
3123         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
3124             sizeof(mrioc->driver_info));
3125
3126         memset(&iocinit_req, 0, sizeof(iocinit_req));
3127         mutex_lock(&mrioc->init_cmds.mutex);
3128         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3129                 retval = -1;
3130                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
3131                 mutex_unlock(&mrioc->init_cmds.mutex);
3132                 goto out;
3133         }
3134         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3135         mrioc->init_cmds.is_waiting = 1;
3136         mrioc->init_cmds.callback = NULL;
3137         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3138         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
3139         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
3140         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
3141         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
3142         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
3143         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
3144         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
3145         iocinit_req.reply_free_queue_address =
3146             cpu_to_le64(mrioc->reply_free_q_dma);
3147         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSE_BUF_SZ);
3148         iocinit_req.sense_buffer_free_queue_depth =
3149             cpu_to_le16(mrioc->sense_buf_q_sz);
3150         iocinit_req.sense_buffer_free_queue_address =
3151             cpu_to_le64(mrioc->sense_buf_q_dma);
3152         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
3153
3154         current_time = ktime_get_real();
3155         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
3156
3157         init_completion(&mrioc->init_cmds.done);
3158         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
3159             sizeof(iocinit_req), 1);
3160         if (retval) {
3161                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
3162                 goto out_unlock;
3163         }
3164         wait_for_completion_timeout(&mrioc->init_cmds.done,
3165             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3166         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3167                 mpi3mr_check_rh_fault_ioc(mrioc,
3168                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
3169                 ioc_err(mrioc, "ioc_init timed out\n");
3170                 retval = -1;
3171                 goto out_unlock;
3172         }
3173         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3174             != MPI3_IOCSTATUS_SUCCESS) {
3175                 ioc_err(mrioc,
3176                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3177                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3178                     mrioc->init_cmds.ioc_loginfo);
3179                 retval = -1;
3180                 goto out_unlock;
3181         }
3182
3183         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3184         writel(mrioc->reply_free_queue_host_index,
3185             &mrioc->sysif_regs->reply_free_host_index);
3186
3187         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3188         writel(mrioc->sbq_host_index,
3189             &mrioc->sysif_regs->sense_buffer_free_host_index);
3190 out_unlock:
3191         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3192         mutex_unlock(&mrioc->init_cmds.mutex);
3193
3194 out:
3195         if (drv_info)
3196                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
3197                     data_dma);
3198
3199         return retval;
3200 }
3201
3202 /**
3203  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3204  * @mrioc: Adapter instance reference
3205  * @event: MPI event ID
3206  *
3207  * Un mask the specific event by resetting the event_mask
3208  * bitmap.
3209  *
3210  * Return: 0 on success, non-zero on failures.
3211  */
3212 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
3213 {
3214         u32 desired_event;
3215         u8 word;
3216
3217         if (event >= 128)
3218                 return;
3219
3220         desired_event = (1 << (event % 32));
3221         word = event / 32;
3222
3223         mrioc->event_masks[word] &= ~desired_event;
3224 }
3225
3226 /**
3227  * mpi3mr_issue_event_notification - Send event notification
3228  * @mrioc: Adapter instance reference
3229  *
3230  * Issue event notification MPI request through admin queue and
3231  * wait for the completion of it or time out.
3232  *
3233  * Return: 0 on success, non-zero on failures.
3234  */
3235 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
3236 {
3237         struct mpi3_event_notification_request evtnotify_req;
3238         int retval = 0;
3239         u8 i;
3240
3241         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
3242         mutex_lock(&mrioc->init_cmds.mutex);
3243         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3244                 retval = -1;
3245                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
3246                 mutex_unlock(&mrioc->init_cmds.mutex);
3247                 goto out;
3248         }
3249         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3250         mrioc->init_cmds.is_waiting = 1;
3251         mrioc->init_cmds.callback = NULL;
3252         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3253         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
3254         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3255                 evtnotify_req.event_masks[i] =
3256                     cpu_to_le32(mrioc->event_masks[i]);
3257         init_completion(&mrioc->init_cmds.done);
3258         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
3259             sizeof(evtnotify_req), 1);
3260         if (retval) {
3261                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
3262                 goto out_unlock;
3263         }
3264         wait_for_completion_timeout(&mrioc->init_cmds.done,
3265             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3266         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3267                 ioc_err(mrioc, "event notification timed out\n");
3268                 mpi3mr_check_rh_fault_ioc(mrioc,
3269                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
3270                 retval = -1;
3271                 goto out_unlock;
3272         }
3273         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3274             != MPI3_IOCSTATUS_SUCCESS) {
3275                 ioc_err(mrioc,
3276                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3277                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3278                     mrioc->init_cmds.ioc_loginfo);
3279                 retval = -1;
3280                 goto out_unlock;
3281         }
3282
3283 out_unlock:
3284         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3285         mutex_unlock(&mrioc->init_cmds.mutex);
3286 out:
3287         return retval;
3288 }
3289
3290 /**
3291  * mpi3mr_process_event_ack - Process event acknowledgment
3292  * @mrioc: Adapter instance reference
3293  * @event: MPI3 event ID
3294  * @event_ctx: event context
3295  *
3296  * Send event acknowledgment through admin queue and wait for
3297  * it to complete.
3298  *
3299  * Return: 0 on success, non-zero on failures.
3300  */
3301 int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
3302         u32 event_ctx)
3303 {
3304         struct mpi3_event_ack_request evtack_req;
3305         int retval = 0;
3306
3307         memset(&evtack_req, 0, sizeof(evtack_req));
3308         mutex_lock(&mrioc->init_cmds.mutex);
3309         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3310                 retval = -1;
3311                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
3312                 mutex_unlock(&mrioc->init_cmds.mutex);
3313                 goto out;
3314         }
3315         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3316         mrioc->init_cmds.is_waiting = 1;
3317         mrioc->init_cmds.callback = NULL;
3318         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3319         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
3320         evtack_req.event = event;
3321         evtack_req.event_context = cpu_to_le32(event_ctx);
3322
3323         init_completion(&mrioc->init_cmds.done);
3324         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
3325             sizeof(evtack_req), 1);
3326         if (retval) {
3327                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
3328                 goto out_unlock;
3329         }
3330         wait_for_completion_timeout(&mrioc->init_cmds.done,
3331             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3332         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3333                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
3334                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
3335                         mpi3mr_soft_reset_handler(mrioc,
3336                             MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
3337                 retval = -1;
3338                 goto out_unlock;
3339         }
3340         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3341             != MPI3_IOCSTATUS_SUCCESS) {
3342                 ioc_err(mrioc,
3343                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3344                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3345                     mrioc->init_cmds.ioc_loginfo);
3346                 retval = -1;
3347                 goto out_unlock;
3348         }
3349
3350 out_unlock:
3351         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3352         mutex_unlock(&mrioc->init_cmds.mutex);
3353 out:
3354         return retval;
3355 }
3356
3357 /**
3358  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3359  * @mrioc: Adapter instance reference
3360  *
3361  * Allocate chain buffers and set a bitmap to indicate free
3362  * chain buffers. Chain buffers are used to pass the SGE
3363  * information along with MPI3 SCSI IO requests for host I/O.
3364  *
3365  * Return: 0 on success, non-zero on failure
3366  */
3367 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
3368 {
3369         int retval = 0;
3370         u32 sz, i;
3371         u16 num_chains;
3372
3373         if (mrioc->chain_sgl_list)
3374                 return retval;
3375
3376         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
3377
3378         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
3379             | SHOST_DIX_TYPE1_PROTECTION
3380             | SHOST_DIX_TYPE2_PROTECTION
3381             | SHOST_DIX_TYPE3_PROTECTION))
3382                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
3383
3384         mrioc->chain_buf_count = num_chains;
3385         sz = sizeof(struct chain_element) * num_chains;
3386         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
3387         if (!mrioc->chain_sgl_list)
3388                 goto out_failed;
3389
3390         sz = MPI3MR_PAGE_SIZE_4K;
3391         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
3392             &mrioc->pdev->dev, sz, 16, 0);
3393         if (!mrioc->chain_buf_pool) {
3394                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
3395                 goto out_failed;
3396         }
3397
3398         for (i = 0; i < num_chains; i++) {
3399                 mrioc->chain_sgl_list[i].addr =
3400                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
3401                     &mrioc->chain_sgl_list[i].dma_addr);
3402
3403                 if (!mrioc->chain_sgl_list[i].addr)
3404                         goto out_failed;
3405         }
3406         mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);
3407         if (!mrioc->chain_bitmap)
3408                 goto out_failed;
3409         return retval;
3410 out_failed:
3411         retval = -1;
3412         return retval;
3413 }
3414
3415 /**
3416  * mpi3mr_port_enable_complete - Mark port enable complete
3417  * @mrioc: Adapter instance reference
3418  * @drv_cmd: Internal command tracker
3419  *
3420  * Call back for asynchronous port enable request sets the
3421  * driver command to indicate port enable request is complete.
3422  *
3423  * Return: Nothing
3424  */
3425 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
3426         struct mpi3mr_drv_cmd *drv_cmd)
3427 {
3428         drv_cmd->callback = NULL;
3429         mrioc->scan_started = 0;
3430         if (drv_cmd->state & MPI3MR_CMD_RESET)
3431                 mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
3432         else
3433                 mrioc->scan_failed = drv_cmd->ioc_status;
3434         drv_cmd->state = MPI3MR_CMD_NOTUSED;
3435 }
3436
3437 /**
3438  * mpi3mr_issue_port_enable - Issue Port Enable
3439  * @mrioc: Adapter instance reference
3440  * @async: Flag to wait for completion or not
3441  *
3442  * Issue Port Enable MPI request through admin queue and if the
3443  * async flag is not set wait for the completion of the port
3444  * enable or time out.
3445  *
3446  * Return: 0 on success, non-zero on failures.
3447  */
3448 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
3449 {
3450         struct mpi3_port_enable_request pe_req;
3451         int retval = 0;
3452         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
3453
3454         memset(&pe_req, 0, sizeof(pe_req));
3455         mutex_lock(&mrioc->init_cmds.mutex);
3456         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3457                 retval = -1;
3458                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
3459                 mutex_unlock(&mrioc->init_cmds.mutex);
3460                 goto out;
3461         }
3462         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3463         if (async) {
3464                 mrioc->init_cmds.is_waiting = 0;
3465                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
3466         } else {
3467                 mrioc->init_cmds.is_waiting = 1;
3468                 mrioc->init_cmds.callback = NULL;
3469                 init_completion(&mrioc->init_cmds.done);
3470         }
3471         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3472         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
3473
3474         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
3475         if (retval) {
3476                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
3477                 goto out_unlock;
3478         }
3479         if (async) {
3480                 mutex_unlock(&mrioc->init_cmds.mutex);
3481                 goto out;
3482         }
3483
3484         wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));
3485         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3486                 ioc_err(mrioc, "port enable timed out\n");
3487                 retval = -1;
3488                 mpi3mr_check_rh_fault_ioc(mrioc, MPI3MR_RESET_FROM_PE_TIMEOUT);
3489                 goto out_unlock;
3490         }
3491         mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
3492
3493 out_unlock:
3494         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3495         mutex_unlock(&mrioc->init_cmds.mutex);
3496 out:
3497         return retval;
3498 }
3499
3500 /* Protocol type to name mapper structure */
3501 static const struct {
3502         u8 protocol;
3503         char *name;
3504 } mpi3mr_protocols[] = {
3505         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
3506         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3507         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3508 };
3509
3510 /* Capability to name mapper structure*/
3511 static const struct {
3512         u32 capability;
3513         char *name;
3514 } mpi3mr_capabilities[] = {
3515         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3516         { MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED, "MultiPath" },
3517 };
3518
3519 /**
3520  * mpi3mr_print_ioc_info - Display controller information
3521  * @mrioc: Adapter instance reference
3522  *
3523  * Display controller personalit, capability, supported
3524  * protocols etc.
3525  *
3526  * Return: Nothing
3527  */
3528 static void
3529 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3530 {
3531         int i = 0, bytes_written = 0;
3532         char personality[16];
3533         char protocol[50] = {0};
3534         char capabilities[100] = {0};
3535         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3536
3537         switch (mrioc->facts.personality) {
3538         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3539                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3540                 break;
3541         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3542                 strncpy(personality, "RAID", sizeof(personality));
3543                 break;
3544         default:
3545                 strncpy(personality, "Unknown", sizeof(personality));
3546                 break;
3547         }
3548
3549         ioc_info(mrioc, "Running in %s Personality", personality);
3550
3551         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3552             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3553             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3554
3555         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3556                 if (mrioc->facts.protocol_flags &
3557                     mpi3mr_protocols[i].protocol) {
3558                         bytes_written += scnprintf(protocol + bytes_written,
3559                                     sizeof(protocol) - bytes_written, "%s%s",
3560                                     bytes_written ? "," : "",
3561                                     mpi3mr_protocols[i].name);
3562                 }
3563         }
3564
3565         bytes_written = 0;
3566         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3567                 if (mrioc->facts.protocol_flags &
3568                     mpi3mr_capabilities[i].capability) {
3569                         bytes_written += scnprintf(capabilities + bytes_written,
3570                                     sizeof(capabilities) - bytes_written, "%s%s",
3571                                     bytes_written ? "," : "",
3572                                     mpi3mr_capabilities[i].name);
3573                 }
3574         }
3575
3576         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3577                  protocol, capabilities);
3578 }
3579
3580 /**
3581  * mpi3mr_cleanup_resources - Free PCI resources
3582  * @mrioc: Adapter instance reference
3583  *
3584  * Unmap PCI device memory and disable PCI device.
3585  *
3586  * Return: 0 on success and non-zero on failure.
3587  */
3588 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3589 {
3590         struct pci_dev *pdev = mrioc->pdev;
3591
3592         mpi3mr_cleanup_isr(mrioc);
3593
3594         if (mrioc->sysif_regs) {
3595                 iounmap((void __iomem *)mrioc->sysif_regs);
3596                 mrioc->sysif_regs = NULL;
3597         }
3598
3599         if (pci_is_enabled(pdev)) {
3600                 if (mrioc->bars)
3601                         pci_release_selected_regions(pdev, mrioc->bars);
3602                 pci_disable_device(pdev);
3603         }
3604 }
3605
3606 /**
3607  * mpi3mr_setup_resources - Enable PCI resources
3608  * @mrioc: Adapter instance reference
3609  *
3610  * Enable PCI device memory, MSI-x registers and set DMA mask.
3611  *
3612  * Return: 0 on success and non-zero on failure.
3613  */
3614 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3615 {
3616         struct pci_dev *pdev = mrioc->pdev;
3617         u32 memap_sz = 0;
3618         int i, retval = 0, capb = 0;
3619         u16 message_control;
3620         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3621             (((dma_get_required_mask(&pdev->dev) > DMA_BIT_MASK(32)) &&
3622             (sizeof(dma_addr_t) > 4)) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3623
3624         if (pci_enable_device_mem(pdev)) {
3625                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3626                 retval = -ENODEV;
3627                 goto out_failed;
3628         }
3629
3630         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3631         if (!capb) {
3632                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3633                 retval = -ENODEV;
3634                 goto out_failed;
3635         }
3636         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3637
3638         if (pci_request_selected_regions(pdev, mrioc->bars,
3639             mrioc->driver_name)) {
3640                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3641                 retval = -ENODEV;
3642                 goto out_failed;
3643         }
3644
3645         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3646                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3647                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3648                         memap_sz = pci_resource_len(pdev, i);
3649                         mrioc->sysif_regs =
3650                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3651                         break;
3652                 }
3653         }
3654
3655         pci_set_master(pdev);
3656
3657         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3658         if (retval) {
3659                 if (dma_mask != DMA_BIT_MASK(32)) {
3660                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3661                         dma_mask = DMA_BIT_MASK(32);
3662                         retval = dma_set_mask_and_coherent(&pdev->dev,
3663                             dma_mask);
3664                 }
3665                 if (retval) {
3666                         mrioc->dma_mask = 0;
3667                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3668                         goto out_failed;
3669                 }
3670         }
3671         mrioc->dma_mask = dma_mask;
3672
3673         if (!mrioc->sysif_regs) {
3674                 ioc_err(mrioc,
3675                     "Unable to map adapter memory or resource not found\n");
3676                 retval = -EINVAL;
3677                 goto out_failed;
3678         }
3679
3680         pci_read_config_word(pdev, capb + 2, &message_control);
3681         mrioc->msix_count = (message_control & 0x3FF) + 1;
3682
3683         pci_save_state(pdev);
3684
3685         pci_set_drvdata(pdev, mrioc->shost);
3686
3687         mpi3mr_ioc_disable_intr(mrioc);
3688
3689         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3690             (unsigned long long)mrioc->sysif_regs_phys,
3691             mrioc->sysif_regs, memap_sz);
3692         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3693             mrioc->msix_count);
3694
3695         if (!reset_devices && poll_queues > 0)
3696                 mrioc->requested_poll_qcount = min_t(int, poll_queues,
3697                                 mrioc->msix_count - 2);
3698         return retval;
3699
3700 out_failed:
3701         mpi3mr_cleanup_resources(mrioc);
3702         return retval;
3703 }
3704
3705 /**
3706  * mpi3mr_enable_events - Enable required events
3707  * @mrioc: Adapter instance reference
3708  *
3709  * This routine unmasks the events required by the driver by
3710  * sennding appropriate event mask bitmapt through an event
3711  * notification request.
3712  *
3713  * Return: 0 on success and non-zero on failure.
3714  */
3715 static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc)
3716 {
3717         int retval = 0;
3718         u32  i;
3719
3720         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3721                 mrioc->event_masks[i] = -1;
3722
3723         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3724         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3725         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3726         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3727         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_ADDED);
3728         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3729         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3730         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3731         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3732         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3733         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3734         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PREPARE_FOR_RESET);
3735         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3736         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3737
3738         retval = mpi3mr_issue_event_notification(mrioc);
3739         if (retval)
3740                 ioc_err(mrioc, "failed to issue event notification %d\n",
3741                     retval);
3742         return retval;
3743 }
3744
3745 /**
3746  * mpi3mr_init_ioc - Initialize the controller
3747  * @mrioc: Adapter instance reference
3748  *
3749  * This the controller initialization routine, executed either
3750  * after soft reset or from pci probe callback.
3751  * Setup the required resources, memory map the controller
3752  * registers, create admin and operational reply queue pairs,
3753  * allocate required memory for reply pool, sense buffer pool,
3754  * issue IOC init request to the firmware, unmask the events and
3755  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3756  * volumes.
3757  *
3758  * Return: 0 on success and non-zero on failure.
3759  */
3760 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
3761 {
3762         int retval = 0;
3763         u8 retry = 0;
3764         struct mpi3_ioc_facts_data facts_data;
3765         u32 sz;
3766
3767 retry_init:
3768         retval = mpi3mr_bring_ioc_ready(mrioc);
3769         if (retval) {
3770                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3771                     retval);
3772                 goto out_failed_noretry;
3773         }
3774
3775         retval = mpi3mr_setup_isr(mrioc, 1);
3776         if (retval) {
3777                 ioc_err(mrioc, "Failed to setup ISR error %d\n",
3778                     retval);
3779                 goto out_failed_noretry;
3780         }
3781
3782         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3783         if (retval) {
3784                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3785                     retval);
3786                 goto out_failed;
3787         }
3788
3789         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
3790
3791         mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;
3792         atomic_set(&mrioc->pend_large_data_sz, 0);
3793
3794         if (reset_devices)
3795                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
3796                     MPI3MR_HOST_IOS_KDUMP);
3797
3798         if (!(mrioc->facts.ioc_capabilities &
3799             MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED)) {
3800                 mrioc->sas_transport_enabled = 1;
3801                 mrioc->scsi_device_channel = 1;
3802                 mrioc->shost->max_channel = 1;
3803                 mrioc->shost->transportt = mpi3mr_transport_template;
3804         }
3805
3806         mrioc->reply_sz = mrioc->facts.reply_sz;
3807
3808         retval = mpi3mr_check_reset_dma_mask(mrioc);
3809         if (retval) {
3810                 ioc_err(mrioc, "Resetting dma mask failed %d\n",
3811                     retval);
3812                 goto out_failed_noretry;
3813         }
3814
3815         mpi3mr_print_ioc_info(mrioc);
3816
3817         dprint_init(mrioc, "allocating config page buffers\n");
3818         mrioc->cfg_page = dma_alloc_coherent(&mrioc->pdev->dev,
3819             MPI3MR_DEFAULT_CFG_PAGE_SZ, &mrioc->cfg_page_dma, GFP_KERNEL);
3820         if (!mrioc->cfg_page)
3821                 goto out_failed_noretry;
3822
3823         mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ;
3824
3825         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3826         if (retval) {
3827                 ioc_err(mrioc,
3828                     "%s :Failed to allocated reply sense buffers %d\n",
3829                     __func__, retval);
3830                 goto out_failed_noretry;
3831         }
3832
3833         retval = mpi3mr_alloc_chain_bufs(mrioc);
3834         if (retval) {
3835                 ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3836                     retval);
3837                 goto out_failed_noretry;
3838         }
3839
3840         retval = mpi3mr_issue_iocinit(mrioc);
3841         if (retval) {
3842                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3843                     retval);
3844                 goto out_failed;
3845         }
3846
3847         retval = mpi3mr_print_pkg_ver(mrioc);
3848         if (retval) {
3849                 ioc_err(mrioc, "failed to get package version\n");
3850                 goto out_failed;
3851         }
3852
3853         retval = mpi3mr_setup_isr(mrioc, 0);
3854         if (retval) {
3855                 ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3856                     retval);
3857                 goto out_failed_noretry;
3858         }
3859
3860         retval = mpi3mr_create_op_queues(mrioc);
3861         if (retval) {
3862                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3863                     retval);
3864                 goto out_failed;
3865         }
3866
3867         if (!mrioc->pel_seqnum_virt) {
3868                 dprint_init(mrioc, "allocating memory for pel_seqnum_virt\n");
3869                 mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
3870                 mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
3871                     mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
3872                     GFP_KERNEL);
3873                 if (!mrioc->pel_seqnum_virt) {
3874                         retval = -ENOMEM;
3875                         goto out_failed_noretry;
3876                 }
3877         }
3878
3879         if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {
3880                 dprint_init(mrioc, "allocating memory for throttle groups\n");
3881                 sz = sizeof(struct mpi3mr_throttle_group_info);
3882                 mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);
3883                 if (!mrioc->throttle_groups)
3884                         goto out_failed_noretry;
3885         }
3886
3887         retval = mpi3mr_enable_events(mrioc);
3888         if (retval) {
3889                 ioc_err(mrioc, "failed to enable events %d\n",
3890                     retval);
3891                 goto out_failed;
3892         }
3893
3894         ioc_info(mrioc, "controller initialization completed successfully\n");
3895         return retval;
3896 out_failed:
3897         if (retry < 2) {
3898                 retry++;
3899                 ioc_warn(mrioc, "retrying controller initialization, retry_count:%d\n",
3900                     retry);
3901                 mpi3mr_memset_buffers(mrioc);
3902                 goto retry_init;
3903         }
3904 out_failed_noretry:
3905         ioc_err(mrioc, "controller initialization failed\n");
3906         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
3907             MPI3MR_RESET_FROM_CTLR_CLEANUP);
3908         mrioc->unrecoverable = 1;
3909         return retval;
3910 }
3911
3912 /**
3913  * mpi3mr_reinit_ioc - Re-Initialize the controller
3914  * @mrioc: Adapter instance reference
3915  * @is_resume: Called from resume or reset path
3916  *
3917  * This the controller re-initialization routine, executed from
3918  * the soft reset handler or resume callback. Creates
3919  * operational reply queue pairs, allocate required memory for
3920  * reply pool, sense buffer pool, issue IOC init request to the
3921  * firmware, unmask the events and issue port enable to discover
3922  * SAS/SATA/NVMe devices and RAID volumes.
3923  *
3924  * Return: 0 on success and non-zero on failure.
3925  */
3926 int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
3927 {
3928         int retval = 0;
3929         u8 retry = 0;
3930         struct mpi3_ioc_facts_data facts_data;
3931         u32 pe_timeout, ioc_status;
3932
3933 retry_init:
3934         pe_timeout =
3935             (MPI3MR_PORTENABLE_TIMEOUT / MPI3MR_PORTENABLE_POLL_INTERVAL);
3936
3937         dprint_reset(mrioc, "bringing up the controller to ready state\n");
3938         retval = mpi3mr_bring_ioc_ready(mrioc);
3939         if (retval) {
3940                 ioc_err(mrioc, "failed to bring to ready state\n");
3941                 goto out_failed_noretry;
3942         }
3943
3944         if (is_resume) {
3945                 dprint_reset(mrioc, "setting up single ISR\n");
3946                 retval = mpi3mr_setup_isr(mrioc, 1);
3947                 if (retval) {
3948                         ioc_err(mrioc, "failed to setup ISR\n");
3949                         goto out_failed_noretry;
3950                 }
3951         } else
3952                 mpi3mr_ioc_enable_intr(mrioc);
3953
3954         dprint_reset(mrioc, "getting ioc_facts\n");
3955         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3956         if (retval) {
3957                 ioc_err(mrioc, "failed to get ioc_facts\n");
3958                 goto out_failed;
3959         }
3960
3961         dprint_reset(mrioc, "validating ioc_facts\n");
3962         retval = mpi3mr_revalidate_factsdata(mrioc);
3963         if (retval) {
3964                 ioc_err(mrioc, "failed to revalidate ioc_facts data\n");
3965                 goto out_failed_noretry;
3966         }
3967
3968         mpi3mr_print_ioc_info(mrioc);
3969
3970         dprint_reset(mrioc, "sending ioc_init\n");
3971         retval = mpi3mr_issue_iocinit(mrioc);
3972         if (retval) {
3973                 ioc_err(mrioc, "failed to send ioc_init\n");
3974                 goto out_failed;
3975         }
3976
3977         dprint_reset(mrioc, "getting package version\n");
3978         retval = mpi3mr_print_pkg_ver(mrioc);
3979         if (retval) {
3980                 ioc_err(mrioc, "failed to get package version\n");
3981                 goto out_failed;
3982         }
3983
3984         if (is_resume) {
3985                 dprint_reset(mrioc, "setting up multiple ISR\n");
3986                 retval = mpi3mr_setup_isr(mrioc, 0);
3987                 if (retval) {
3988                         ioc_err(mrioc, "failed to re-setup ISR\n");
3989                         goto out_failed_noretry;
3990                 }
3991         }
3992
3993         dprint_reset(mrioc, "creating operational queue pairs\n");
3994         retval = mpi3mr_create_op_queues(mrioc);
3995         if (retval) {
3996                 ioc_err(mrioc, "failed to create operational queue pairs\n");
3997                 goto out_failed;
3998         }
3999
4000         if (!mrioc->pel_seqnum_virt) {
4001                 dprint_reset(mrioc, "allocating memory for pel_seqnum_virt\n");
4002                 mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4003                 mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
4004                     mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
4005                     GFP_KERNEL);
4006                 if (!mrioc->pel_seqnum_virt) {
4007                         retval = -ENOMEM;
4008                         goto out_failed_noretry;
4009                 }
4010         }
4011
4012         if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {
4013                 ioc_err(mrioc,
4014                     "cannot create minimum number of operational queues expected:%d created:%d\n",
4015                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
4016                 goto out_failed_noretry;
4017         }
4018
4019         dprint_reset(mrioc, "enabling events\n");
4020         retval = mpi3mr_enable_events(mrioc);
4021         if (retval) {
4022                 ioc_err(mrioc, "failed to enable events\n");
4023                 goto out_failed;
4024         }
4025
4026         mrioc->device_refresh_on = 1;
4027         mpi3mr_add_event_wait_for_device_refresh(mrioc);
4028
4029         ioc_info(mrioc, "sending port enable\n");
4030         retval = mpi3mr_issue_port_enable(mrioc, 1);
4031         if (retval) {
4032                 ioc_err(mrioc, "failed to issue port enable\n");
4033                 goto out_failed;
4034         }
4035         do {
4036                 ssleep(MPI3MR_PORTENABLE_POLL_INTERVAL);
4037                 if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)
4038                         break;
4039                 if (!pci_device_is_present(mrioc->pdev))
4040                         mrioc->unrecoverable = 1;
4041                 if (mrioc->unrecoverable) {
4042                         retval = -1;
4043                         goto out_failed_noretry;
4044                 }
4045                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4046                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
4047                     (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
4048                         mpi3mr_print_fault_info(mrioc);
4049                         mrioc->init_cmds.is_waiting = 0;
4050                         mrioc->init_cmds.callback = NULL;
4051                         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4052                         goto out_failed;
4053                 }
4054         } while (--pe_timeout);
4055
4056         if (!pe_timeout) {
4057                 ioc_err(mrioc, "port enable timed out\n");
4058                 mpi3mr_check_rh_fault_ioc(mrioc,
4059                     MPI3MR_RESET_FROM_PE_TIMEOUT);
4060                 mrioc->init_cmds.is_waiting = 0;
4061                 mrioc->init_cmds.callback = NULL;
4062                 mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4063                 goto out_failed;
4064         } else if (mrioc->scan_failed) {
4065                 ioc_err(mrioc,
4066                     "port enable failed with status=0x%04x\n",
4067                     mrioc->scan_failed);
4068         } else
4069                 ioc_info(mrioc, "port enable completed successfully\n");
4070
4071         ioc_info(mrioc, "controller %s completed successfully\n",
4072             (is_resume)?"resume":"re-initialization");
4073         return retval;
4074 out_failed:
4075         if (retry < 2) {
4076                 retry++;
4077                 ioc_warn(mrioc, "retrying controller %s, retry_count:%d\n",
4078                     (is_resume)?"resume":"re-initialization", retry);
4079                 mpi3mr_memset_buffers(mrioc);
4080                 goto retry_init;
4081         }
4082 out_failed_noretry:
4083         ioc_err(mrioc, "controller %s is failed\n",
4084             (is_resume)?"resume":"re-initialization");
4085         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4086             MPI3MR_RESET_FROM_CTLR_CLEANUP);
4087         mrioc->unrecoverable = 1;
4088         return retval;
4089 }
4090
4091 /**
4092  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4093  *                                      segments
4094  * @mrioc: Adapter instance reference
4095  * @qidx: Operational reply queue index
4096  *
4097  * Return: Nothing.
4098  */
4099 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4100 {
4101         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
4102         struct segments *segments;
4103         int i, size;
4104
4105         if (!op_reply_q->q_segments)
4106                 return;
4107
4108         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
4109         segments = op_reply_q->q_segments;
4110         for (i = 0; i < op_reply_q->num_segments; i++)
4111                 memset(segments[i].segment, 0, size);
4112 }
4113
4114 /**
4115  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4116  *                                      segments
4117  * @mrioc: Adapter instance reference
4118  * @qidx: Operational request queue index
4119  *
4120  * Return: Nothing.
4121  */
4122 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4123 {
4124         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
4125         struct segments *segments;
4126         int i, size;
4127
4128         if (!op_req_q->q_segments)
4129                 return;
4130
4131         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
4132         segments = op_req_q->q_segments;
4133         for (i = 0; i < op_req_q->num_segments; i++)
4134                 memset(segments[i].segment, 0, size);
4135 }
4136
4137 /**
4138  * mpi3mr_memset_buffers - memset memory for a controller
4139  * @mrioc: Adapter instance reference
4140  *
4141  * clear all the memory allocated for a controller, typically
4142  * called post reset to reuse the memory allocated during the
4143  * controller init.
4144  *
4145  * Return: Nothing.
4146  */
4147 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
4148 {
4149         u16 i;
4150         struct mpi3mr_throttle_group_info *tg;
4151
4152         mrioc->change_count = 0;
4153         mrioc->active_poll_qcount = 0;
4154         mrioc->default_qcount = 0;
4155         if (mrioc->admin_req_base)
4156                 memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
4157         if (mrioc->admin_reply_base)
4158                 memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
4159
4160         if (mrioc->init_cmds.reply) {
4161                 memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
4162                 memset(mrioc->bsg_cmds.reply, 0,
4163                     sizeof(*mrioc->bsg_cmds.reply));
4164                 memset(mrioc->host_tm_cmds.reply, 0,
4165                     sizeof(*mrioc->host_tm_cmds.reply));
4166                 memset(mrioc->pel_cmds.reply, 0,
4167                     sizeof(*mrioc->pel_cmds.reply));
4168                 memset(mrioc->pel_abort_cmd.reply, 0,
4169                     sizeof(*mrioc->pel_abort_cmd.reply));
4170                 memset(mrioc->transport_cmds.reply, 0,
4171                     sizeof(*mrioc->transport_cmds.reply));
4172                 for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
4173                         memset(mrioc->dev_rmhs_cmds[i].reply, 0,
4174                             sizeof(*mrioc->dev_rmhs_cmds[i].reply));
4175                 for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++)
4176                         memset(mrioc->evtack_cmds[i].reply, 0,
4177                             sizeof(*mrioc->evtack_cmds[i].reply));
4178                 bitmap_clear(mrioc->removepend_bitmap, 0,
4179                              mrioc->dev_handle_bitmap_bits);
4180                 bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4181                 bitmap_clear(mrioc->evtack_cmds_bitmap, 0,
4182                              MPI3MR_NUM_EVTACKCMD);
4183         }
4184
4185         for (i = 0; i < mrioc->num_queues; i++) {
4186                 mrioc->op_reply_qinfo[i].qid = 0;
4187                 mrioc->op_reply_qinfo[i].ci = 0;
4188                 mrioc->op_reply_qinfo[i].num_replies = 0;
4189                 mrioc->op_reply_qinfo[i].ephase = 0;
4190                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
4191                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
4192                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
4193
4194                 mrioc->req_qinfo[i].ci = 0;
4195                 mrioc->req_qinfo[i].pi = 0;
4196                 mrioc->req_qinfo[i].num_requests = 0;
4197                 mrioc->req_qinfo[i].qid = 0;
4198                 mrioc->req_qinfo[i].reply_qid = 0;
4199                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
4200                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
4201         }
4202
4203         atomic_set(&mrioc->pend_large_data_sz, 0);
4204         if (mrioc->throttle_groups) {
4205                 tg = mrioc->throttle_groups;
4206                 for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {
4207                         tg->id = 0;
4208                         tg->fw_qd = 0;
4209                         tg->modified_qd = 0;
4210                         tg->io_divert = 0;
4211                         tg->need_qd_reduction = 0;
4212                         tg->high = 0;
4213                         tg->low = 0;
4214                         tg->qd_reduction = 0;
4215                         atomic_set(&tg->pend_large_data_sz, 0);
4216                 }
4217         }
4218 }
4219
4220 /**
4221  * mpi3mr_free_mem - Free memory allocated for a controller
4222  * @mrioc: Adapter instance reference
4223  *
4224  * Free all the memory allocated for a controller.
4225  *
4226  * Return: Nothing.
4227  */
4228 void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
4229 {
4230         u16 i;
4231         struct mpi3mr_intr_info *intr_info;
4232
4233         mpi3mr_free_enclosure_list(mrioc);
4234
4235         if (mrioc->sense_buf_pool) {
4236                 if (mrioc->sense_buf)
4237                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
4238                             mrioc->sense_buf_dma);
4239                 dma_pool_destroy(mrioc->sense_buf_pool);
4240                 mrioc->sense_buf = NULL;
4241                 mrioc->sense_buf_pool = NULL;
4242         }
4243         if (mrioc->sense_buf_q_pool) {
4244                 if (mrioc->sense_buf_q)
4245                         dma_pool_free(mrioc->sense_buf_q_pool,
4246                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
4247                 dma_pool_destroy(mrioc->sense_buf_q_pool);
4248                 mrioc->sense_buf_q = NULL;
4249                 mrioc->sense_buf_q_pool = NULL;
4250         }
4251
4252         if (mrioc->reply_buf_pool) {
4253                 if (mrioc->reply_buf)
4254                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
4255                             mrioc->reply_buf_dma);
4256                 dma_pool_destroy(mrioc->reply_buf_pool);
4257                 mrioc->reply_buf = NULL;
4258                 mrioc->reply_buf_pool = NULL;
4259         }
4260         if (mrioc->reply_free_q_pool) {
4261                 if (mrioc->reply_free_q)
4262                         dma_pool_free(mrioc->reply_free_q_pool,
4263                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
4264                 dma_pool_destroy(mrioc->reply_free_q_pool);
4265                 mrioc->reply_free_q = NULL;
4266                 mrioc->reply_free_q_pool = NULL;
4267         }
4268
4269         for (i = 0; i < mrioc->num_op_req_q; i++)
4270                 mpi3mr_free_op_req_q_segments(mrioc, i);
4271
4272         for (i = 0; i < mrioc->num_op_reply_q; i++)
4273                 mpi3mr_free_op_reply_q_segments(mrioc, i);
4274
4275         for (i = 0; i < mrioc->intr_info_count; i++) {
4276                 intr_info = mrioc->intr_info + i;
4277                 intr_info->op_reply_q = NULL;
4278         }
4279
4280         kfree(mrioc->req_qinfo);
4281         mrioc->req_qinfo = NULL;
4282         mrioc->num_op_req_q = 0;
4283
4284         kfree(mrioc->op_reply_qinfo);
4285         mrioc->op_reply_qinfo = NULL;
4286         mrioc->num_op_reply_q = 0;
4287
4288         kfree(mrioc->init_cmds.reply);
4289         mrioc->init_cmds.reply = NULL;
4290
4291         kfree(mrioc->bsg_cmds.reply);
4292         mrioc->bsg_cmds.reply = NULL;
4293
4294         kfree(mrioc->host_tm_cmds.reply);
4295         mrioc->host_tm_cmds.reply = NULL;
4296
4297         kfree(mrioc->pel_cmds.reply);
4298         mrioc->pel_cmds.reply = NULL;
4299
4300         kfree(mrioc->pel_abort_cmd.reply);
4301         mrioc->pel_abort_cmd.reply = NULL;
4302
4303         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4304                 kfree(mrioc->evtack_cmds[i].reply);
4305                 mrioc->evtack_cmds[i].reply = NULL;
4306         }
4307
4308         bitmap_free(mrioc->removepend_bitmap);
4309         mrioc->removepend_bitmap = NULL;
4310
4311         bitmap_free(mrioc->devrem_bitmap);
4312         mrioc->devrem_bitmap = NULL;
4313
4314         bitmap_free(mrioc->evtack_cmds_bitmap);
4315         mrioc->evtack_cmds_bitmap = NULL;
4316
4317         bitmap_free(mrioc->chain_bitmap);
4318         mrioc->chain_bitmap = NULL;
4319
4320         kfree(mrioc->transport_cmds.reply);
4321         mrioc->transport_cmds.reply = NULL;
4322
4323         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4324                 kfree(mrioc->dev_rmhs_cmds[i].reply);
4325                 mrioc->dev_rmhs_cmds[i].reply = NULL;
4326         }
4327
4328         if (mrioc->chain_buf_pool) {
4329                 for (i = 0; i < mrioc->chain_buf_count; i++) {
4330                         if (mrioc->chain_sgl_list[i].addr) {
4331                                 dma_pool_free(mrioc->chain_buf_pool,
4332                                     mrioc->chain_sgl_list[i].addr,
4333                                     mrioc->chain_sgl_list[i].dma_addr);
4334                                 mrioc->chain_sgl_list[i].addr = NULL;
4335                         }
4336                 }
4337                 dma_pool_destroy(mrioc->chain_buf_pool);
4338                 mrioc->chain_buf_pool = NULL;
4339         }
4340
4341         kfree(mrioc->chain_sgl_list);
4342         mrioc->chain_sgl_list = NULL;
4343
4344         if (mrioc->admin_reply_base) {
4345                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
4346                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
4347                 mrioc->admin_reply_base = NULL;
4348         }
4349         if (mrioc->admin_req_base) {
4350                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
4351                     mrioc->admin_req_base, mrioc->admin_req_dma);
4352                 mrioc->admin_req_base = NULL;
4353         }
4354
4355         if (mrioc->pel_seqnum_virt) {
4356                 dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,
4357                     mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);
4358                 mrioc->pel_seqnum_virt = NULL;
4359         }
4360
4361         kfree(mrioc->logdata_buf);
4362         mrioc->logdata_buf = NULL;
4363
4364 }
4365
4366 /**
4367  * mpi3mr_issue_ioc_shutdown - shutdown controller
4368  * @mrioc: Adapter instance reference
4369  *
4370  * Send shutodwn notification to the controller and wait for the
4371  * shutdown_timeout for it to be completed.
4372  *
4373  * Return: Nothing.
4374  */
4375 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
4376 {
4377         u32 ioc_config, ioc_status;
4378         u8 retval = 1;
4379         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
4380
4381         ioc_info(mrioc, "Issuing shutdown Notification\n");
4382         if (mrioc->unrecoverable) {
4383                 ioc_warn(mrioc,
4384                     "IOC is unrecoverable shutdown is not issued\n");
4385                 return;
4386         }
4387         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4388         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4389             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
4390                 ioc_info(mrioc, "shutdown already in progress\n");
4391                 return;
4392         }
4393
4394         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4395         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
4396         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
4397
4398         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
4399
4400         if (mrioc->facts.shutdown_timeout)
4401                 timeout = mrioc->facts.shutdown_timeout * 10;
4402
4403         do {
4404                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4405                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4406                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
4407                         retval = 0;
4408                         break;
4409                 }
4410                 msleep(100);
4411         } while (--timeout);
4412
4413         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4414         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4415
4416         if (retval) {
4417                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4418                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
4419                         ioc_warn(mrioc,
4420                             "shutdown still in progress after timeout\n");
4421         }
4422
4423         ioc_info(mrioc,
4424             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
4425             (!retval) ? "successful" : "failed", ioc_status,
4426             ioc_config);
4427 }
4428
4429 /**
4430  * mpi3mr_cleanup_ioc - Cleanup controller
4431  * @mrioc: Adapter instance reference
4432  *
4433  * controller cleanup handler, Message unit reset or soft reset
4434  * and shutdown notification is issued to the controller.
4435  *
4436  * Return: Nothing.
4437  */
4438 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
4439 {
4440         enum mpi3mr_iocstate ioc_state;
4441
4442         dprint_exit(mrioc, "cleaning up the controller\n");
4443         mpi3mr_ioc_disable_intr(mrioc);
4444
4445         ioc_state = mpi3mr_get_iocstate(mrioc);
4446
4447         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
4448             (ioc_state == MRIOC_STATE_READY)) {
4449                 if (mpi3mr_issue_and_process_mur(mrioc,
4450                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
4451                         mpi3mr_issue_reset(mrioc,
4452                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
4453                             MPI3MR_RESET_FROM_MUR_FAILURE);
4454                 mpi3mr_issue_ioc_shutdown(mrioc);
4455         }
4456         dprint_exit(mrioc, "controller cleanup completed\n");
4457 }
4458
4459 /**
4460  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4461  * @mrioc: Adapter instance reference
4462  * @cmdptr: Internal command tracker
4463  *
4464  * Complete an internal driver commands with state indicating it
4465  * is completed due to reset.
4466  *
4467  * Return: Nothing.
4468  */
4469 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
4470         struct mpi3mr_drv_cmd *cmdptr)
4471 {
4472         if (cmdptr->state & MPI3MR_CMD_PENDING) {
4473                 cmdptr->state |= MPI3MR_CMD_RESET;
4474                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
4475                 if (cmdptr->is_waiting) {
4476                         complete(&cmdptr->done);
4477                         cmdptr->is_waiting = 0;
4478                 } else if (cmdptr->callback)
4479                         cmdptr->callback(mrioc, cmdptr);
4480         }
4481 }
4482
4483 /**
4484  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4485  * @mrioc: Adapter instance reference
4486  *
4487  * Flush all internal driver commands post reset
4488  *
4489  * Return: Nothing.
4490  */
4491 void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
4492 {
4493         struct mpi3mr_drv_cmd *cmdptr;
4494         u8 i;
4495
4496         cmdptr = &mrioc->init_cmds;
4497         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4498
4499         cmdptr = &mrioc->cfg_cmds;
4500         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4501
4502         cmdptr = &mrioc->bsg_cmds;
4503         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4504         cmdptr = &mrioc->host_tm_cmds;
4505         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4506
4507         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4508                 cmdptr = &mrioc->dev_rmhs_cmds[i];
4509                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4510         }
4511
4512         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4513                 cmdptr = &mrioc->evtack_cmds[i];
4514                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4515         }
4516
4517         cmdptr = &mrioc->pel_cmds;
4518         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4519
4520         cmdptr = &mrioc->pel_abort_cmd;
4521         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4522
4523         cmdptr = &mrioc->transport_cmds;
4524         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4525 }
4526
4527 /**
4528  * mpi3mr_pel_wait_post - Issue PEL Wait
4529  * @mrioc: Adapter instance reference
4530  * @drv_cmd: Internal command tracker
4531  *
4532  * Issue PEL Wait MPI request through admin queue and return.
4533  *
4534  * Return: Nothing.
4535  */
4536 static void mpi3mr_pel_wait_post(struct mpi3mr_ioc *mrioc,
4537         struct mpi3mr_drv_cmd *drv_cmd)
4538 {
4539         struct mpi3_pel_req_action_wait pel_wait;
4540
4541         mrioc->pel_abort_requested = false;
4542
4543         memset(&pel_wait, 0, sizeof(pel_wait));
4544         drv_cmd->state = MPI3MR_CMD_PENDING;
4545         drv_cmd->is_waiting = 0;
4546         drv_cmd->callback = mpi3mr_pel_wait_complete;
4547         drv_cmd->ioc_status = 0;
4548         drv_cmd->ioc_loginfo = 0;
4549         pel_wait.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4550         pel_wait.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4551         pel_wait.action = MPI3_PEL_ACTION_WAIT;
4552         pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);
4553         pel_wait.locale = cpu_to_le16(mrioc->pel_locale);
4554         pel_wait.class = cpu_to_le16(mrioc->pel_class);
4555         pel_wait.wait_time = MPI3_PEL_WAITTIME_INFINITE_WAIT;
4556         dprint_bsg_info(mrioc, "sending pel_wait seqnum(%d), class(%d), locale(0x%08x)\n",
4557             mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);
4558
4559         if (mpi3mr_admin_request_post(mrioc, &pel_wait, sizeof(pel_wait), 0)) {
4560                 dprint_bsg_err(mrioc,
4561                             "Issuing PELWait: Admin post failed\n");
4562                 drv_cmd->state = MPI3MR_CMD_NOTUSED;
4563                 drv_cmd->callback = NULL;
4564                 drv_cmd->retry_count = 0;
4565                 mrioc->pel_enabled = false;
4566         }
4567 }
4568
4569 /**
4570  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
4571  * @mrioc: Adapter instance reference
4572  * @drv_cmd: Internal command tracker
4573  *
4574  * Issue PEL get sequence number MPI request through admin queue
4575  * and return.
4576  *
4577  * Return: 0 on success, non-zero on failure.
4578  */
4579 int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc,
4580         struct mpi3mr_drv_cmd *drv_cmd)
4581 {
4582         struct mpi3_pel_req_action_get_sequence_numbers pel_getseq_req;
4583         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
4584         int retval = 0;
4585
4586         memset(&pel_getseq_req, 0, sizeof(pel_getseq_req));
4587         mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;
4588         mrioc->pel_cmds.is_waiting = 0;
4589         mrioc->pel_cmds.ioc_status = 0;
4590         mrioc->pel_cmds.ioc_loginfo = 0;
4591         mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;
4592         pel_getseq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4593         pel_getseq_req.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4594         pel_getseq_req.action = MPI3_PEL_ACTION_GET_SEQNUM;
4595         mpi3mr_add_sg_single(&pel_getseq_req.sgl, sgl_flags,
4596             mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);
4597
4598         retval = mpi3mr_admin_request_post(mrioc, &pel_getseq_req,
4599                         sizeof(pel_getseq_req), 0);
4600         if (retval) {
4601                 if (drv_cmd) {
4602                         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4603                         drv_cmd->callback = NULL;
4604                         drv_cmd->retry_count = 0;
4605                 }
4606                 mrioc->pel_enabled = false;
4607         }
4608
4609         return retval;
4610 }
4611
4612 /**
4613  * mpi3mr_pel_wait_complete - PELWait Completion callback
4614  * @mrioc: Adapter instance reference
4615  * @drv_cmd: Internal command tracker
4616  *
4617  * This is a callback handler for the PELWait request and
4618  * firmware completes a PELWait request when it is aborted or a
4619  * new PEL entry is available. This sends AEN to the application
4620  * and if the PELwait completion is not due to PELAbort then
4621  * this will send a request for new PEL Sequence number
4622  *
4623  * Return: Nothing.
4624  */
4625 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
4626         struct mpi3mr_drv_cmd *drv_cmd)
4627 {
4628         struct mpi3_pel_reply *pel_reply = NULL;
4629         u16 ioc_status, pe_log_status;
4630         bool do_retry = false;
4631
4632         if (drv_cmd->state & MPI3MR_CMD_RESET)
4633                 goto cleanup_drv_cmd;
4634
4635         ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4636         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4637                 ioc_err(mrioc, "%s: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
4638                         __func__, ioc_status, drv_cmd->ioc_loginfo);
4639                 dprint_bsg_err(mrioc,
4640                     "pel_wait: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4641                     ioc_status, drv_cmd->ioc_loginfo);
4642                 do_retry = true;
4643         }
4644
4645         if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4646                 pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4647
4648         if (!pel_reply) {
4649                 dprint_bsg_err(mrioc,
4650                     "pel_wait: failed due to no reply\n");
4651                 goto out_failed;
4652         }
4653
4654         pe_log_status = le16_to_cpu(pel_reply->pe_log_status);
4655         if ((pe_log_status != MPI3_PEL_STATUS_SUCCESS) &&
4656             (pe_log_status != MPI3_PEL_STATUS_ABORTED)) {
4657                 ioc_err(mrioc, "%s: Failed pe_log_status(0x%04x)\n",
4658                         __func__, pe_log_status);
4659                 dprint_bsg_err(mrioc,
4660                     "pel_wait: failed due to pel_log_status(0x%04x)\n",
4661                     pe_log_status);
4662                 do_retry = true;
4663         }
4664
4665         if (do_retry) {
4666                 if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4667                         drv_cmd->retry_count++;
4668                         dprint_bsg_err(mrioc, "pel_wait: retrying(%d)\n",
4669                             drv_cmd->retry_count);
4670                         mpi3mr_pel_wait_post(mrioc, drv_cmd);
4671                         return;
4672                 }
4673                 dprint_bsg_err(mrioc,
4674                     "pel_wait: failed after all retries(%d)\n",
4675                     drv_cmd->retry_count);
4676                 goto out_failed;
4677         }
4678         atomic64_inc(&event_counter);
4679         if (!mrioc->pel_abort_requested) {
4680                 mrioc->pel_cmds.retry_count = 0;
4681                 mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);
4682         }
4683
4684         return;
4685 out_failed:
4686         mrioc->pel_enabled = false;
4687 cleanup_drv_cmd:
4688         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4689         drv_cmd->callback = NULL;
4690         drv_cmd->retry_count = 0;
4691 }
4692
4693 /**
4694  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
4695  * @mrioc: Adapter instance reference
4696  * @drv_cmd: Internal command tracker
4697  *
4698  * This is a callback handler for the PEL get sequence number
4699  * request and a new PEL wait request will be issued to the
4700  * firmware from this
4701  *
4702  * Return: Nothing.
4703  */
4704 void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc,
4705         struct mpi3mr_drv_cmd *drv_cmd)
4706 {
4707         struct mpi3_pel_reply *pel_reply = NULL;
4708         struct mpi3_pel_seq *pel_seqnum_virt;
4709         u16 ioc_status;
4710         bool do_retry = false;
4711
4712         pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;
4713
4714         if (drv_cmd->state & MPI3MR_CMD_RESET)
4715                 goto cleanup_drv_cmd;
4716
4717         ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4718         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4719                 dprint_bsg_err(mrioc,
4720                     "pel_get_seqnum: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4721                     ioc_status, drv_cmd->ioc_loginfo);
4722                 do_retry = true;
4723         }
4724
4725         if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4726                 pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4727         if (!pel_reply) {
4728                 dprint_bsg_err(mrioc,
4729                     "pel_get_seqnum: failed due to no reply\n");
4730                 goto out_failed;
4731         }
4732
4733         if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {
4734                 dprint_bsg_err(mrioc,
4735                     "pel_get_seqnum: failed due to pel_log_status(0x%04x)\n",
4736                     le16_to_cpu(pel_reply->pe_log_status));
4737                 do_retry = true;
4738         }
4739
4740         if (do_retry) {
4741                 if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4742                         drv_cmd->retry_count++;
4743                         dprint_bsg_err(mrioc,
4744                             "pel_get_seqnum: retrying(%d)\n",
4745                             drv_cmd->retry_count);
4746                         mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd);
4747                         return;
4748                 }
4749
4750                 dprint_bsg_err(mrioc,
4751                     "pel_get_seqnum: failed after all retries(%d)\n",
4752                     drv_cmd->retry_count);
4753                 goto out_failed;
4754         }
4755         mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;
4756         drv_cmd->retry_count = 0;
4757         mpi3mr_pel_wait_post(mrioc, drv_cmd);
4758
4759         return;
4760 out_failed:
4761         mrioc->pel_enabled = false;
4762 cleanup_drv_cmd:
4763         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4764         drv_cmd->callback = NULL;
4765         drv_cmd->retry_count = 0;
4766 }
4767
4768 /**
4769  * mpi3mr_soft_reset_handler - Reset the controller
4770  * @mrioc: Adapter instance reference
4771  * @reset_reason: Reset reason code
4772  * @snapdump: Flag to generate snapdump in firmware or not
4773  *
4774  * This is an handler for recovering controller by issuing soft
4775  * reset are diag fault reset.  This is a blocking function and
4776  * when one reset is executed if any other resets they will be
4777  * blocked. All BSG requests will be blocked during the reset. If
4778  * controller reset is successful then the controller will be
4779  * reinitalized, otherwise the controller will be marked as not
4780  * recoverable
4781  *
4782  * In snapdump bit is set, the controller is issued with diag
4783  * fault reset so that the firmware can create a snap dump and
4784  * post that the firmware will result in F000 fault and the
4785  * driver will issue soft reset to recover from that.
4786  *
4787  * Return: 0 on success, non-zero on failure.
4788  */
4789 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
4790         u32 reset_reason, u8 snapdump)
4791 {
4792         int retval = 0, i;
4793         unsigned long flags;
4794         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
4795
4796         /* Block the reset handler until diag save in progress*/
4797         dprint_reset(mrioc,
4798             "soft_reset_handler: check and block on diagsave_timeout(%d)\n",
4799             mrioc->diagsave_timeout);
4800         while (mrioc->diagsave_timeout)
4801                 ssleep(1);
4802         /*
4803          * Block new resets until the currently executing one is finished and
4804          * return the status of the existing reset for all blocked resets
4805          */
4806         dprint_reset(mrioc, "soft_reset_handler: acquiring reset_mutex\n");
4807         if (!mutex_trylock(&mrioc->reset_mutex)) {
4808                 ioc_info(mrioc,
4809                     "controller reset triggered by %s is blocked due to another reset in progress\n",
4810                     mpi3mr_reset_rc_name(reset_reason));
4811                 do {
4812                         ssleep(1);
4813                 } while (mrioc->reset_in_progress == 1);
4814                 ioc_info(mrioc,
4815                     "returning previous reset result(%d) for the reset triggered by %s\n",
4816                     mrioc->prev_reset_result,
4817                     mpi3mr_reset_rc_name(reset_reason));
4818                 return mrioc->prev_reset_result;
4819         }
4820         ioc_info(mrioc, "controller reset is triggered by %s\n",
4821             mpi3mr_reset_rc_name(reset_reason));
4822
4823         mrioc->device_refresh_on = 0;
4824         mrioc->reset_in_progress = 1;
4825         mrioc->stop_bsgs = 1;
4826         mrioc->prev_reset_result = -1;
4827
4828         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
4829             (reset_reason != MPI3MR_RESET_FROM_FIRMWARE) &&
4830             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
4831                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
4832                         mrioc->event_masks[i] = -1;
4833
4834                 dprint_reset(mrioc, "soft_reset_handler: masking events\n");
4835                 mpi3mr_issue_event_notification(mrioc);
4836         }
4837
4838         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
4839
4840         mpi3mr_ioc_disable_intr(mrioc);
4841
4842         if (snapdump) {
4843                 mpi3mr_set_diagsave(mrioc);
4844                 retval = mpi3mr_issue_reset(mrioc,
4845                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
4846                 if (!retval) {
4847                         do {
4848                                 host_diagnostic =
4849                                     readl(&mrioc->sysif_regs->host_diagnostic);
4850                                 if (!(host_diagnostic &
4851                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
4852                                         break;
4853                                 msleep(100);
4854                         } while (--timeout);
4855                 }
4856         }
4857
4858         retval = mpi3mr_issue_reset(mrioc,
4859             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
4860         if (retval) {
4861                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
4862                 goto out;
4863         }
4864         if (mrioc->num_io_throttle_group !=
4865             mrioc->facts.max_io_throttle_group) {
4866                 ioc_err(mrioc,
4867                     "max io throttle group doesn't match old(%d), new(%d)\n",
4868                     mrioc->num_io_throttle_group,
4869                     mrioc->facts.max_io_throttle_group);
4870                 retval = -EPERM;
4871                 goto out;
4872         }
4873
4874         mpi3mr_flush_delayed_cmd_lists(mrioc);
4875         mpi3mr_flush_drv_cmds(mrioc);
4876         bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4877         bitmap_clear(mrioc->removepend_bitmap, 0,
4878                      mrioc->dev_handle_bitmap_bits);
4879         bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);
4880         mpi3mr_flush_host_io(mrioc);
4881         mpi3mr_cleanup_fwevt_list(mrioc);
4882         mpi3mr_invalidate_devhandles(mrioc);
4883         mpi3mr_free_enclosure_list(mrioc);
4884
4885         if (mrioc->prepare_for_reset) {
4886                 mrioc->prepare_for_reset = 0;
4887                 mrioc->prepare_for_reset_timeout_counter = 0;
4888         }
4889         mpi3mr_memset_buffers(mrioc);
4890         retval = mpi3mr_reinit_ioc(mrioc, 0);
4891         if (retval) {
4892                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
4893                     mrioc->name, reset_reason);
4894                 goto out;
4895         }
4896         ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
4897
4898 out:
4899         if (!retval) {
4900                 mrioc->diagsave_timeout = 0;
4901                 mrioc->reset_in_progress = 0;
4902                 mrioc->pel_abort_requested = 0;
4903                 if (mrioc->pel_enabled) {
4904                         mrioc->pel_cmds.retry_count = 0;
4905                         mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);
4906                 }
4907
4908                 mrioc->device_refresh_on = 0;
4909
4910                 mrioc->ts_update_counter = 0;
4911                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
4912                 if (mrioc->watchdog_work_q)
4913                         queue_delayed_work(mrioc->watchdog_work_q,
4914                             &mrioc->watchdog_work,
4915                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
4916                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
4917                 mrioc->stop_bsgs = 0;
4918                 if (mrioc->pel_enabled)
4919                         atomic64_inc(&event_counter);
4920         } else {
4921                 mpi3mr_issue_reset(mrioc,
4922                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
4923                 mrioc->device_refresh_on = 0;
4924                 mrioc->unrecoverable = 1;
4925                 mrioc->reset_in_progress = 0;
4926                 retval = -1;
4927                 mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
4928         }
4929         mrioc->prev_reset_result = retval;
4930         mutex_unlock(&mrioc->reset_mutex);
4931         ioc_info(mrioc, "controller reset is %s\n",
4932             ((retval == 0) ? "successful" : "failed"));
4933         return retval;
4934 }
4935
4936
4937 /**
4938  * mpi3mr_free_config_dma_memory - free memory for config page
4939  * @mrioc: Adapter instance reference
4940  * @mem_desc: memory descriptor structure
4941  *
4942  * Check whether the size of the buffer specified by the memory
4943  * descriptor is greater than the default page size if so then
4944  * free the memory pointed by the descriptor.
4945  *
4946  * Return: Nothing.
4947  */
4948 static void mpi3mr_free_config_dma_memory(struct mpi3mr_ioc *mrioc,
4949         struct dma_memory_desc *mem_desc)
4950 {
4951         if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) {
4952                 dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,
4953                     mem_desc->addr, mem_desc->dma_addr);
4954                 mem_desc->addr = NULL;
4955         }
4956 }
4957
4958 /**
4959  * mpi3mr_alloc_config_dma_memory - Alloc memory for config page
4960  * @mrioc: Adapter instance reference
4961  * @mem_desc: Memory descriptor to hold dma memory info
4962  *
4963  * This function allocates new dmaable memory or provides the
4964  * default config page dmaable memory based on the memory size
4965  * described by the descriptor.
4966  *
4967  * Return: 0 on success, non-zero on failure.
4968  */
4969 static int mpi3mr_alloc_config_dma_memory(struct mpi3mr_ioc *mrioc,
4970         struct dma_memory_desc *mem_desc)
4971 {
4972         if (mem_desc->size > mrioc->cfg_page_sz) {
4973                 mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,
4974                     mem_desc->size, &mem_desc->dma_addr, GFP_KERNEL);
4975                 if (!mem_desc->addr)
4976                         return -ENOMEM;
4977         } else {
4978                 mem_desc->addr = mrioc->cfg_page;
4979                 mem_desc->dma_addr = mrioc->cfg_page_dma;
4980                 memset(mem_desc->addr, 0, mrioc->cfg_page_sz);
4981         }
4982         return 0;
4983 }
4984
4985 /**
4986  * mpi3mr_post_cfg_req - Issue config requests and wait
4987  * @mrioc: Adapter instance reference
4988  * @cfg_req: Configuration request
4989  * @timeout: Timeout in seconds
4990  * @ioc_status: Pointer to return ioc status
4991  *
4992  * A generic function for posting MPI3 configuration request to
4993  * the firmware. This blocks for the completion of request for
4994  * timeout seconds and if the request times out this function
4995  * faults the controller with proper reason code.
4996  *
4997  * On successful completion of the request this function returns
4998  * appropriate ioc status from the firmware back to the caller.
4999  *
5000  * Return: 0 on success, non-zero on failure.
5001  */
5002 static int mpi3mr_post_cfg_req(struct mpi3mr_ioc *mrioc,
5003         struct mpi3_config_request *cfg_req, int timeout, u16 *ioc_status)
5004 {
5005         int retval = 0;
5006
5007         mutex_lock(&mrioc->cfg_cmds.mutex);
5008         if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {
5009                 retval = -1;
5010                 ioc_err(mrioc, "sending config request failed due to command in use\n");
5011                 mutex_unlock(&mrioc->cfg_cmds.mutex);
5012                 goto out;
5013         }
5014         mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;
5015         mrioc->cfg_cmds.is_waiting = 1;
5016         mrioc->cfg_cmds.callback = NULL;
5017         mrioc->cfg_cmds.ioc_status = 0;
5018         mrioc->cfg_cmds.ioc_loginfo = 0;
5019
5020         cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);
5021         cfg_req->function = MPI3_FUNCTION_CONFIG;
5022
5023         init_completion(&mrioc->cfg_cmds.done);
5024         dprint_cfg_info(mrioc, "posting config request\n");
5025         if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5026                 dprint_dump(cfg_req, sizeof(struct mpi3_config_request),
5027                     "mpi3_cfg_req");
5028         retval = mpi3mr_admin_request_post(mrioc, cfg_req, sizeof(*cfg_req), 1);
5029         if (retval) {
5030                 ioc_err(mrioc, "posting config request failed\n");
5031                 goto out_unlock;
5032         }
5033         wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));
5034         if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {
5035                 mpi3mr_check_rh_fault_ioc(mrioc,
5036                     MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT);
5037                 ioc_err(mrioc, "config request timed out\n");
5038                 retval = -1;
5039                 goto out_unlock;
5040         }
5041         *ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
5042         if ((*ioc_status) != MPI3_IOCSTATUS_SUCCESS)
5043                 dprint_cfg_err(mrioc,
5044                     "cfg_page request returned with ioc_status(0x%04x), log_info(0x%08x)\n",
5045                     *ioc_status, mrioc->cfg_cmds.ioc_loginfo);
5046
5047 out_unlock:
5048         mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;
5049         mutex_unlock(&mrioc->cfg_cmds.mutex);
5050
5051 out:
5052         return retval;
5053 }
5054
5055 /**
5056  * mpi3mr_process_cfg_req - config page request processor
5057  * @mrioc: Adapter instance reference
5058  * @cfg_req: Configuration request
5059  * @cfg_hdr: Configuration page header
5060  * @timeout: Timeout in seconds
5061  * @ioc_status: Pointer to return ioc status
5062  * @cfg_buf: Memory pointer to copy config page or header
5063  * @cfg_buf_sz: Size of the memory to get config page or header
5064  *
5065  * This is handler for config page read, write and config page
5066  * header read operations.
5067  *
5068  * This function expects the cfg_req to be populated with page
5069  * type, page number, action for the header read and with page
5070  * address for all other operations.
5071  *
5072  * The cfg_hdr can be passed as null for reading required header
5073  * details for read/write pages the cfg_hdr should point valid
5074  * configuration page header.
5075  *
5076  * This allocates dmaable memory based on the size of the config
5077  * buffer and set the SGE of the cfg_req.
5078  *
5079  * For write actions, the config page data has to be passed in
5080  * the cfg_buf and size of the data has to be mentioned in the
5081  * cfg_buf_sz.
5082  *
5083  * For read/header actions, on successful completion of the
5084  * request with successful ioc_status the data will be copied
5085  * into the cfg_buf limited to a minimum of actual page size and
5086  * cfg_buf_sz
5087  *
5088  *
5089  * Return: 0 on success, non-zero on failure.
5090  */
5091 static int mpi3mr_process_cfg_req(struct mpi3mr_ioc *mrioc,
5092         struct mpi3_config_request *cfg_req,
5093         struct mpi3_config_page_header *cfg_hdr, int timeout, u16 *ioc_status,
5094         void *cfg_buf, u32 cfg_buf_sz)
5095 {
5096         struct dma_memory_desc mem_desc;
5097         int retval = -1;
5098         u8 invalid_action = 0;
5099         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
5100
5101         memset(&mem_desc, 0, sizeof(struct dma_memory_desc));
5102
5103         if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)
5104                 mem_desc.size = sizeof(struct mpi3_config_page_header);
5105         else {
5106                 if (!cfg_hdr) {
5107                         ioc_err(mrioc, "null config header passed for config action(%d), page_type(0x%02x), page_num(%d)\n",
5108                             cfg_req->action, cfg_req->page_type,
5109                             cfg_req->page_number);
5110                         goto out;
5111                 }
5112                 switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {
5113                 case MPI3_CONFIG_PAGEATTR_READ_ONLY:
5114                         if (cfg_req->action
5115                             != MPI3_CONFIG_ACTION_READ_CURRENT)
5116                                 invalid_action = 1;
5117                         break;
5118                 case MPI3_CONFIG_PAGEATTR_CHANGEABLE:
5119                         if ((cfg_req->action ==
5120                              MPI3_CONFIG_ACTION_READ_PERSISTENT) ||
5121                             (cfg_req->action ==
5122                              MPI3_CONFIG_ACTION_WRITE_PERSISTENT))
5123                                 invalid_action = 1;
5124                         break;
5125                 case MPI3_CONFIG_PAGEATTR_PERSISTENT:
5126                 default:
5127                         break;
5128                 }
5129                 if (invalid_action) {
5130                         ioc_err(mrioc,
5131                             "config action(%d) is not allowed for page_type(0x%02x), page_num(%d) with page_attribute(0x%02x)\n",
5132                             cfg_req->action, cfg_req->page_type,
5133                             cfg_req->page_number, cfg_hdr->page_attribute);
5134                         goto out;
5135                 }
5136                 mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;
5137                 cfg_req->page_length = cfg_hdr->page_length;
5138                 cfg_req->page_version = cfg_hdr->page_version;
5139         }
5140         if (mpi3mr_alloc_config_dma_memory(mrioc, &mem_desc))
5141                 goto out;
5142
5143         mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,
5144             mem_desc.dma_addr);
5145
5146         if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||
5147             (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5148                 memcpy(mem_desc.addr, cfg_buf, min_t(u16, mem_desc.size,
5149                     cfg_buf_sz));
5150                 dprint_cfg_info(mrioc, "config buffer to be written\n");
5151                 if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5152                         dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");
5153         }
5154
5155         if (mpi3mr_post_cfg_req(mrioc, cfg_req, timeout, ioc_status))
5156                 goto out;
5157
5158         retval = 0;
5159         if ((*ioc_status == MPI3_IOCSTATUS_SUCCESS) &&
5160             (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&
5161             (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5162                 memcpy(cfg_buf, mem_desc.addr, min_t(u16, mem_desc.size,
5163                     cfg_buf_sz));
5164                 dprint_cfg_info(mrioc, "config buffer read\n");
5165                 if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5166                         dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");
5167         }
5168
5169 out:
5170         mpi3mr_free_config_dma_memory(mrioc, &mem_desc);
5171         return retval;
5172 }
5173
5174 /**
5175  * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5176  * @mrioc: Adapter instance reference
5177  * @ioc_status: Pointer to return ioc status
5178  * @dev_pg0: Pointer to return device page 0
5179  * @pg_sz: Size of the memory allocated to the page pointer
5180  * @form: The form to be used for addressing the page
5181  * @form_spec: Form specific information like device handle
5182  *
5183  * This is handler for config page read for a specific device
5184  * page0. The ioc_status has the controller returned ioc_status.
5185  * This routine doesn't check ioc_status to decide whether the
5186  * page read is success or not and it is the callers
5187  * responsibility.
5188  *
5189  * Return: 0 on success, non-zero on failure.
5190  */
5191 int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5192         struct mpi3_device_page0 *dev_pg0, u16 pg_sz, u32 form, u32 form_spec)
5193 {
5194         struct mpi3_config_page_header cfg_hdr;
5195         struct mpi3_config_request cfg_req;
5196         u32 page_address;
5197
5198         memset(dev_pg0, 0, pg_sz);
5199         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5200         memset(&cfg_req, 0, sizeof(cfg_req));
5201
5202         cfg_req.function = MPI3_FUNCTION_CONFIG;
5203         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5204         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DEVICE;
5205         cfg_req.page_number = 0;
5206         cfg_req.page_address = 0;
5207
5208         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5209             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5210                 ioc_err(mrioc, "device page0 header read failed\n");
5211                 goto out_failed;
5212         }
5213         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5214                 ioc_err(mrioc, "device page0 header read failed with ioc_status(0x%04x)\n",
5215                     *ioc_status);
5216                 goto out_failed;
5217         }
5218         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5219         page_address = ((form & MPI3_DEVICE_PGAD_FORM_MASK) |
5220             (form_spec & MPI3_DEVICE_PGAD_HANDLE_MASK));
5221         cfg_req.page_address = cpu_to_le32(page_address);
5222         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5223             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, dev_pg0, pg_sz)) {
5224                 ioc_err(mrioc, "device page0 read failed\n");
5225                 goto out_failed;
5226         }
5227         return 0;
5228 out_failed:
5229         return -1;
5230 }
5231
5232
5233 /**
5234  * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5235  * @mrioc: Adapter instance reference
5236  * @ioc_status: Pointer to return ioc status
5237  * @phy_pg0: Pointer to return SAS Phy page 0
5238  * @pg_sz: Size of the memory allocated to the page pointer
5239  * @form: The form to be used for addressing the page
5240  * @form_spec: Form specific information like phy number
5241  *
5242  * This is handler for config page read for a specific SAS Phy
5243  * page0. The ioc_status has the controller returned ioc_status.
5244  * This routine doesn't check ioc_status to decide whether the
5245  * page read is success or not and it is the callers
5246  * responsibility.
5247  *
5248  * Return: 0 on success, non-zero on failure.
5249  */
5250 int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5251         struct mpi3_sas_phy_page0 *phy_pg0, u16 pg_sz, u32 form,
5252         u32 form_spec)
5253 {
5254         struct mpi3_config_page_header cfg_hdr;
5255         struct mpi3_config_request cfg_req;
5256         u32 page_address;
5257
5258         memset(phy_pg0, 0, pg_sz);
5259         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5260         memset(&cfg_req, 0, sizeof(cfg_req));
5261
5262         cfg_req.function = MPI3_FUNCTION_CONFIG;
5263         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5264         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5265         cfg_req.page_number = 0;
5266         cfg_req.page_address = 0;
5267
5268         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5269             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5270                 ioc_err(mrioc, "sas phy page0 header read failed\n");
5271                 goto out_failed;
5272         }
5273         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5274                 ioc_err(mrioc, "sas phy page0 header read failed with ioc_status(0x%04x)\n",
5275                     *ioc_status);
5276                 goto out_failed;
5277         }
5278         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5279         page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5280             (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5281         cfg_req.page_address = cpu_to_le32(page_address);
5282         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5283             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg0, pg_sz)) {
5284                 ioc_err(mrioc, "sas phy page0 read failed\n");
5285                 goto out_failed;
5286         }
5287         return 0;
5288 out_failed:
5289         return -1;
5290 }
5291
5292 /**
5293  * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5294  * @mrioc: Adapter instance reference
5295  * @ioc_status: Pointer to return ioc status
5296  * @phy_pg1: Pointer to return SAS Phy page 1
5297  * @pg_sz: Size of the memory allocated to the page pointer
5298  * @form: The form to be used for addressing the page
5299  * @form_spec: Form specific information like phy number
5300  *
5301  * This is handler for config page read for a specific SAS Phy
5302  * page1. The ioc_status has the controller returned ioc_status.
5303  * This routine doesn't check ioc_status to decide whether the
5304  * page read is success or not and it is the callers
5305  * responsibility.
5306  *
5307  * Return: 0 on success, non-zero on failure.
5308  */
5309 int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5310         struct mpi3_sas_phy_page1 *phy_pg1, u16 pg_sz, u32 form,
5311         u32 form_spec)
5312 {
5313         struct mpi3_config_page_header cfg_hdr;
5314         struct mpi3_config_request cfg_req;
5315         u32 page_address;
5316
5317         memset(phy_pg1, 0, pg_sz);
5318         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5319         memset(&cfg_req, 0, sizeof(cfg_req));
5320
5321         cfg_req.function = MPI3_FUNCTION_CONFIG;
5322         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5323         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5324         cfg_req.page_number = 1;
5325         cfg_req.page_address = 0;
5326
5327         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5328             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5329                 ioc_err(mrioc, "sas phy page1 header read failed\n");
5330                 goto out_failed;
5331         }
5332         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5333                 ioc_err(mrioc, "sas phy page1 header read failed with ioc_status(0x%04x)\n",
5334                     *ioc_status);
5335                 goto out_failed;
5336         }
5337         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5338         page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5339             (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5340         cfg_req.page_address = cpu_to_le32(page_address);
5341         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5342             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg1, pg_sz)) {
5343                 ioc_err(mrioc, "sas phy page1 read failed\n");
5344                 goto out_failed;
5345         }
5346         return 0;
5347 out_failed:
5348         return -1;
5349 }
5350
5351
5352 /**
5353  * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5354  * @mrioc: Adapter instance reference
5355  * @ioc_status: Pointer to return ioc status
5356  * @exp_pg0: Pointer to return SAS Expander page 0
5357  * @pg_sz: Size of the memory allocated to the page pointer
5358  * @form: The form to be used for addressing the page
5359  * @form_spec: Form specific information like device handle
5360  *
5361  * This is handler for config page read for a specific SAS
5362  * Expander page0. The ioc_status has the controller returned
5363  * ioc_status. This routine doesn't check ioc_status to decide
5364  * whether the page read is success or not and it is the callers
5365  * responsibility.
5366  *
5367  * Return: 0 on success, non-zero on failure.
5368  */
5369 int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5370         struct mpi3_sas_expander_page0 *exp_pg0, u16 pg_sz, u32 form,
5371         u32 form_spec)
5372 {
5373         struct mpi3_config_page_header cfg_hdr;
5374         struct mpi3_config_request cfg_req;
5375         u32 page_address;
5376
5377         memset(exp_pg0, 0, pg_sz);
5378         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5379         memset(&cfg_req, 0, sizeof(cfg_req));
5380
5381         cfg_req.function = MPI3_FUNCTION_CONFIG;
5382         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5383         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5384         cfg_req.page_number = 0;
5385         cfg_req.page_address = 0;
5386
5387         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5388             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5389                 ioc_err(mrioc, "expander page0 header read failed\n");
5390                 goto out_failed;
5391         }
5392         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5393                 ioc_err(mrioc, "expander page0 header read failed with ioc_status(0x%04x)\n",
5394                     *ioc_status);
5395                 goto out_failed;
5396         }
5397         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5398         page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5399             (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5400             MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5401         cfg_req.page_address = cpu_to_le32(page_address);
5402         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5403             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg0, pg_sz)) {
5404                 ioc_err(mrioc, "expander page0 read failed\n");
5405                 goto out_failed;
5406         }
5407         return 0;
5408 out_failed:
5409         return -1;
5410 }
5411
5412 /**
5413  * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5414  * @mrioc: Adapter instance reference
5415  * @ioc_status: Pointer to return ioc status
5416  * @exp_pg1: Pointer to return SAS Expander page 1
5417  * @pg_sz: Size of the memory allocated to the page pointer
5418  * @form: The form to be used for addressing the page
5419  * @form_spec: Form specific information like phy number
5420  *
5421  * This is handler for config page read for a specific SAS
5422  * Expander page1. The ioc_status has the controller returned
5423  * ioc_status. This routine doesn't check ioc_status to decide
5424  * whether the page read is success or not and it is the callers
5425  * responsibility.
5426  *
5427  * Return: 0 on success, non-zero on failure.
5428  */
5429 int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5430         struct mpi3_sas_expander_page1 *exp_pg1, u16 pg_sz, u32 form,
5431         u32 form_spec)
5432 {
5433         struct mpi3_config_page_header cfg_hdr;
5434         struct mpi3_config_request cfg_req;
5435         u32 page_address;
5436
5437         memset(exp_pg1, 0, pg_sz);
5438         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5439         memset(&cfg_req, 0, sizeof(cfg_req));
5440
5441         cfg_req.function = MPI3_FUNCTION_CONFIG;
5442         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5443         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5444         cfg_req.page_number = 1;
5445         cfg_req.page_address = 0;
5446
5447         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5448             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5449                 ioc_err(mrioc, "expander page1 header read failed\n");
5450                 goto out_failed;
5451         }
5452         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5453                 ioc_err(mrioc, "expander page1 header read failed with ioc_status(0x%04x)\n",
5454                     *ioc_status);
5455                 goto out_failed;
5456         }
5457         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5458         page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5459             (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5460             MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5461         cfg_req.page_address = cpu_to_le32(page_address);
5462         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5463             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg1, pg_sz)) {
5464                 ioc_err(mrioc, "expander page1 read failed\n");
5465                 goto out_failed;
5466         }
5467         return 0;
5468 out_failed:
5469         return -1;
5470 }
5471
5472 /**
5473  * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
5474  * @mrioc: Adapter instance reference
5475  * @ioc_status: Pointer to return ioc status
5476  * @encl_pg0: Pointer to return Enclosure page 0
5477  * @pg_sz: Size of the memory allocated to the page pointer
5478  * @form: The form to be used for addressing the page
5479  * @form_spec: Form specific information like device handle
5480  *
5481  * This is handler for config page read for a specific Enclosure
5482  * page0. The ioc_status has the controller returned ioc_status.
5483  * This routine doesn't check ioc_status to decide whether the
5484  * page read is success or not and it is the callers
5485  * responsibility.
5486  *
5487  * Return: 0 on success, non-zero on failure.
5488  */
5489 int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5490         struct mpi3_enclosure_page0 *encl_pg0, u16 pg_sz, u32 form,
5491         u32 form_spec)
5492 {
5493         struct mpi3_config_page_header cfg_hdr;
5494         struct mpi3_config_request cfg_req;
5495         u32 page_address;
5496
5497         memset(encl_pg0, 0, pg_sz);
5498         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5499         memset(&cfg_req, 0, sizeof(cfg_req));
5500
5501         cfg_req.function = MPI3_FUNCTION_CONFIG;
5502         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5503         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_ENCLOSURE;
5504         cfg_req.page_number = 0;
5505         cfg_req.page_address = 0;
5506
5507         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5508             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5509                 ioc_err(mrioc, "enclosure page0 header read failed\n");
5510                 goto out_failed;
5511         }
5512         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5513                 ioc_err(mrioc, "enclosure page0 header read failed with ioc_status(0x%04x)\n",
5514                     *ioc_status);
5515                 goto out_failed;
5516         }
5517         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5518         page_address = ((form & MPI3_ENCLOS_PGAD_FORM_MASK) |
5519             (form_spec & MPI3_ENCLOS_PGAD_HANDLE_MASK));
5520         cfg_req.page_address = cpu_to_le32(page_address);
5521         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5522             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, encl_pg0, pg_sz)) {
5523                 ioc_err(mrioc, "enclosure page0 read failed\n");
5524                 goto out_failed;
5525         }
5526         return 0;
5527 out_failed:
5528         return -1;
5529 }
5530
5531
5532 /**
5533  * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
5534  * @mrioc: Adapter instance reference
5535  * @sas_io_unit_pg0: Pointer to return SAS IO Unit page 0
5536  * @pg_sz: Size of the memory allocated to the page pointer
5537  *
5538  * This is handler for config page read for the SAS IO Unit
5539  * page0. This routine checks ioc_status to decide whether the
5540  * page read is success or not.
5541  *
5542  * Return: 0 on success, non-zero on failure.
5543  */
5544 int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc,
5545         struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0, u16 pg_sz)
5546 {
5547         struct mpi3_config_page_header cfg_hdr;
5548         struct mpi3_config_request cfg_req;
5549         u16 ioc_status = 0;
5550
5551         memset(sas_io_unit_pg0, 0, pg_sz);
5552         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5553         memset(&cfg_req, 0, sizeof(cfg_req));
5554
5555         cfg_req.function = MPI3_FUNCTION_CONFIG;
5556         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5557         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5558         cfg_req.page_number = 0;
5559         cfg_req.page_address = 0;
5560
5561         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5562             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5563                 ioc_err(mrioc, "sas io unit page0 header read failed\n");
5564                 goto out_failed;
5565         }
5566         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5567                 ioc_err(mrioc, "sas io unit page0 header read failed with ioc_status(0x%04x)\n",
5568                     ioc_status);
5569                 goto out_failed;
5570         }
5571         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5572
5573         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5574             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg0, pg_sz)) {
5575                 ioc_err(mrioc, "sas io unit page0 read failed\n");
5576                 goto out_failed;
5577         }
5578         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5579                 ioc_err(mrioc, "sas io unit page0 read failed with ioc_status(0x%04x)\n",
5580                     ioc_status);
5581                 goto out_failed;
5582         }
5583         return 0;
5584 out_failed:
5585         return -1;
5586 }
5587
5588 /**
5589  * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
5590  * @mrioc: Adapter instance reference
5591  * @sas_io_unit_pg1: Pointer to return SAS IO Unit page 1
5592  * @pg_sz: Size of the memory allocated to the page pointer
5593  *
5594  * This is handler for config page read for the SAS IO Unit
5595  * page1. This routine checks ioc_status to decide whether the
5596  * page read is success or not.
5597  *
5598  * Return: 0 on success, non-zero on failure.
5599  */
5600 int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5601         struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5602 {
5603         struct mpi3_config_page_header cfg_hdr;
5604         struct mpi3_config_request cfg_req;
5605         u16 ioc_status = 0;
5606
5607         memset(sas_io_unit_pg1, 0, pg_sz);
5608         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5609         memset(&cfg_req, 0, sizeof(cfg_req));
5610
5611         cfg_req.function = MPI3_FUNCTION_CONFIG;
5612         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5613         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5614         cfg_req.page_number = 1;
5615         cfg_req.page_address = 0;
5616
5617         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5618             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5619                 ioc_err(mrioc, "sas io unit page1 header read failed\n");
5620                 goto out_failed;
5621         }
5622         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5623                 ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5624                     ioc_status);
5625                 goto out_failed;
5626         }
5627         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5628
5629         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5630             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5631                 ioc_err(mrioc, "sas io unit page1 read failed\n");
5632                 goto out_failed;
5633         }
5634         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5635                 ioc_err(mrioc, "sas io unit page1 read failed with ioc_status(0x%04x)\n",
5636                     ioc_status);
5637                 goto out_failed;
5638         }
5639         return 0;
5640 out_failed:
5641         return -1;
5642 }
5643
5644 /**
5645  * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
5646  * @mrioc: Adapter instance reference
5647  * @sas_io_unit_pg1: Pointer to the SAS IO Unit page 1 to write
5648  * @pg_sz: Size of the memory allocated to the page pointer
5649  *
5650  * This is handler for config page write for the SAS IO Unit
5651  * page1. This routine checks ioc_status to decide whether the
5652  * page read is success or not. This will modify both current
5653  * and persistent page.
5654  *
5655  * Return: 0 on success, non-zero on failure.
5656  */
5657 int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5658         struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5659 {
5660         struct mpi3_config_page_header cfg_hdr;
5661         struct mpi3_config_request cfg_req;
5662         u16 ioc_status = 0;
5663
5664         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5665         memset(&cfg_req, 0, sizeof(cfg_req));
5666
5667         cfg_req.function = MPI3_FUNCTION_CONFIG;
5668         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5669         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5670         cfg_req.page_number = 1;
5671         cfg_req.page_address = 0;
5672
5673         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5674             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5675                 ioc_err(mrioc, "sas io unit page1 header read failed\n");
5676                 goto out_failed;
5677         }
5678         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5679                 ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5680                     ioc_status);
5681                 goto out_failed;
5682         }
5683         cfg_req.action = MPI3_CONFIG_ACTION_WRITE_CURRENT;
5684
5685         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5686             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5687                 ioc_err(mrioc, "sas io unit page1 write current failed\n");
5688                 goto out_failed;
5689         }
5690         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5691                 ioc_err(mrioc, "sas io unit page1 write current failed with ioc_status(0x%04x)\n",
5692                     ioc_status);
5693                 goto out_failed;
5694         }
5695
5696         cfg_req.action = MPI3_CONFIG_ACTION_WRITE_PERSISTENT;
5697
5698         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5699             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5700                 ioc_err(mrioc, "sas io unit page1 write persistent failed\n");
5701                 goto out_failed;
5702         }
5703         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5704                 ioc_err(mrioc, "sas io unit page1 write persistent failed with ioc_status(0x%04x)\n",
5705                     ioc_status);
5706                 goto out_failed;
5707         }
5708         return 0;
5709 out_failed:
5710         return -1;
5711 }
5712
5713 /**
5714  * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
5715  * @mrioc: Adapter instance reference
5716  * @driver_pg1: Pointer to return Driver page 1
5717  * @pg_sz: Size of the memory allocated to the page pointer
5718  *
5719  * This is handler for config page read for the Driver page1.
5720  * This routine checks ioc_status to decide whether the page
5721  * read is success or not.
5722  *
5723  * Return: 0 on success, non-zero on failure.
5724  */
5725 int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc,
5726         struct mpi3_driver_page1 *driver_pg1, u16 pg_sz)
5727 {
5728         struct mpi3_config_page_header cfg_hdr;
5729         struct mpi3_config_request cfg_req;
5730         u16 ioc_status = 0;
5731
5732         memset(driver_pg1, 0, pg_sz);
5733         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5734         memset(&cfg_req, 0, sizeof(cfg_req));
5735
5736         cfg_req.function = MPI3_FUNCTION_CONFIG;
5737         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5738         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DRIVER;
5739         cfg_req.page_number = 1;
5740         cfg_req.page_address = 0;
5741
5742         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5743             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5744                 ioc_err(mrioc, "driver page1 header read failed\n");
5745                 goto out_failed;
5746         }
5747         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5748                 ioc_err(mrioc, "driver page1 header read failed with ioc_status(0x%04x)\n",
5749                     ioc_status);
5750                 goto out_failed;
5751         }
5752         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5753
5754         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5755             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, driver_pg1, pg_sz)) {
5756                 ioc_err(mrioc, "driver page1 read failed\n");
5757                 goto out_failed;
5758         }
5759         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5760                 ioc_err(mrioc, "driver page1 read failed with ioc_status(0x%04x)\n",
5761                     ioc_status);
5762                 goto out_failed;
5763         }
5764         return 0;
5765 out_failed:
5766         return -1;
5767 }